From 891a0e48949f1dd11f0af5dbb01f85290d6b844e Mon Sep 17 00:00:00 2001 From: Bennu Date: Thu, 13 Jul 2023 18:56:54 +0800 Subject: [PATCH 001/327] Create OWNERS --- OWNERS | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 OWNERS diff --git a/OWNERS b/OWNERS new file mode 100644 index 000000000..0997b8dda --- /dev/null +++ b/OWNERS @@ -0,0 +1,6 @@ +filters: + ".*": + approvers: + - XuanYang-cn + reviewers: + - XuanYang-cn From def30674e02a804e1857e00c1c5ac329376170e3 Mon Sep 17 00:00:00 2001 From: "min.tian" Date: Mon, 24 Jul 2023 11:49:08 +0800 Subject: [PATCH 002/327] add recall data for leaderboard Signed-off-by: min.tian --- vectordb_bench/results/getLeaderboardData.py | 1 + vectordb_bench/results/leaderboard.json | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/vectordb_bench/results/getLeaderboardData.py b/vectordb_bench/results/getLeaderboardData.py index 2850c164b..645e75593 100644 --- a/vectordb_bench/results/getLeaderboardData.py +++ b/vectordb_bench/results/getLeaderboardData.py @@ -29,6 +29,7 @@ def main(): "case": d.task_config.case_config.case_id.case_name, "qps": d.metrics.qps, "latency": d.metrics.serial_latency_p99, + "recall": d.metrics.recall, "label": taskLabelToCode[d.label], } for d in results diff --git a/vectordb_bench/results/leaderboard.json b/vectordb_bench/results/leaderboard.json index a40459896..9bbbacc0c 100644 --- a/vectordb_bench/results/leaderboard.json +++ b/vectordb_bench/results/leaderboard.json @@ -1 +1 @@ -[{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":15.2269,"latency":861.8,"label":1,"qp$":31.76903818068016},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":15.1749,"latency":774.3,"label":1,"qp$":31.660546630502814},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":27.6181,"latency":305.5,"label":1,"qp$":57.621740037554765},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"label":-2,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":100.6667,"latency":21.1,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":101.1399,"latency":19.7,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":52.2606,"latency":18.3,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":61.0661,"latency":49.8,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":58.9326,"latency":44.6,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":42.5977,"latency":54.9,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":536.0726,"latency":8.200000000000001,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":467.179,"latency":7.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":431.7512,"latency":8.3,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":330.0144,"latency":9.0,"label":1,"qp$":2075.5622641509435},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":271.6585,"latency":10.1,"label":1,"qp$":1708.5440251572327},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":216.5226,"latency":12.9,"label":1,"qp$":1361.777358490566},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":123.9553,"latency":23.0,"label":1,"qp$":389.79654088050313},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":59.1479,"latency":44.5,"label":1,"qp$":185.99968553459118},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":40.999,"latency":55.300000000000004,"label":1,"qp$":128.92767295597486},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":579.9416,"latency":9.4,"label":1,"qp$":1823.71572327044},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":425.2529,"latency":11.299999999999999,"label":1,"qp$":1337.2732704402515},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":397.0539,"latency":13.799999999999999,"label":1,"qp$":1248.5971698113208},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":67.9121,"latency":179.5,"label":1,"qp$":2.083193251533742},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.7636,"latency":1921.3,"label":1,"qp$":0.02342331288343558},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":32.0,"latency":124.5,"label":1,"qp$":0.9815950920245399},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":63.1365,"latency":145.7,"label":1,"qp$":6.251138613861386},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.7512,"latency":1983.8,"label":1,"qp$":0.07437623762376237},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":30.1358,"latency":129.8,"label":1,"qp$":2.983742574257426},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":516.27,"latency":7.0,"label":1,"qp$":3246.9811320754716},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":354.8416,"latency":10.0,"label":1,"qp$":2231.708176100629},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":427.5229,"latency":8.7,"label":1,"qp$":2688.8232704402517},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":2884.689,"latency":5.3,"label":1,"qp$":2267.837264150943},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":1689.5799,"latency":6.6,"label":1,"qp$":1328.2860849056603},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1517.6792,"latency":10.0,"label":1,"qp$":1193.1440251572328},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":822.5318,"latency":5.6,"label":1,"qp$":646.6444968553459},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":378.9146,"latency":10.3,"label":1,"qp$":297.8888364779874},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":218.6854,"latency":16.2,"label":1,"qp$":171.92248427672953},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":274.5407,"latency":4.8999999999999995,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":236.5672,"latency":10.3,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":309.4833,"latency":4.3,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":178.6585,"latency":13.700000000000001,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":178.3732,"latency":15.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":229.3526,"latency":12.5,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":1258.7043,"latency":4.8999999999999995,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":1075.8776,"latency":5.3,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1494.8493,"latency":4.7,"label":1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":537.4975,"latency":18.9,"label":1,"qp$":376.9267180925666},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":372.0466,"latency":17.8,"label":1,"qp$":260.9022440392707},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1156.2898,"latency":14.4,"label":1,"qp$":810.8624123422161},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":110.248,"latency":69.0,"label":1,"qp$":77.31276297335204},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":87.2601,"latency":27.799999999999997,"label":1,"qp$":61.1922159887798},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":125.7846,"latency":23.099999999999998,"label":1,"qp$":88.20799438990183},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":240.7209,"latency":17.4,"label":1,"qp$":844.0424263674614},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":189.4399,"latency":17.5,"label":1,"qp$":664.2352734922861},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":313.5116,"latency":16.1,"label":1,"qp$":1099.2692847124824},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":18.7634,"latency":153.70000000000002,"label":1,"qp$":192.84069886947586},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":18.3619,"latency":79.8,"label":1,"qp$":188.7142857142857},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":25.2744,"latency":61.199999999999996,"label":1,"qp$":259.7574511819116},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":261.798,"latency":23.099999999999998,"label":1,"qp$":1793.13698630137},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":166.1851,"latency":23.900000000000002,"label":1,"qp$":1138.2541095890413},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":121.7169,"latency":29.0,"label":1,"qp$":833.677397260274},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":46.6189,"latency":43.1,"label":1,"qp$":479.1253854059609},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":42.4856,"latency":44.0,"label":1,"qp$":436.6454265159301},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":138.9479,"latency":26.200000000000003,"label":1,"qp$":1428.0359712230218},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":20.7437,"latency":75.80000000000001,"label":1,"qp$":106.37794871794871},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":20.2993,"latency":76.5,"label":1,"qp$":104.09897435897435},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":26.4719,"latency":67.0,"label":1,"qp$":135.75333333333333},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":8.6675,"latency":180.2,"label":1,"qp$":44.44871794871795},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":7.8121,"latency":167.7,"label":1,"qp$":40.06205128205128},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":16.869,"latency":87.8,"label":1,"qp$":86.50769230769231},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":365.0835,"latency":23.599999999999998,"label":1,"qp$":312.57148972602744},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":325.5271,"latency":25.1,"label":1,"qp$":278.7047089041096},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":596.7942,"latency":24.2,"label":1,"qp$":510.95393835616443},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":303.2044,"latency":27.400000000000002,"label":1,"qp$":259.5928082191781},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":136.0345,"latency":31.9,"label":1,"qp$":116.46789383561645},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":66.7221,"latency":42.1,"label":1,"qp$":57.125085616438355},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":131.2549,"latency":30.200000000000003,"label":1,"qp$":168.49152759948652},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":127.9337,"latency":30.099999999999998,"label":1,"qp$":164.2281129653402},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":595.8462,"latency":23.400000000000002,"label":1,"qp$":764.8860077021822},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-1node","db_name":"PgVector-2c8g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":10.6271,"latency":730.7,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-1node","db_name":"PgVector-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":10.8507,"latency":733.1999999999999,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-1node","db_name":"PgVector-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":75.7055,"latency":121.2,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-1node","db_name":"PgVector-2c8g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-1node","db_name":"PgVector-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-1node","db_name":"PgVector-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"label":-1,"qp$":0.0}] \ No newline at end of file +[{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":15.2269,"latency":861.8,"recall":0.9888,"label":1,"qp$":31.76903818068016},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":15.1749,"latency":774.3,"recall":0.989,"label":1,"qp$":31.660546630502814},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":27.6181,"latency":305.5,"recall":0.9999,"label":1,"qp$":57.621740037554765},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":100.6667,"latency":21.1,"recall":0.9909,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":101.1399,"latency":19.7,"recall":0.9907,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":52.2606,"latency":18.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":61.0661,"latency":49.8,"recall":0.9911,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":58.9326,"latency":44.6,"recall":0.9911,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":42.5977,"latency":54.9,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":536.0726,"latency":8.200000000000001,"recall":0.9728,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":467.179,"latency":7.0,"recall":0.9697,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":431.7512,"latency":8.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":330.0144,"latency":9.0,"recall":0.9507,"label":1,"qp$":2075.5622641509435},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":271.6585,"latency":10.1,"recall":0.9678,"label":1,"qp$":1708.5440251572327},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":216.5226,"latency":12.9,"recall":1.0,"label":1,"qp$":1361.777358490566},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":123.9553,"latency":23.0,"recall":0.971,"label":1,"qp$":389.79654088050313},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":59.1479,"latency":44.5,"recall":0.9906,"label":1,"qp$":185.99968553459118},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":40.999,"latency":55.300000000000004,"recall":1.0,"label":1,"qp$":128.92767295597486},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":579.9416,"latency":9.4,"recall":0.9213,"label":1,"qp$":1823.71572327044},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":425.2529,"latency":11.299999999999999,"recall":0.9686,"label":1,"qp$":1337.2732704402515},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":397.0539,"latency":13.799999999999999,"recall":1.0,"label":1,"qp$":1248.5971698113208},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":67.9121,"latency":179.5,"recall":0.9909,"label":1,"qp$":2.083193251533742},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.7636,"latency":1921.3,"recall":0.9908,"label":1,"qp$":0.02342331288343558},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":32.0,"latency":124.5,"recall":1.0,"label":1,"qp$":0.9815950920245399},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":63.1365,"latency":145.7,"recall":0.991,"label":1,"qp$":6.251138613861386},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.7512,"latency":1983.8,"recall":0.9908,"label":1,"qp$":0.07437623762376237},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":30.1358,"latency":129.8,"recall":1.0,"label":1,"qp$":2.983742574257426},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":516.27,"latency":7.0,"recall":0.9463,"label":1,"qp$":3246.9811320754716},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":354.8416,"latency":10.0,"recall":0.9802,"label":1,"qp$":2231.708176100629},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":427.5229,"latency":8.7,"recall":1.0,"label":1,"qp$":2688.8232704402517},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":2884.689,"latency":5.3,"recall":0.8801,"label":1,"qp$":2267.837264150943},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":1689.5799,"latency":6.6,"recall":0.9493,"label":1,"qp$":1328.2860849056603},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1517.6792,"latency":10.0,"recall":1.0,"label":1,"qp$":1193.1440251572328},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":822.5318,"latency":5.6,"recall":0.9294,"label":1,"qp$":646.6444968553459},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":378.9146,"latency":10.3,"recall":0.9758,"label":1,"qp$":297.8888364779874},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":218.6854,"latency":16.2,"recall":1.0,"label":1,"qp$":171.92248427672953},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":274.5407,"latency":4.8999999999999995,"recall":0.9807,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":236.5672,"latency":10.3,"recall":0.981,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":309.4833,"latency":4.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":178.6585,"latency":13.700000000000001,"recall":0.9843,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":178.3732,"latency":15.0,"recall":0.9844,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":229.3526,"latency":12.5,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":1258.7043,"latency":4.8999999999999995,"recall":0.9799,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":1075.8776,"latency":5.3,"recall":0.98,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1494.8493,"latency":4.7,"recall":1.0,"label":1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":537.4975,"latency":18.9,"recall":0.8903,"label":1,"qp$":376.9267180925666},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":372.0466,"latency":17.8,"recall":0.8904,"label":1,"qp$":260.9022440392707},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1156.2898,"latency":14.4,"recall":0.9989,"label":1,"qp$":810.8624123422161},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":110.248,"latency":69.0,"recall":0.898,"label":1,"qp$":77.31276297335204},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":87.2601,"latency":27.799999999999997,"recall":0.898,"label":1,"qp$":61.1922159887798},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":125.7846,"latency":23.099999999999998,"recall":0.975,"label":1,"qp$":88.20799438990183},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":240.7209,"latency":17.4,"recall":0.8887,"label":1,"qp$":844.0424263674614},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":189.4399,"latency":17.5,"recall":0.8889,"label":1,"qp$":664.2352734922861},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":313.5116,"latency":16.1,"recall":0.9999,"label":1,"qp$":1099.2692847124824},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":18.7634,"latency":153.70000000000002,"recall":0.8737,"label":1,"qp$":192.84069886947586},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":18.3619,"latency":79.8,"recall":0.8741,"label":1,"qp$":188.7142857142857},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":25.2744,"latency":61.199999999999996,"recall":0.9979,"label":1,"qp$":259.7574511819116},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":261.798,"latency":23.099999999999998,"recall":0.9262,"label":1,"qp$":1793.13698630137},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":166.1851,"latency":23.900000000000002,"recall":0.9264,"label":1,"qp$":1138.2541095890413},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":121.7169,"latency":29.0,"recall":0.9693,"label":1,"qp$":833.677397260274},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":46.6189,"latency":43.1,"recall":0.8737,"label":1,"qp$":479.1253854059609},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":42.4856,"latency":44.0,"recall":0.8741,"label":1,"qp$":436.6454265159301},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":138.9479,"latency":26.200000000000003,"recall":0.9979,"label":1,"qp$":1428.0359712230218},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":20.7437,"latency":75.80000000000001,"recall":0.9291,"label":1,"qp$":106.37794871794871},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":20.2993,"latency":76.5,"recall":0.9293,"label":1,"qp$":104.09897435897435},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":26.4719,"latency":67.0,"recall":1.0,"label":1,"qp$":135.75333333333333},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":8.6675,"latency":180.2,"recall":0.8369,"label":1,"qp$":44.44871794871795},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":7.8121,"latency":167.7,"recall":0.8369,"label":1,"qp$":40.06205128205128},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":16.869,"latency":87.8,"recall":0.9814,"label":1,"qp$":86.50769230769231},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":365.0835,"latency":23.599999999999998,"recall":0.945,"label":1,"qp$":312.57148972602744},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":325.5271,"latency":25.1,"recall":0.9452,"label":1,"qp$":278.7047089041096},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":596.7942,"latency":24.2,"recall":0.9693,"label":1,"qp$":510.95393835616443},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":303.2044,"latency":27.400000000000002,"recall":0.9246,"label":1,"qp$":259.5928082191781},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":136.0345,"latency":31.9,"recall":0.9244,"label":1,"qp$":116.46789383561645},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":66.7221,"latency":42.1,"recall":0.963,"label":1,"qp$":57.125085616438355},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":131.2549,"latency":30.200000000000003,"recall":0.9867,"label":1,"qp$":168.49152759948652},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":127.9337,"latency":30.099999999999998,"recall":0.9869,"label":1,"qp$":164.2281129653402},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":595.8462,"latency":23.400000000000002,"recall":1.0,"label":1,"qp$":764.8860077021822},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-1node","db_name":"PgVector-2c8g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":10.6271,"latency":730.7,"recall":0.8898,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-1node","db_name":"PgVector-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":10.8507,"latency":733.1999999999999,"recall":0.8897,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-1node","db_name":"PgVector-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":75.7055,"latency":121.2,"recall":0.9999,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-1node","db_name":"PgVector-2c8g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-1node","db_name":"PgVector-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-1node","db_name":"PgVector-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0}] \ No newline at end of file From 3bb19d61a33b6970a387d926e3b5c68f2da95866 Mon Sep 17 00:00:00 2001 From: Slayden Gruneberg Date: Thu, 20 Jul 2023 11:59:01 -0700 Subject: [PATCH 003/327] chroma draft complete --- pyproject.toml | 3 +- tests/test_chroma.py | 118 +++++++++++++++ vectordb_bench/backend/clients/__init__.py | 5 +- .../backend/clients/chroma/chroma.py | 138 ++++++++++++++++++ .../backend/clients/chroma/config.py | 14 ++ vectordb_bench/frontend/const/styles.py | 1 + 6 files changed, 277 insertions(+), 2 deletions(-) create mode 100644 tests/test_chroma.py create mode 100644 vectordb_bench/backend/clients/chroma/chroma.py create mode 100644 vectordb_bench/backend/clients/chroma/config.py diff --git a/pyproject.toml b/pyproject.toml index 537a7cd7e..44074ba39 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,8 @@ dependencies = [ "psutil", "polars", "pgvector", - "sqlalchemy" + "sqlalchemy", + "chromadb", ] dynamic = ["version"] diff --git a/tests/test_chroma.py b/tests/test_chroma.py new file mode 100644 index 000000000..33536594a --- /dev/null +++ b/tests/test_chroma.py @@ -0,0 +1,118 @@ +import logging +from vectordb_bench.models import ( + DB, +) +from vectordb_bench.backend.clients.chroma.config import ChromaConfig +import numpy as np +import chromadb + + +log = logging.getLogger(__name__) + +""" Tests for Chroma, assumes Chroma is running on localhost:8000, + Chroma docs: https://docs.trychroma.com/usage-guide + To configure Chroma to run in a docker container client/server + + To get running: clone chroma repo and run docker-compose up in chroma directory: + 1. git clone chroma repo https://github.com/chroma-core/chroma + 2. cd chroma, docker-compose up -d --build # start chroma server + 3. default port is 8000, default host is localhost""" + + + +dict = {} #Assumes chroma is acception connections on localhost:8000 +dict['name'] = "chroma" +dict['host'] = "localhost" +dict['port'] = 8000 +dict['password'] = "chroma" + + + +class TestChroma: + def test_insert_and_search(self): + assert DB.Chroma.value == "Chroma" + + dbcls = DB.Chroma.init_cls + dbConfig = dbcls.config_cls() + + + dim = 16 + chrma = dbcls( + dim=dim, + db_config=dict, + db_case_config=None, + indice="example", + drop_old=True, + ) + + count = 10_000 + filter_value = 0.9 + embeddings = [[np.random.random() for _ in range(dim)] for _ in range(count)] + + + # insert + with chrma.init(): + assert (chrma.client.heartbeat() is not None), "chroma client is not connected" + res = chrma.insert_embeddings(embeddings=embeddings, metadata=range(count)) + # bulk_insert return + assert ( + res[0] == count + ), f"the return count of bulk insert ({res}) is not equal to count ({count})" + + # count entries in chroma database + countRes = chrma.collection.count() + + assert ( + countRes == count + ), f"the return count of redis client ({countRes}) is not equal to count ({count})" + + # search + with chrma.init(): + test_id = np.random.randint(count) + #log.info(f"test_id: {test_id}") + q = embeddings[test_id] + + res = chrma.search_embedding(query=q, k=100) + print(res) + assert ( + res[0] == int(test_id) + ), f"the most nearest neighbor ({res[0]}) id is not test_id ({int(test_id)}" + + + # search with filters, assumes filter format {id: int, metadata: >=int} + with chrma.init(): + filter_value = int(count * filter_value) + test_id = np.random.randint(filter_value, count) + q = embeddings[test_id] + + + res = chrma.search_embedding( + query=q, k=100, filters={"metadata": filter_value} + ) + assert ( + res[0] == int(test_id) + ), f"the most nearest neighbor ({res[0]}) id is not test_id ({test_id})" + isFilter = True + id_list = [] + for id in res: + id_list.append(id) + if int(id) < filter_value: + isFilter = False + break + assert isFilter, f"Filter not working, id_list: {id_list}" + + #Test id filter + res = chrma.search_embedding( + query=q, k=100, filters={"id": 9999} + ) + assert ( + res[0] == 9999 + ) + + #Test two filters, id and metadata + res = chrma.search_embedding( + query=q, k=100, filters={"metadata": filter_value, "id": 9999} + ) + assert ( + res[0] == 9999 and len(res) == 1, f"filters failed, got: ({res[0]}), expected ({9999})" + ) \ No newline at end of file diff --git a/vectordb_bench/backend/clients/__init__.py b/vectordb_bench/backend/clients/__init__.py index 64571473d..d54b323d4 100644 --- a/vectordb_bench/backend/clients/__init__.py +++ b/vectordb_bench/backend/clients/__init__.py @@ -16,6 +16,7 @@ from .qdrant_cloud.qdrant_cloud import QdrantCloud from .zilliz_cloud.zilliz_cloud import ZillizCloud from .pgvector.pgvector import PgVector +from .chroma.chroma import ChromaClient class DB(Enum): """Database types @@ -36,6 +37,7 @@ class DB(Enum): QdrantCloud = "QdrantCloud" WeaviateCloud = "WeaviateCloud" PgVector = "PgVector" + Chroma = "Chroma" @property @@ -50,7 +52,8 @@ def init_cls(self) -> Type[VectorDB]: DB.ElasticCloud: ElasticCloud, DB.QdrantCloud: QdrantCloud, DB.Pinecone: Pinecone, - DB.PgVector: PgVector + DB.PgVector: PgVector, + DB.Chroma: ChromaClient } for db in DB: diff --git a/vectordb_bench/backend/clients/chroma/chroma.py b/vectordb_bench/backend/clients/chroma/chroma.py new file mode 100644 index 000000000..a87ea5718 --- /dev/null +++ b/vectordb_bench/backend/clients/chroma/chroma.py @@ -0,0 +1,138 @@ +import chromadb +import logging +import numpy as np +from contextlib import contextmanager +from typing import Any, Type +from ..api import VectorDB, DBConfig, DBCaseConfig, EmptyDBCaseConfig, IndexType +from .config import ChromaConfig +from chromadb.config import Settings + +log = logging.getLogger(__name__) +class ChromaClient(VectorDB): + """Chroma client for VectorDB. + To set up Chroma in docker, see https://docs.trychroma.com/usage-guide + or the instructions in tests/test_chroma.py + + To change to running in process, modify the HttpClient() in __init__() and init(). + """ + + def __init__( + self, + dim: int, + db_config: dict, + db_case_config: DBCaseConfig, + drop_old: bool = False, + + **kwargs + ): + + self.db_config = db_config + self.case_config = db_case_config + self.collection_name = 'example2' + + client = chromadb.HttpClient(host=self.db_config["host"], + port=self.db_config["port"]) + assert client.heartbeat() is not None + if drop_old: + try: + client.reset() # Reset the database + except: + drop_old = False + log.info(f"Chroma client drop_old collection: {self.collection_name}") + + + def config_cls() -> Type[DBConfig]: + return ChromaConfig + + def case_config_cls(index_type: IndexType | None = None) -> Type[DBCaseConfig]: + return EmptyDBCaseConfig + + @contextmanager + def init(self) -> None: + """ create and destory connections to database. + + Examples: + >>> with self.init(): + >>> self.insert_embeddings() + """ + #create connection + self.client = chromadb.HttpClient(host=self.db_config["host"], + port=self.db_config["port"]) + + self.collection = self.client.get_or_create_collection('example2') + yield + self.client = None + self.collection = None + + def ready_to_search(self) -> bool: + pass + + def ready_to_load(self) -> bool: + pass + + def optimize(self) -> None: + pass + + def insert_embeddings( + self, + embeddings: list[list[float]], + metadata: list[int], + **kwargs: Any, + ) -> (int, Exception): + """Insert embeddings into the database. + + Args: + embeddings(list[list[float]]): list of embeddings + metadata(list[int]): list of metadata + kwargs: other arguments + + Returns: + (int, Exception): number of embeddings inserted and exception if any + """ + ids=[str(i) for i in metadata] + metadata = [{"id": int(i)} for i in metadata] + if len(embeddings) > 0: + self.collection.add(embeddings=embeddings, ids=ids, metadatas=metadata) + return len(embeddings), None + + def search_embedding( + self, + query: list[float], + k: int = 100, + filters: dict | None = None, + timeout: int | None = None, + **kwargs: Any, + ) -> dict: + """Search embeddings from the database. + Args: + embedding(list[float]): embedding to search + k(int): number of results to return + kwargs: other arguments + + Returns: + Dict {ids: list[list[int]], + embedding: list[list[float]] + distance: list[list[float]]} + """ + if filters: + # assumes benchmark test filters of format: {'metadata': '>=10000', 'id': 10000} + metadata_value = filters.get("metadata") + id_value = filters.get("id") + if metadata_value and id_value: + results = self.collection.query( + query_embeddings=query, n_results=k, + where={"$and": [{"id": {"$eq": id_value}}, + {"id": {"$gt": metadata_value}} + ]} + ) + elif metadata_value: + results = self.collection.query(query_embeddings=query, n_results=k, + where={"id": {"$gt": metadata_value}}) + else: + results = self.collection.query(query_embeddings=query, n_results=k, + where={"id": {"$eq": id_value}}) + #return list of id's in results + return [int(i) for i in results.get('ids')[0]] + results = self.collection.query(query_embeddings=query, n_results=k) + return [int(i) for i in results.get('ids')[0]] + \ No newline at end of file diff --git a/vectordb_bench/backend/clients/chroma/config.py b/vectordb_bench/backend/clients/chroma/config.py new file mode 100644 index 000000000..85c59c973 --- /dev/null +++ b/vectordb_bench/backend/clients/chroma/config.py @@ -0,0 +1,14 @@ +from pydantic import SecretStr +from ..api import DBConfig + +class ChromaConfig(DBConfig): + password: SecretStr + host: SecretStr + port: int + + def to_dict(self) -> dict: + return { + "host": self.host.get_secret_value(), + "port": self.port, + "password": self.password.get_secret_value(), + } \ No newline at end of file diff --git a/vectordb_bench/frontend/const/styles.py b/vectordb_bench/frontend/const/styles.py index db0d7043e..cce373a11 100644 --- a/vectordb_bench/frontend/const/styles.py +++ b/vectordb_bench/frontend/const/styles.py @@ -43,6 +43,7 @@ def getPatternShape(i): DB.QdrantCloud: "https://assets.zilliz.com/qdrant_b691674fcd.png", DB.WeaviateCloud: "https://assets.zilliz.com/weaviate_4f6f171ebe.png", DB.PgVector: "https://assets.zilliz.com/PG_Vector_d464f2ef5f.png", + DB.Chroma: "https://assets.zilliz.com/chroma_ceb3f06ed7.png", } # RedisCloud color: #0D6EFD From c8059a9a793d158fcaac270d53982ad40ebb9c7f Mon Sep 17 00:00:00 2001 From: yangxuan Date: Wed, 26 Jul 2023 16:36:26 +0800 Subject: [PATCH 004/327] Remove shared memory Signed-off-by: yangxuan --- vectordb_bench/backend/runner/mp_runner.py | 13 +++---- vectordb_bench/backend/task_runner.py | 6 ++-- vectordb_bench/backend/utils.py | 41 ---------------------- 3 files changed, 7 insertions(+), 53 deletions(-) diff --git a/vectordb_bench/backend/runner/mp_runner.py b/vectordb_bench/backend/runner/mp_runner.py index d02671a57..86e245a1a 100644 --- a/vectordb_bench/backend/runner/mp_runner.py +++ b/vectordb_bench/backend/runner/mp_runner.py @@ -4,9 +4,7 @@ import multiprocessing as mp import logging from typing import Iterable -import numpy as np from ..clients import api -from .. import utils from ... import config @@ -25,7 +23,7 @@ class MultiProcessingSearchRunner: def __init__( self, db: api.VectorDB, - test_data: np.ndarray, + test_data: list[list[float]], k: int = 100, filters: dict | None = None, concurrencies: Iterable[int] = (1, 5, 10, 15, 20, 25, 30, 35), @@ -37,17 +35,16 @@ def __init__( self.concurrencies = concurrencies self.duration = duration - self.test_data = utils.SharedNumpyArray(test_data) + self.test_data = test_data log.debug(f"test dataset columns: {len(test_data)}") - def search(self, test_np: utils.SharedNumpyArray, q: mp.Queue, cond: mp.Condition) -> tuple[int, float]: + def search(self, test_data: list[list[float]], q: mp.Queue, cond: mp.Condition) -> tuple[int, float]: # sync all process q.put(1) with cond: cond.wait() with self.db.init(): - test_data = test_np.read().tolist() num, idx = len(test_data), 0 start_time = time.perf_counter() @@ -135,6 +132,4 @@ def run(self) -> float: return self._run_all_concurrencies_mem_efficient() def stop(self) -> None: - if self.test_data: - self.test_data.unlink() - self.test_data = None + pass diff --git a/vectordb_bench/backend/task_runner.py b/vectordb_bench/backend/task_runner.py index 2ac8f335f..21ca9d28f 100644 --- a/vectordb_bench/backend/task_runner.py +++ b/vectordb_bench/backend/task_runner.py @@ -48,7 +48,7 @@ class CaseRunner(BaseModel): status: RunningStatus db: api.VectorDB | None = None - test_emb: np.ndarray | None = None + test_emb: list[list[float]] | None = None search_runner: MultiProcessingSearchRunner | None = None serial_search_runner: SerialSearchRunner | None = None @@ -210,13 +210,13 @@ def _init_search_runner(self): test_emb = np.stack(self.ca.dataset.test_data["emb"]) if self.normalize: test_emb = test_emb / np.linalg.norm(test_emb, axis=1)[:, np.newaxis] - self.test_emb = test_emb + self.test_emb = test_emb.tolist() gt_df = self.ca.dataset.get_ground_truth(self.ca.filter_rate) self.serial_search_runner = SerialSearchRunner( db=self.db, - test_data=self.test_emb.tolist(), + test_data=self.test_emb, ground_truth=gt_df, filters=self.ca.filters, ) diff --git a/vectordb_bench/backend/utils.py b/vectordb_bench/backend/utils.py index 927e997c0..d53da31a6 100644 --- a/vectordb_bench/backend/utils.py +++ b/vectordb_bench/backend/utils.py @@ -1,8 +1,5 @@ import time from functools import wraps -from multiprocessing.shared_memory import SharedMemory - -import numpy as np def numerize(n) -> str: @@ -45,41 +42,3 @@ def inner(*args, **kwargs): delta = time.perf_counter() - pref return result, delta return inner - - -class SharedNumpyArray: - ''' Wraps a numpy array so that it can be shared quickly among processes, - avoiding unnecessary copying and (de)serializing. - ''' - def __init__(self, array: np.ndarray): - ''' - Creates the shared memory and copies the array therein - ''' - # create the shared memory location of the same size of the array - self._shared = SharedMemory(create=True, size=array.nbytes) - - # save data type and shape, necessary to read the data correctly - self._dtype, self._shape = array.dtype, array.shape - - # create a new numpy array that uses the shared memory we created. - # at first, it is filled with zeros - res = np.ndarray( - self._shape, dtype=self._dtype, buffer=self._shared.buf - ) - - # copy data from the array to the shared memory. numpy will - # take care of copying everything in the correct format - res[:] = array[:] - - def read(self) -> np.ndarray: - '''Reads the array from the shared memory without unnecessary copying. ''' - # simply create an array of the correct shape and type, - # using the shared memory location we created earlier - return np.ndarray(self._shape, self._dtype, buffer=self._shared.buf) - - def unlink(self) -> None: - ''' Releases the allocated memory. Call when finished using the data, - or when the data was copied somewhere else. - ''' - self._shared.close() - self._shared.unlink() From c64db6edbda5a3417f581fde692b7b45c1b4c9a2 Mon Sep 17 00:00:00 2001 From: Slayden Gruneberg Date: Thu, 13 Jul 2023 14:27:10 -0700 Subject: [PATCH 005/327] redis draft complete --- pyproject.toml | 1 + tests/test_redis.py | 107 ++++++++++++ vectordb_bench/backend/clients/__init__.py | 3 + .../backend/clients/redis/config.py | 14 ++ vectordb_bench/backend/clients/redis/redis.py | 159 ++++++++++++++++++ vectordb_bench/frontend/const/styles.py | 2 + 6 files changed, 286 insertions(+) create mode 100644 tests/test_redis.py create mode 100644 vectordb_bench/backend/clients/redis/config.py create mode 100644 vectordb_bench/backend/clients/redis/redis.py diff --git a/pyproject.toml b/pyproject.toml index 44074ba39..7ac8855a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,7 @@ dependencies = [ "polars", "pgvector", "sqlalchemy", + "redis", "chromadb", ] dynamic = ["version"] diff --git a/tests/test_redis.py b/tests/test_redis.py new file mode 100644 index 000000000..6783ee5a9 --- /dev/null +++ b/tests/test_redis.py @@ -0,0 +1,107 @@ +import logging +from vectordb_bench.models import ( + DB, +) +from vectordb_bench.backend.clients.redis.config import RedisConfig +import numpy as np + + +log = logging.getLogger(__name__) + +# Tests for Redis, assumes Redis is running on localhost:6379, can be modified by changing the dict below +dict = {} +dict['name'] = "redis" +dict['host'] = "localhost" +dict['port'] = 6379 +dict['password'] = "redis" + + + +class TestRedis: + def test_insert_and_search(self): + assert DB.Redis.value == "Redis" + dbcls = DB.Redis.init_cls + dbConfig = dbcls.config_cls() + + + dim = 16 + rdb = dbcls( + dim=dim, + db_config=dict, + db_case_config=None, + indice="test_redis", + drop_old=True, + ) + + count = 10_000 + filter_value = 0.9 + embeddings = [[np.random.random() for _ in range(dim)] for _ in range(count)] + + + # insert + with rdb.init(): + assert (rdb.conn.ping() == True), "redis client is not connected" + res = rdb.insert_embeddings(embeddings=embeddings, metadata=range(count)) + # bulk_insert return + assert ( + res[0] == count + ), f"the return count of bulk insert ({res}) is not equal to count ({count})" + + # count entries in redis database + countRes = rdb.conn.dbsize() + + assert ( + countRes == count + ), f"the return count of redis client ({countRes}) is not equal to count ({count})" + + # search + with rdb.init(): + test_id = np.random.randint(count) + #log.info(f"test_id: {test_id}") + q = embeddings[test_id] + + res = rdb.search_embedding(query=q, k=100) + #log.info(f"search_results_id: {res}") + print(res) + # res of format [2757, 2944, 8893, 6695, 5571, 608, 455, 3464, 1584, 1807, 8452, 4311...] + assert ( + res[0] == int(test_id) + ), f"the most nearest neighbor ({res[0]}) id is not test_id ({str(test_id)}" + + # search with filters + with rdb.init(): + filter_value = int(count * filter_value) + test_id = np.random.randint(filter_value, count) + q = embeddings[test_id] + + + res = rdb.search_embedding( + query=q, k=100, filters={"metadata": filter_value} + ) + assert ( + res[0] == int(test_id) + ), f"the most nearest neighbor ({res[0]}) id is not test_id ({test_id})" + isFilter = True + id_list = [] + for id in res: + id_list.append(id) + if int(id) < filter_value: + isFilter = False + break + assert isFilter, f"filters failed, got: ({id}), expected less than ({filter_value})" + + #Test id filter for exact match + res = rdb.search_embedding( + query=q, k=100, filters={"id": 9999} + ) + assert ( + res[0] == 9999 + ) + + #Test two filters, id and metadata + res = rdb.search_embedding( + query=q, k=100, filters={"metadata": filter_value, "id": 9999} + ) + assert ( + res[0] == 9999 and len(res) == 1, f"filters failed, got: ({res[0]}), expected ({9999})" + ) \ No newline at end of file diff --git a/vectordb_bench/backend/clients/__init__.py b/vectordb_bench/backend/clients/__init__.py index d54b323d4..77cf063ea 100644 --- a/vectordb_bench/backend/clients/__init__.py +++ b/vectordb_bench/backend/clients/__init__.py @@ -16,6 +16,7 @@ from .qdrant_cloud.qdrant_cloud import QdrantCloud from .zilliz_cloud.zilliz_cloud import ZillizCloud from .pgvector.pgvector import PgVector +from .redis.redis import Redis from .chroma.chroma import ChromaClient class DB(Enum): @@ -37,6 +38,7 @@ class DB(Enum): QdrantCloud = "QdrantCloud" WeaviateCloud = "WeaviateCloud" PgVector = "PgVector" + Redis = "Redis" Chroma = "Chroma" @@ -53,6 +55,7 @@ def init_cls(self) -> Type[VectorDB]: DB.QdrantCloud: QdrantCloud, DB.Pinecone: Pinecone, DB.PgVector: PgVector, + DB.Redis: Redis, DB.Chroma: ChromaClient } diff --git a/vectordb_bench/backend/clients/redis/config.py b/vectordb_bench/backend/clients/redis/config.py new file mode 100644 index 000000000..133521e97 --- /dev/null +++ b/vectordb_bench/backend/clients/redis/config.py @@ -0,0 +1,14 @@ +from pydantic import SecretStr +from ..api import DBConfig + +class RedisConfig(DBConfig): + password: SecretStr + host: SecretStr + port: int = None + + def to_dict(self) -> dict: + return { + "host": self.host.get_secret_value(), + "port": self.port, + "password": self.password.get_secret_value(), + } \ No newline at end of file diff --git a/vectordb_bench/backend/clients/redis/redis.py b/vectordb_bench/backend/clients/redis/redis.py new file mode 100644 index 000000000..a4eb33848 --- /dev/null +++ b/vectordb_bench/backend/clients/redis/redis.py @@ -0,0 +1,159 @@ +import logging +from contextlib import contextmanager +from typing import Any, Type +from ..api import VectorDB, DBConfig, DBCaseConfig, EmptyDBCaseConfig, IndexType +from .config import RedisConfig +import redis +from redis.commands.search.field import TagField, VectorField, NumericField +from redis.commands.search.indexDefinition import IndexDefinition, IndexType +from redis.commands.search.query import Query +import numpy as np + + +log = logging.getLogger(__name__) +INDEX_NAME = "index" # Vector Index Name + +class Redis(VectorDB): + def __init__( + self, + dim: int, + db_config: dict, + db_case_config: DBCaseConfig, + drop_old: bool = False, + + **kwargs + ): + + self.db_config = db_config + self.case_config = db_case_config + self.collection_name = INDEX_NAME + + # Create a redis connection, if db has password configured, add it to the connection here and in init(): + # password=self.db_config["password"] + conn = redis.Redis(host=self.db_config["host"], port=self.db_config["port"], db=0) + + + if drop_old: + try: + conn.ft(INDEX_NAME).info() + conn.ft(INDEX_NAME).dropindex() + except redis.exceptions.ResponseError: + drop_old = False + log.info(f"Redis client drop_old collection: {self.collection_name}") + + self.make_index(dim, conn) + conn.close() + conn = None + + def make_index(self, vector_dimensions: int, conn: redis.Redis): + try: + # check to see if index exists + conn.ft(INDEX_NAME).info() + except: + schema = ( + TagField("id"), + NumericField("metadata"), + VectorField("vector", # Vector Field Name + "FLAT", { # Vector Index Type: FLAT or HNSW + "TYPE": "FLOAT32", # FLOAT32 or FLOAT64 + "DIM": vector_dimensions, # Number of Vector Dimensions + "DISTANCE_METRIC": "COSINE", # Vector Search Distance Metric + } + ), + ) + + definition = IndexDefinition(index_type=IndexType.HASH) + + rs = conn.ft(INDEX_NAME) + rs.create_index(schema, definition=definition) + + + def config_cls() -> Type[DBConfig]: + return RedisConfig + + def case_config_cls(index_type: IndexType | None = None) -> Type[DBCaseConfig]: + return EmptyDBCaseConfig + + @contextmanager + def init(self) -> None: + """ create and destory connections to database. + + Examples: + >>> with self.init(): + >>> self.insert_embeddings() + """ + self.conn = redis.Redis(host=self.db_config["host"], port=self.db_config["port"], db=0) + yield + self.conn.close() + self.conn = None + + + def ready_to_search(self) -> bool: + """Check if the database is ready to search.""" + pass + + + def ready_to_load(self) -> bool: + pass + + def optimize(self) -> None: + pass + + + def insert_embeddings( + + self, + embeddings: list[list[float]], + metadata: list[int], + **kwargs: Any, + ) -> (int, Exception): + """Insert embeddings into the database. + Should call self.init() first. + """ + try: + with self.conn.pipeline() as pipe: + for i, embedding in enumerate(embeddings): + embedding = np.array(embedding).astype(np.float32) + pipe.hset(metadata[i], mapping = { + "id": str(metadata[i]), + "metadata": metadata[i], + "vector": embedding.tobytes(), + }) + res = pipe.execute() + except Exception as e: + return 0, e + + return len(res), None + + def search_embedding( + self, + query: list[float], + k: int = 100, + filters: dict | None = None, + timeout: int | None = None, + **kwargs: Any, + ) -> (list[int]): + assert self.conn is not None + + query_vector = np.array(query).astype(np.float32).tobytes() + query_obj = Query(f"*=>[KNN {k} @vector $vec as score]").sort_by("score").return_fields("id", "score").paging(0, k).dialect(2) + query_params = {"vec": query_vector} + + if filters: + # benchmark test filters of format: {'metadata': '>=10000', 'id': 10000} + # gets exact match for id, and range for metadata if they exist in filters + id_value = filters.get("id") + metadata_value = filters.get("metadata") + if id_value and metadata_value: + query_obj = Query(f"(@metadata:[{metadata_value} +inf] @id:{ {id_value} })=>[KNN {k} @vector $vec as score]").sort_by("score").return_fields("id", "score").paging(0, k).dialect(2) + elif id_value: + #gets exact match for id + query_obj = Query(f"@id:{ {id_value} }=>[KNN {k} @vector $vec as score]").sort_by("score").return_fields("id", "score").paging(0, k).dialect(2) + else: #metadata only case, greater than or equal to metadata value + query_obj = Query(f"@metadata:[{metadata_value} +inf]=>[KNN {k} @vector $vec as score]").sort_by("score").return_fields("id", "score").paging(0, k).dialect(2) + res = self.conn.ft(INDEX_NAME).search(query_obj, query_params) + # doc in res of format {'id': '9831', 'payload': None, 'score': '1.19209289551e-07'} + return [int(doc["id"]) for doc in res.docs] + + + \ No newline at end of file diff --git a/vectordb_bench/frontend/const/styles.py b/vectordb_bench/frontend/const/styles.py index cce373a11..dd69897e8 100644 --- a/vectordb_bench/frontend/const/styles.py +++ b/vectordb_bench/frontend/const/styles.py @@ -43,6 +43,7 @@ def getPatternShape(i): DB.QdrantCloud: "https://assets.zilliz.com/qdrant_b691674fcd.png", DB.WeaviateCloud: "https://assets.zilliz.com/weaviate_4f6f171ebe.png", DB.PgVector: "https://assets.zilliz.com/PG_Vector_d464f2ef5f.png", + DB.Redis: "https://assets.zilliz.com/Redis_Cloud_74b8bfef39.png", DB.Chroma: "https://assets.zilliz.com/chroma_ceb3f06ed7.png", } @@ -56,4 +57,5 @@ def getPatternShape(i): DB.QdrantCloud.value: "#D91AD9", DB.WeaviateCloud.value: "#20C997", DB.PgVector.value: "#4C779A", + DB.Redis.value: "#0D6EFD", } From b23080f1b1f9c9db82b753ff73aa23d25346f3c1 Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Wed, 28 Jun 2023 11:01:36 -0400 Subject: [PATCH 006/327] Update elastic_cloud runner to adhere to best practices fixing search fixing conflicts --- .../clients/elastic_cloud/elastic_cloud.py | 29 +++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/vectordb_bench/backend/clients/elastic_cloud/elastic_cloud.py b/vectordb_bench/backend/clients/elastic_cloud/elastic_cloud.py index 60436dc82..8fd33de40 100644 --- a/vectordb_bench/backend/clients/elastic_cloud/elastic_cloud.py +++ b/vectordb_bench/backend/clients/elastic_cloud/elastic_cloud.py @@ -1,4 +1,5 @@ import logging +import time from contextlib import contextmanager from typing import Iterable, Type from ..api import VectorDB, DBCaseConfig, DBConfig, IndexType @@ -56,7 +57,7 @@ def case_config_cls(cls, index_type: IndexType | None = None) -> Type[DBCaseConf def init(self) -> None: """connect to elasticsearch""" from elasticsearch import Elasticsearch - self.client = Elasticsearch(**self.db_config, request_timeout=30) + self.client = Elasticsearch(**self.db_config, request_timeout=180) yield # self.client.transport.close() @@ -65,8 +66,9 @@ def init(self) -> None: def _create_indice(self, client) -> None: mappings = { + "_source": {"excludes": [self.vector_col_name]}, "properties": { - self.id_col_name: {"type": "integer"}, + self.id_col_name: {"type": "integer", "store": True}, self.vector_col_name: { "dims": self.dim, **self.case_config.index_param(), @@ -137,8 +139,16 @@ def search_embedding( } size = k try: - search_res = self.client.search(index=self.indice, knn=knn, size=size) - res = [d["_source"][self.id_col_name] for d in search_res["hits"]["hits"]] + res = self.client.search( + index=self.indice, + knn=knn, + size=size, + _source=False, + docvalue_fields=[self.id_col_name], + stored_fields="_none_", + filter_path=[f"hits.hits.fields.{self.id_col_name}"], + ) + res = [h["fields"][self.id_col_name][0] for h in res["hits"]["hits"]] return res except Exception as e: @@ -147,7 +157,16 @@ def search_embedding( def optimize(self): """optimize will be called between insertion and search in performance cases.""" - pass + assert self.client is not None, "should self.init() first" + self.client.indices.refresh(index=self.indice) + force_merge_task_id = self.client.indices.forcemerge(index=self.indice, max_num_segments=1, wait_for_completion=False)['task'] + log.info(f"Elasticsearch force merge task id: {force_merge_task_id}") + SECONDS_WAITING_FOR_FORCE_MERGE_API_CALL_SEC = 30 + while True: + time.sleep(SECONDS_WAITING_FOR_FORCE_MERGE_API_CALL_SEC) + task_status = self.client.tasks.get(task_id=force_merge_task_id) + if task_status['completed']: + return def ready_to_load(self): """ready_to_load will be called before load in load cases.""" From ffb450e68741467e86d043030187499ab2ed9378 Mon Sep 17 00:00:00 2001 From: yangxuan Date: Tue, 25 Jul 2023 18:45:54 +0800 Subject: [PATCH 007/327] Seperate test result files by DB Signed-off-by: yangxuan --- tests/test_models.py | 12 +- vectordb_bench/backend/dataset.py | 16 +- vectordb_bench/backend/result_collector.py | 22 +- vectordb_bench/interface.py | 2 +- vectordb_bench/models.py | 35 +- ...result_20230727_standard_elasticcloud.json | 247 + .../result_20230727_standard_milvus.json | 717 +++ .../result_20230727_standard_pgvector.json | 239 + .../result_20230727_standard_pinecone.json | 1099 +++++ .../result_20230727_standard_qdrantcloud.json | 507 ++ ...esult_20230727_standard_weaviatecloud.json | 679 +++ .../result_20230727_standard_zillizcloud.json | 763 +++ .../results/result_20230705_standard.json | 4241 ----------------- 13 files changed, 4310 insertions(+), 4269 deletions(-) create mode 100644 vectordb_bench/results/ElasticCloud/result_20230727_standard_elasticcloud.json create mode 100644 vectordb_bench/results/Milvus/result_20230727_standard_milvus.json create mode 100644 vectordb_bench/results/PgVector/result_20230727_standard_pgvector.json create mode 100644 vectordb_bench/results/Pinecone/result_20230727_standard_pinecone.json create mode 100644 vectordb_bench/results/QdrantCloud/result_20230727_standard_qdrantcloud.json create mode 100644 vectordb_bench/results/WeaviateCloud/result_20230727_standard_weaviatecloud.json create mode 100644 vectordb_bench/results/ZillizCloud/result_20230727_standard_zillizcloud.json delete mode 100644 vectordb_bench/results/result_20230705_standard.json diff --git a/tests/test_models.py b/tests/test_models.py index c61774a22..d68dd6afb 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -30,17 +30,16 @@ def test_test_result(self): ) test_result = TestResult(run_id=10000, results=[result]) - test_result.write_file() + test_result.flush() with pytest.raises(ValueError): result = TestResult.read_file('nosuchfile.json') def test_test_result_read_write(self): result_dir = config.RESULTS_LOCAL_DIR - for json_file in result_dir.glob("*.json"): + for json_file in result_dir.rglob("result*.json"): res = TestResult.read_file(json_file) - res.task_label = f"Milvus-{res.run_id}" - res.write_file() + res.flush() def test_test_result_merge(self): result_dir = config.RESULTS_LOCAL_DIR @@ -61,10 +60,11 @@ def test_test_result_merge(self): task_label="standard", results=all_results, ) - tr.write_file() + tr.flush() def test_test_result_display(self): result_dir = config.RESULTS_LOCAL_DIR - for json_file in result_dir.glob("result*.json"): + for json_file in result_dir.rglob("result*.json"): + log.info(json_file) res = TestResult.read_file(json_file) res.display() diff --git a/vectordb_bench/backend/dataset.py b/vectordb_bench/backend/dataset.py index 3a04ef14b..eafecd1b7 100644 --- a/vectordb_bench/backend/dataset.py +++ b/vectordb_bench/backend/dataset.py @@ -92,7 +92,6 @@ class SIFT(BaseDataset): metric_type: MetricType = MetricType.L2 use_shuffled: bool = False _size_label: dict = { - 500_000: "SMALL", 5_000_000: "MEDIUM", 50_000_000: "LARGE", @@ -102,11 +101,11 @@ class SIFT(BaseDataset): class DatasetManager(BaseModel): """Download dataset if not int the local directory. Provide data for cases. - DataSet is iterable, each iteration will return the next batch of data in pandas.DataFrame + DatasetManager is iterable, each iteration will return the next batch of data in pandas.DataFrame Examples: - >>> cohere_s = DataSet(data=Cohere_S) - >>> for data in cohere_s: + >>> cohere = Dataset.COHERE.manager(100_000) + >>> for data in cohere: >>> print(data.columns) """ data: BaseDataset @@ -115,8 +114,7 @@ class DatasetManager(BaseModel): def __eq__(self, obj): if isinstance(obj, DatasetManager): - return self.data.name == obj.data.name and \ - self.data.label == obj.data.label + return self.data.name == obj.data.name and self.data.label == obj.data.label return False @property @@ -124,7 +122,7 @@ def data_dir(self) -> pathlib.Path: """ data local directory: config.DATASET_LOCAL_DIR/{dataset_name}/{dataset_dirname} Examples: - >>> sift_s = DataSet(data=SIFT_L()) + >>> sift_s = Dataset.SIFT.manager(500_000) >>> sift_s.relative_path '/tmp/vectordb_bench/dataset/sift/sift_small_500k/' """ @@ -135,7 +133,7 @@ def download_dir(self) -> str: """ data s3 directory: config.DEFAULT_DATASET_URL/{dataset_dirname} Examples: - >>> sift_s = DataSet(data=SIFT_L()) + >>> sift_s = Dataset.SIFT.manager(500_000) >>> sift_s.download_dir 'assets.zilliz.com/benchmark/sift_small_500k' """ @@ -144,7 +142,6 @@ def download_dir(self) -> str: def __iter__(self): return DataSetIterator(self) - def _validate_local_file(self): if not self.data_dir.exists(): log.info(f"local file path not exist, creating it: {self.data_dir}") @@ -238,7 +235,6 @@ def prepare(self, check=True) -> bool: - train*.parquet: for training - test.parquet: for testing - neighbors.parquet: ground_truth of the test.parquet - - neighbors_90p.parquet: ground_truth of the test.parquet after filtering 90% data - neighbors_head_1p.parquet: ground_truth of the test.parquet after filtering 1% data - neighbors_99p.parquet: ground_truth of the test.parquet after filtering 99% data """ diff --git a/vectordb_bench/backend/result_collector.py b/vectordb_bench/backend/result_collector.py index b01db4a78..d4c073c1a 100644 --- a/vectordb_bench/backend/result_collector.py +++ b/vectordb_bench/backend/result_collector.py @@ -1,15 +1,27 @@ import pathlib from ..models import TestResult +import logging + +log = logging.getLogger(__name__) + class ResultCollector: @classmethod def collect(cls, result_dir: pathlib.Path) -> list[TestResult]: - results = [] - if not result_dir.exists() or len(list(result_dir.glob("result_*.json"))) == 0: + reg = "result_*.json" + results_d = {} + if not result_dir.exists() or len(list(result_dir.rglob(reg))) == 0: return [] - for json_file in result_dir.glob("result_*.json"): - results.append(TestResult.read_file(json_file, trans_unit=True)) - return results + for json_file in result_dir.rglob(reg): + file_result = TestResult.read_file(json_file, trans_unit=True) + + # Group result files of the same run_id into one TestResult + if file_result.run_id in results_d: + results_d[file_result.run_id].results.extend(file_result.results) + else: + results_d[file_result.run_id] = file_result + + return list(results_d.values()) diff --git a/vectordb_bench/interface.py b/vectordb_bench/interface.py index f303e5d90..737912d15 100644 --- a/vectordb_bench/interface.py +++ b/vectordb_bench/interface.py @@ -176,7 +176,7 @@ def _async_task_v2(self, running_task: TaskRunner, send_conn: Connection) -> Non results=c_results, ) test_result.display() - test_result.write_file() + test_result.flush() send_conn.send((SIGNAL.SUCCESS, None)) send_conn.close() diff --git a/vectordb_bench/models.py b/vectordb_bench/models.py index c50f2ea3e..28b34916c 100644 --- a/vectordb_bench/models.py +++ b/vectordb_bench/models.py @@ -84,19 +84,42 @@ class CaseResult(BaseModel): class TestResult(BaseModel): - """ROOT/result_{date.today()}_{task_label}.json""" - run_id: str task_label: str results: list[CaseResult] - def write_file(self): - result_dir = config.RESULTS_LOCAL_DIR + file_fmt: str = "result_{}_{}_{}.json" # result_20230718_statndard_milvus.json + + def flush(self): + db2case = self.get_db_results() + + result_root = config.RESULTS_LOCAL_DIR + for db, result in db2case.items(): + self.write_db_file( + result_dir=result_root.joinpath(db.value), + partial=TestResult( + run_id=self.run_id, + task_label=self.task_label, + results=result), + db=db.value.lower(), + ) + + + def get_db_results(self) -> dict[DB, CaseResult]: + db2case = {} + for res in self.results: + if res.task_config.db in db2case: + db2case[res.task_config.db].append(res) + else: + db2case[res.task_config.db] = [res] + return db2case + + def write_db_file(self, result_dir: pathlib.Path, partial: Self, db: str): if not result_dir.exists(): log.info(f"local result directory not exist, creating it: {result_dir}") result_dir.mkdir(parents=True) - file_name = f'result_{date.today().strftime("%Y%m%d")}_{self.task_label}.json' + file_name = self.file_fmt.format(date.today().strftime("%Y%m%d"), partial.task_label, db) result_file = result_dir.joinpath(file_name) if result_file.exists(): log.warning( @@ -105,7 +128,7 @@ def write_file(self): log.info(f"write results to disk {result_file}") with open(result_file, "w") as f: - b = self.json(exclude={"db_config": {"password", "api_key"}}) + b = partial.json(exclude={"db_config": {"password", "api_key"}}) f.write(b) @classmethod diff --git a/vectordb_bench/results/ElasticCloud/result_20230727_standard_elasticcloud.json b/vectordb_bench/results/ElasticCloud/result_20230727_standard_elasticcloud.json new file mode 100644 index 000000000..b07c26e81 --- /dev/null +++ b/vectordb_bench/results/ElasticCloud/result_20230727_standard_elasticcloud.json @@ -0,0 +1,247 @@ +{ + "run_id": "5c1e8bd468224ffda1b39b08cdc342c3", + "task_label": "standard", + "results": [ + { + "metrics": { + "max_load_count": 3200000, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "upTo2.5c8g", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "metric_type": "L2", + "efConstruction": 360, + "M": 30, + "num_candidates": null + }, + "case_config": { + "case_id": 2, + "custom_case": {} + } + }, + "label": "?" + }, + { + "metrics": { + "max_load_count": 8600000, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "upTo2.5c8g", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "metric_type": "COSINE", + "efConstruction": 360, + "M": 30, + "num_candidates": null + }, + "case_config": { + "case_id": 1, + "custom_case": {} + } + }, + "label": "?" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 26412.1028, + "qps": 15.2269, + "serial_latency_p99": 0.8618, + "recall": 0.9888 + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "upTo2.5c8g", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "metric_type": "COSINE", + "efConstruction": 360, + "M": 30, + "num_candidates": 100 + }, + "case_config": { + "case_id": 5, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 26412.1028, + "qps": 15.1749, + "serial_latency_p99": 0.7743, + "recall": 0.989 + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "upTo2.5c8g", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "metric_type": "COSINE", + "efConstruction": 360, + "M": 30, + "num_candidates": 100 + }, + "case_config": { + "case_id": 7, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 26412.1028, + "qps": 27.6181, + "serial_latency_p99": 0.3055, + "recall": 0.9999 + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "upTo2.5c8g", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "metric_type": "COSINE", + "efConstruction": 360, + "M": 30, + "num_candidates": 100 + }, + "case_config": { + "case_id": 9, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "upTo2.5c8g", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "metric_type": "COSINE", + "efConstruction": 360, + "M": 30, + "num_candidates": 100 + }, + "case_config": { + "case_id": 4, + "custom_case": {} + } + }, + "label": "?" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "upTo2.5c8g", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "metric_type": "COSINE", + "efConstruction": 360, + "M": 30, + "num_candidates": 100 + }, + "case_config": { + "case_id": 6, + "custom_case": {} + } + }, + "label": "?" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "upTo2.5c8g", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "metric_type": "COSINE", + "efConstruction": 360, + "M": 30, + "num_candidates": 100 + }, + "case_config": { + "case_id": 8, + "custom_case": {} + } + }, + "label": "?" + } + ], + "file_fmt": "result_{}_{}_{}.json" +} diff --git a/vectordb_bench/results/Milvus/result_20230727_standard_milvus.json b/vectordb_bench/results/Milvus/result_20230727_standard_milvus.json new file mode 100644 index 000000000..cc9944ef3 --- /dev/null +++ b/vectordb_bench/results/Milvus/result_20230727_standard_milvus.json @@ -0,0 +1,717 @@ +{ + "run_id": "5c1e8bd468224ffda1b39b08cdc342c3", + "task_label": "standard", + "results": [ + { + "metrics": { + "max_load_count": 0, + "load_duration": 6003.7507, + "qps": 100.6667, + "serial_latency_p99": 0.0211, + "recall": 0.9909 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "2c8g-disk", + "uri": "**********" + }, + "db_case_config": { + "index": "DISKANN", + "metric_type": "COSINE", + "search_list": 100 + }, + "case_config": { + "case_id": 5, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 6003.7507, + "qps": 101.1399, + "serial_latency_p99": 0.0197, + "recall": 0.9907 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "2c8g-disk", + "uri": "**********" + }, + "db_case_config": { + "index": "DISKANN", + "metric_type": "COSINE", + "search_list": 100 + }, + "case_config": { + "case_id": 7, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 6003.7507, + "qps": 52.2606, + "serial_latency_p99": 0.0183, + "recall": 1 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "2c8g-disk", + "uri": "**********" + }, + "db_case_config": { + "index": "DISKANN", + "metric_type": "COSINE", + "search_list": 100 + }, + "case_config": { + "case_id": 9, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "2c8g-disk", + "uri": "**********" + }, + "db_case_config": { + "index": "DISKANN", + "metric_type": "COSINE", + "search_list": 100 + }, + "case_config": { + "case_id": 4, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "2c8g-disk", + "uri": "**********" + }, + "db_case_config": { + "index": "DISKANN", + "metric_type": "COSINE", + "search_list": 100 + }, + "case_config": { + "case_id": 6, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "2c8g-disk", + "uri": "**********" + }, + "db_case_config": { + "index": "DISKANN", + "metric_type": "COSINE", + "search_list": 100 + }, + "case_config": { + "case_id": 8, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 18790.8759, + "qps": 61.0661, + "serial_latency_p99": 0.0498, + "recall": 0.9911 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "4c16g-disk", + "uri": "**********" + }, + "db_case_config": { + "index": "DISKANN", + "metric_type": "COSINE", + "search_list": 100 + }, + "case_config": { + "case_id": 4, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 18790.8759, + "qps": 58.9326, + "serial_latency_p99": 0.0446, + "recall": 0.9911 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "4c16g-disk", + "uri": "**********" + }, + "db_case_config": { + "index": "DISKANN", + "metric_type": "COSINE", + "search_list": 100 + }, + "case_config": { + "case_id": 6, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 18790.8759, + "qps": 42.5977, + "serial_latency_p99": 0.0549, + "recall": 1 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "4c16g-disk", + "uri": "**********" + }, + "db_case_config": { + "index": "DISKANN", + "metric_type": "COSINE", + "search_list": 100 + }, + "case_config": { + "case_id": 8, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1782.0061, + "qps": 536.0726, + "serial_latency_p99": 0.0082, + "recall": 0.9728 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "4c16g-disk", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 5, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1782.0061, + "qps": 467.179, + "serial_latency_p99": 0.007, + "recall": 0.9697 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "4c16g-disk", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 7, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1455.573, + "qps": 431.7512, + "serial_latency_p99": 0.0083, + "recall": 1 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "4c16g-disk", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 9, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 9100000, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "2c8g-hnsw", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 1, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 1000000, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "2c8g-hnsw", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 2, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 2379.7118, + "qps": 274.5407, + "serial_latency_p99": 0.0049, + "recall": 0.9807 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "2c8g-hnsw", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 5, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 2379.7118, + "qps": 236.5672, + "serial_latency_p99": 0.0103, + "recall": 0.981 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "2c8g-hnsw", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 7, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 2379.7118, + "qps": 309.4833, + "serial_latency_p99": 0.0043, + "recall": 1 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "2c8g-hnsw", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 9, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "2c8g-hnsw", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 4, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "2c8g-hnsw", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 6, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "2c8g-hnsw", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 8, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 5713.7159, + "qps": 178.6585, + "serial_latency_p99": 0.0137, + "recall": 0.9843 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-hnsw", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 4, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 5713.7159, + "qps": 178.3732, + "serial_latency_p99": 0.015, + "recall": 0.9844 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-hnsw", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 6, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 5713.7159, + "qps": 229.3526, + "serial_latency_p99": 0.0125, + "recall": 1 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-hnsw", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 8, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 581.8141, + "qps": 1258.7043, + "serial_latency_p99": 0.0049, + "recall": 0.9799 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-hnsw", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 5, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 581.8141, + "qps": 1075.8776, + "serial_latency_p99": 0.0053, + "recall": 0.98 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-hnsw", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 7, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 581.8141, + "qps": 1494.8493, + "serial_latency_p99": 0.0047, + "recall": 1 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-hnsw", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 9, + "custom_case": {} + } + }, + "label": ":)" + } + ], + "file_fmt": "result_{}_{}_{}.json" +} diff --git a/vectordb_bench/results/PgVector/result_20230727_standard_pgvector.json b/vectordb_bench/results/PgVector/result_20230727_standard_pgvector.json new file mode 100644 index 000000000..5385e1f5c --- /dev/null +++ b/vectordb_bench/results/PgVector/result_20230727_standard_pgvector.json @@ -0,0 +1,239 @@ +{ + "run_id": "5c1e8bd468224ffda1b39b08cdc342c3", + "task_label": "standard", + "results": [ + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "PgVector", + "db_config": { + "db_label": "2c8g-1node", + "user_name": "**********", + "password": "**********", + "url": "**********", + "db_name": "**********" + }, + "db_case_config": { + "metric_type": "L2", + "lists": 10, + "probes": 2 + }, + "case_config": { + "case_id": 1, + "custom_case": {} + } + }, + "label": "?" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "PgVector", + "db_config": { + "db_label": "2c8g-1node", + "user_name": "**********", + "password": "**********", + "url": "**********", + "db_name": "**********" + }, + "db_case_config": { + "metric_type": "L2", + "lists": 10, + "probes": 2 + }, + "case_config": { + "case_id": 2, + "custom_case": {} + } + }, + "label": "?" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 10247.5034, + "qps": 10.6271, + "serial_latency_p99": 0.7307, + "recall": 0.8898 + }, + "task_config": { + "db": "PgVector", + "db_config": { + "db_label": "2c8g-1node", + "user_name": "**********", + "password": "**********", + "url": "**********", + "db_name": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "lists": 10, + "probes": 2 + }, + "case_config": { + "case_id": 5, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 10247.5034, + "qps": 10.8507, + "serial_latency_p99": 0.7332, + "recall": 0.8897 + }, + "task_config": { + "db": "PgVector", + "db_config": { + "db_label": "2c8g-1node", + "user_name": "**********", + "password": "**********", + "url": "**********", + "db_name": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "lists": 10, + "probes": 2 + }, + "case_config": { + "case_id": 7, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 10247.5034, + "qps": 75.7055, + "serial_latency_p99": 0.1212, + "recall": 0.9999 + }, + "task_config": { + "db": "PgVector", + "db_config": { + "db_label": "2c8g-1node", + "user_name": "**********", + "password": "**********", + "url": "**********", + "db_name": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "lists": 10, + "probes": 2 + }, + "case_config": { + "case_id": 9, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "PgVector", + "db_config": { + "db_label": "2c8g-1node", + "user_name": "**********", + "password": "**********", + "url": "**********", + "db_name": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "lists": 10, + "probes": 2 + }, + "case_config": { + "case_id": 4, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "PgVector", + "db_config": { + "db_label": "2c8g-1node", + "user_name": "**********", + "password": "**********", + "url": "**********", + "db_name": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "lists": 10, + "probes": 2 + }, + "case_config": { + "case_id": 6, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "PgVector", + "db_config": { + "db_label": "2c8g-1node", + "user_name": "**********", + "password": "**********", + "url": "**********", + "db_name": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "lists": 10, + "probes": 2 + }, + "case_config": { + "case_id": 8, + "custom_case": {} + } + }, + "label": "x" + } + ], + "file_fmt": "result_{}_{}_{}.json" +} diff --git a/vectordb_bench/results/Pinecone/result_20230727_standard_pinecone.json b/vectordb_bench/results/Pinecone/result_20230727_standard_pinecone.json new file mode 100644 index 000000000..4be015d10 --- /dev/null +++ b/vectordb_bench/results/Pinecone/result_20230727_standard_pinecone.json @@ -0,0 +1,1099 @@ +{ + "run_id": "5c1e8bd468224ffda1b39b08cdc342c3", + "task_label": "standard", + "results": [ + { + "metrics": { + "max_load_count": 5500000, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "s1.x1", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 2, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 8300000, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "s1.x1", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 1, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1129.4487, + "qps": 18.7634, + "serial_latency_p99": 0.1537, + "recall": 0.8737 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "s1.x1", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 5, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1129.4487, + "qps": 18.3619, + "serial_latency_p99": 0.0798, + "recall": 0.8741 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "s1.x1", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 7, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1129.4487, + "qps": 25.2744, + "serial_latency_p99": 0.0612, + "recall": 0.9979 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "s1.x1", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 9, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "s1.x1", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 4, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "s1.x1", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 6, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "s1.x1", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 8, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 1600000, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p2.x1", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 2, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 2700000, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p2.x1", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 1, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1582.9409, + "qps": 261.798, + "serial_latency_p99": 0.0231, + "recall": 0.9262 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p2.x1", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 5, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1582.9409, + "qps": 166.1851, + "serial_latency_p99": 0.0239, + "recall": 0.9264 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p2.x1", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 7, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1582.9409, + "qps": 121.7169, + "serial_latency_p99": 0.029, + "recall": 0.9693 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p2.x1", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 9, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p2.x1", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 4, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p2.x1", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 6, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p2.x1", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 8, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 700000, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p1.x1", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 2, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 4100000, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p1.x1", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 1, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1123.9889, + "qps": 46.6189, + "serial_latency_p99": 0.0431, + "recall": 0.8737 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p1.x1", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 5, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1123.9889, + "qps": 42.4856, + "serial_latency_p99": 0.044, + "recall": 0.8741 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p1.x1", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 7, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1123.9889, + "qps": 138.9479, + "serial_latency_p99": 0.0262, + "recall": 0.9979 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p1.x1", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 9, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p1.x1", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 4, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p1.x1", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 6, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p1.x1", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 8, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1450.6205, + "qps": 20.7437, + "serial_latency_p99": 0.0758, + "recall": 0.9291 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "s1.x1-2node", + "api_key": "**********", + "environment": "**********", + "index_name": "benchmark-test" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 5, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1450.6205, + "qps": 20.2993, + "serial_latency_p99": 0.0765, + "recall": 0.9293 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "s1.x1-2node", + "api_key": "**********", + "environment": "**********", + "index_name": "benchmark-test" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 7, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1450.6205, + "qps": 26.4719, + "serial_latency_p99": 0.067, + "recall": 1 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "s1.x1-2node", + "api_key": "**********", + "environment": "**********", + "index_name": "benchmark-test" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 9, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 11374.7189, + "qps": 8.6675, + "serial_latency_p99": 0.1802, + "recall": 0.8369 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "s1.x1-2node", + "api_key": "**********", + "environment": "**********", + "index_name": "benchmark-test" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 4, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 11374.7189, + "qps": 7.8121, + "serial_latency_p99": 0.1677, + "recall": 0.8369 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "s1.x1-2node", + "api_key": "**********", + "environment": "**********", + "index_name": "benchmark-test" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 6, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 11374.7189, + "qps": 16.869, + "serial_latency_p99": 0.0878, + "recall": 0.9814 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "s1.x1-2node", + "api_key": "**********", + "environment": "**********", + "index_name": "benchmark-test" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 8, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1210.2139, + "qps": 365.0835, + "serial_latency_p99": 0.0236, + "recall": 0.945 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p2.x1-8node", + "api_key": "**********", + "environment": "**********", + "index_name": "benchmark-test" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 5, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1210.2139, + "qps": 325.5271, + "serial_latency_p99": 0.0251, + "recall": 0.9452 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p2.x1-8node", + "api_key": "**********", + "environment": "**********", + "index_name": "benchmark-test" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 7, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1210.2139, + "qps": 596.7942, + "serial_latency_p99": 0.0242, + "recall": 0.9693 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p2.x1-8node", + "api_key": "**********", + "environment": "**********", + "index_name": "benchmark-test" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 9, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 14165.6504, + "qps": 303.2044, + "serial_latency_p99": 0.0274, + "recall": 0.9246 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p2.x1-8node", + "api_key": "**********", + "environment": "**********", + "index_name": "benchmark-test" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 4, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 14165.6504, + "qps": 136.0345, + "serial_latency_p99": 0.0319, + "recall": 0.9244 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p2.x1-8node", + "api_key": "**********", + "environment": "**********", + "index_name": "benchmark-test" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 6, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 14165.6504, + "qps": 66.7221, + "serial_latency_p99": 0.0421, + "recall": 0.963 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p2.x1-8node", + "api_key": "**********", + "environment": "**********", + "index_name": "benchmark-test" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 8, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1211.8883, + "qps": 131.2549, + "serial_latency_p99": 0.0302, + "recall": 0.9867 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p1.x1-8node", + "api_key": "**********", + "environment": "**********", + "index_name": "benchmark-test" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 5, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1211.8883, + "qps": 127.9337, + "serial_latency_p99": 0.0301, + "recall": 0.9869 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p1.x1-8node", + "api_key": "**********", + "environment": "**********", + "index_name": "benchmark-test" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 7, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1211.8883, + "qps": 595.8462, + "serial_latency_p99": 0.0234, + "recall": 1 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p1.x1-8node", + "api_key": "**********", + "environment": "**********", + "index_name": "benchmark-test" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 9, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p1.x1-8node", + "api_key": "**********", + "environment": "**********", + "index_name": "benchmark-test" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 4, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p1.x1-8node", + "api_key": "**********", + "environment": "**********", + "index_name": "benchmark-test" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 6, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p1.x1-8node", + "api_key": "**********", + "environment": "**********", + "index_name": "benchmark-test" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 8, + "custom_case": {} + } + }, + "label": "x" + } + ], + "file_fmt": "result_{}_{}_{}.json" +} diff --git a/vectordb_bench/results/QdrantCloud/result_20230727_standard_qdrantcloud.json b/vectordb_bench/results/QdrantCloud/result_20230727_standard_qdrantcloud.json new file mode 100644 index 000000000..e6efb8a86 --- /dev/null +++ b/vectordb_bench/results/QdrantCloud/result_20230727_standard_qdrantcloud.json @@ -0,0 +1,507 @@ +{ + "run_id": "5c1e8bd468224ffda1b39b08cdc342c3", + "task_label": "standard", + "results": [ + { + "metrics": { + "max_load_count": 4000000, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "2c8g-1node", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": null + }, + "case_config": { + "case_id": 1, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 900000, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "2c8g-1node", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": null + }, + "case_config": { + "case_id": 2, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "2c8g-1node", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": null + }, + "case_config": { + "case_id": 5, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "2c8g-1node", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": null + }, + "case_config": { + "case_id": 7, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "2c8g-1node", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": null + }, + "case_config": { + "case_id": 9, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "2c8g-1node", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": null + }, + "case_config": { + "case_id": 4, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "2c8g-1node", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": null + }, + "case_config": { + "case_id": 6, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "2c8g-1node", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": null + }, + "case_config": { + "case_id": 8, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1647.857, + "qps": 537.4975, + "serial_latency_p99": 0.0189, + "recall": 0.8903 + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "4c16g-5node", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": null + }, + "case_config": { + "case_id": 5, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1647.857, + "qps": 372.0466, + "serial_latency_p99": 0.0178, + "recall": 0.8904 + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "4c16g-5node", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": null + }, + "case_config": { + "case_id": 7, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1647.857, + "qps": 1156.2898, + "serial_latency_p99": 0.0144, + "recall": 0.9989 + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "4c16g-5node", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": null + }, + "case_config": { + "case_id": 9, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 14623.578, + "qps": 110.248, + "serial_latency_p99": 0.069, + "recall": 0.898 + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "4c16g-5node", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": null + }, + "case_config": { + "case_id": 4, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 14623.578, + "qps": 87.2601, + "serial_latency_p99": 0.0278, + "recall": 0.898 + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "4c16g-5node", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": null + }, + "case_config": { + "case_id": 6, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 14623.578, + "qps": 125.7846, + "serial_latency_p99": 0.0231, + "recall": 0.975 + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "4c16g-5node", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": null + }, + "case_config": { + "case_id": 8, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1503.1147, + "qps": 240.7209, + "serial_latency_p99": 0.0174, + "recall": 0.8887 + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "4c16g-1node", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": null + }, + "case_config": { + "case_id": 5, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1503.1147, + "qps": 189.4399, + "serial_latency_p99": 0.0175, + "recall": 0.8889 + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "4c16g-1node", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": null + }, + "case_config": { + "case_id": 7, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1503.1147, + "qps": 313.5116, + "serial_latency_p99": 0.0161, + "recall": 0.9999 + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "4c16g-1node", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": null + }, + "case_config": { + "case_id": 9, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "4c16g-1node", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": null + }, + "case_config": { + "case_id": 4, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "4c16g-1node", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": null + }, + "case_config": { + "case_id": 6, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "4c16g-1node", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": null + }, + "case_config": { + "case_id": 8, + "custom_case": {} + } + }, + "label": "x" + } + ], + "file_fmt": "result_{}_{}_{}.json" +} diff --git a/vectordb_bench/results/WeaviateCloud/result_20230727_standard_weaviatecloud.json b/vectordb_bench/results/WeaviateCloud/result_20230727_standard_weaviatecloud.json new file mode 100644 index 000000000..d7e9ca455 --- /dev/null +++ b/vectordb_bench/results/WeaviateCloud/result_20230727_standard_weaviatecloud.json @@ -0,0 +1,679 @@ +{ + "run_id": "5c1e8bd468224ffda1b39b08cdc342c3", + "task_label": "standard", + "results": [ + { + "metrics": { + "max_load_count": 5800000, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "WeaviateCloud", + "db_config": { + "db_label": "bus_crit", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "ef": -1, + "efConstruction": null, + "maxConnections": null + }, + "case_config": { + "case_id": 1, + "custom_case": null + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 1800000, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "WeaviateCloud", + "db_config": { + "db_label": "bus_crit", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "ef": -1, + "efConstruction": null, + "maxConnections": null + }, + "case_config": { + "case_id": 2, + "custom_case": null + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "WeaviateCloud", + "db_config": { + "db_label": "bus_crit", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "ef": -1, + "efConstruction": null, + "maxConnections": null + }, + "case_config": { + "case_id": 4, + "custom_case": null + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "WeaviateCloud", + "db_config": { + "db_label": "bus_crit", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "ef": -1, + "efConstruction": null, + "maxConnections": null + }, + "case_config": { + "case_id": 6, + "custom_case": null + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "WeaviateCloud", + "db_config": { + "db_label": "bus_crit", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "ef": -1, + "efConstruction": null, + "maxConnections": null + }, + "case_config": { + "case_id": 8, + "custom_case": null + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 3674.0953, + "qps": 67.9121, + "serial_latency_p99": 0.1795, + "recall": 0.9909 + }, + "task_config": { + "db": "WeaviateCloud", + "db_config": { + "db_label": "bus_crit", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "ef": -1, + "efConstruction": null, + "maxConnections": null + }, + "case_config": { + "case_id": 5, + "custom_case": null + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 3674.0953, + "qps": 0.7636, + "serial_latency_p99": 1.9213, + "recall": 0.9908 + }, + "task_config": { + "db": "WeaviateCloud", + "db_config": { + "db_label": "bus_crit", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "ef": -1, + "efConstruction": null, + "maxConnections": null + }, + "case_config": { + "case_id": 7, + "custom_case": null + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 3674.0953, + "qps": 32, + "serial_latency_p99": 0.1245, + "recall": 1 + }, + "task_config": { + "db": "WeaviateCloud", + "db_config": { + "db_label": "bus_crit", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "ef": -1, + "efConstruction": null, + "maxConnections": null + }, + "case_config": { + "case_id": 9, + "custom_case": null + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 1480000, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "WeaviateCloud", + "db_config": { + "db_label": "sandbox", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "ef": -1, + "efConstruction": null, + "maxConnections": null + }, + "case_config": { + "case_id": 1, + "custom_case": null + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 455000, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "WeaviateCloud", + "db_config": { + "db_label": "sandbox", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "ef": -1, + "efConstruction": null, + "maxConnections": null + }, + "case_config": { + "case_id": 2, + "custom_case": null + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "WeaviateCloud", + "db_config": { + "db_label": "sandbox", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "ef": -1, + "efConstruction": null, + "maxConnections": null + }, + "case_config": { + "case_id": 5, + "custom_case": null + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "WeaviateCloud", + "db_config": { + "db_label": "sandbox", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "ef": -1, + "efConstruction": null, + "maxConnections": null + }, + "case_config": { + "case_id": 7, + "custom_case": null + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "WeaviateCloud", + "db_config": { + "db_label": "sandbox", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "ef": -1, + "efConstruction": null, + "maxConnections": null + }, + "case_config": { + "case_id": 9, + "custom_case": null + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "WeaviateCloud", + "db_config": { + "db_label": "sandbox", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "ef": -1, + "efConstruction": null, + "maxConnections": null + }, + "case_config": { + "case_id": 4, + "custom_case": null + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "WeaviateCloud", + "db_config": { + "db_label": "sandbox", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "ef": -1, + "efConstruction": null, + "maxConnections": null + }, + "case_config": { + "case_id": 6, + "custom_case": null + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "WeaviateCloud", + "db_config": { + "db_label": "sandbox", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "ef": -1, + "efConstruction": null, + "maxConnections": null + }, + "case_config": { + "case_id": 8, + "custom_case": null + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 5500000, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "WeaviateCloud", + "db_config": { + "db_label": "standard", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "ef": -1, + "efConstruction": null, + "maxConnections": null + }, + "case_config": { + "case_id": 1, + "custom_case": null + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 1800000, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "WeaviateCloud", + "db_config": { + "db_label": "standard", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "ef": -1, + "efConstruction": null, + "maxConnections": null + }, + "case_config": { + "case_id": 2, + "custom_case": null + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 3580.7827, + "qps": 63.1365, + "serial_latency_p99": 0.1457, + "recall": 0.991 + }, + "task_config": { + "db": "WeaviateCloud", + "db_config": { + "db_label": "standard", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "ef": -1, + "efConstruction": null, + "maxConnections": null + }, + "case_config": { + "case_id": 5, + "custom_case": null + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 3580.12, + "qps": 0.7512, + "serial_latency_p99": 1.9838, + "recall": 0.9908 + }, + "task_config": { + "db": "WeaviateCloud", + "db_config": { + "db_label": "standard", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "ef": -1, + "efConstruction": 128, + "maxConnections": 64 + }, + "case_config": { + "case_id": 7, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 30.1358, + "serial_latency_p99": 0.1298, + "recall": 1 + }, + "task_config": { + "db": "WeaviateCloud", + "db_config": { + "db_label": "standard", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "ef": -1, + "efConstruction": 128, + "maxConnections": 64 + }, + "case_config": { + "case_id": 9, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "WeaviateCloud", + "db_config": { + "db_label": "standard", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "ef": -1, + "efConstruction": null, + "maxConnections": null + }, + "case_config": { + "case_id": 4, + "custom_case": null + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "WeaviateCloud", + "db_config": { + "db_label": "standard", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "ef": -1, + "efConstruction": null, + "maxConnections": null + }, + "case_config": { + "case_id": 6, + "custom_case": null + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "WeaviateCloud", + "db_config": { + "db_label": "standard", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "ef": -1, + "efConstruction": null, + "maxConnections": null + }, + "case_config": { + "case_id": 8, + "custom_case": null + } + }, + "label": "x" + } + ], + "file_fmt": "result_{}_{}_{}.json" +} diff --git a/vectordb_bench/results/ZillizCloud/result_20230727_standard_zillizcloud.json b/vectordb_bench/results/ZillizCloud/result_20230727_standard_zillizcloud.json new file mode 100644 index 000000000..919333d77 --- /dev/null +++ b/vectordb_bench/results/ZillizCloud/result_20230727_standard_zillizcloud.json @@ -0,0 +1,763 @@ +{ + "run_id": "5c1e8bd468224ffda1b39b08cdc342c3", + "task_label": "standard", + "results": [ + { + "metrics": { + "max_load_count": 6700000, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-cap", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "L2" + }, + "case_config": { + "case_id": 2, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 12700000, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-cap", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "L2" + }, + "case_config": { + "case_id": 1, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 2412.0463, + "qps": 330.0144, + "serial_latency_p99": 0.009, + "recall": 0.9507 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-cap", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 5, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 2412.0463, + "qps": 271.6585, + "serial_latency_p99": 0.0101, + "recall": 0.9678 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-cap", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 7, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 2412.0463, + "qps": 216.5226, + "serial_latency_p99": 0.0129, + "recall": 1 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-cap", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 9, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-cap", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 4, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-cap", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 6, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-cap", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 8, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 14521.4008, + "qps": 123.9553, + "serial_latency_p99": 0.023, + "recall": 0.971 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "2cu-cap", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 4, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 14521.4008, + "qps": 59.1479, + "serial_latency_p99": 0.0445, + "recall": 0.9906 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "2cu-cap", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 6, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 14521.4008, + "qps": 40.999, + "serial_latency_p99": 0.0553, + "recall": 1 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "2cu-cap", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 8, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 2340.5671, + "qps": 579.9416, + "serial_latency_p99": 0.0094, + "recall": 0.9213 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "2cu-cap", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 5, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 2340.5671, + "qps": 425.2529, + "serial_latency_p99": 0.0113, + "recall": 0.9686 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "2cu-cap", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 7, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 2340.5671, + "qps": 397.0539, + "serial_latency_p99": 0.0138, + "recall": 1 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "2cu-cap", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 9, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 5000000, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-perf", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 1, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 1200000, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-perf", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 2, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1100.6681, + "qps": 516.27, + "serial_latency_p99": 0.007, + "recall": 0.9463 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-perf", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 5, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1100.6681, + "qps": 354.8416, + "serial_latency_p99": 0.01, + "recall": 0.9802 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-perf", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 7, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1100.6681, + "qps": 427.5229, + "serial_latency_p99": 0.0087, + "recall": 1 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-perf", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 9, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-perf", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 4, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-perf", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 6, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0, + "qps": 0, + "serial_latency_p99": 0, + "recall": 0 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-perf", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 8, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1575.253, + "qps": 2884.689, + "serial_latency_p99": 0.0053, + "recall": 0.8801 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 5, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1853.989, + "qps": 1689.5799, + "serial_latency_p99": 0.0066, + "recall": 0.9493 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 7, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1575.253, + "qps": 1517.6792, + "serial_latency_p99": 0.01, + "recall": 1 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 9, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 7635.0576, + "qps": 822.5318, + "serial_latency_p99": 0.0056, + "recall": 0.9294 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 4, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 7635.0576, + "qps": 378.9146, + "serial_latency_p99": 0.0103, + "recall": 0.9758 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 6, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 7635.0576, + "qps": 218.6854, + "serial_latency_p99": 0.0162, + "recall": 1 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 8, + "custom_case": {} + } + }, + "label": ":)" + } + ], + "file_fmt": "result_{}_{}_{}.json" +} diff --git a/vectordb_bench/results/result_20230705_standard.json b/vectordb_bench/results/result_20230705_standard.json deleted file mode 100644 index ed080c9a5..000000000 --- a/vectordb_bench/results/result_20230705_standard.json +++ /dev/null @@ -1,4241 +0,0 @@ -{ - "run_id": "5c1e8bd468224ffda1b39b08cdc342c3", - "task_label": "standard", - "results": [ - { - "metrics": { - "max_load_count": 3200000, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "ElasticCloud", - "db_config": { - "db_label": "upTo2.5c8g", - "cloud_id": "**********", - "password": "**********" - }, - "db_case_config": { - "element_type": "float", - "index": "hnsw", - "metric_type": "L2", - "efConstruction": 360, - "M": 30, - "num_candidates": null - }, - "case_config": { - "case_id": 2, - "custom_case": {} - } - }, - "label": "?" - }, - { - "metrics": { - "max_load_count": 8600000, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "ElasticCloud", - "db_config": { - "db_label": "upTo2.5c8g", - "cloud_id": "**********", - "password": "**********" - }, - "db_case_config": { - "element_type": "float", - "index": "hnsw", - "metric_type": "COSINE", - "efConstruction": 360, - "M": 30, - "num_candidates": null - }, - "case_config": { - "case_id": 1, - "custom_case": {} - } - }, - "label": "?" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 26412.1028, - "qps": 15.2269, - "serial_latency_p99": 0.8618, - "recall": 0.9888 - }, - "task_config": { - "db": "ElasticCloud", - "db_config": { - "db_label": "upTo2.5c8g", - "cloud_id": "**********", - "password": "**********" - }, - "db_case_config": { - "element_type": "float", - "index": "hnsw", - "metric_type": "COSINE", - "efConstruction": 360, - "M": 30, - "num_candidates": 100 - }, - "case_config": { - "case_id": 5, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 26412.1028, - "qps": 15.1749, - "serial_latency_p99": 0.7743, - "recall": 0.989 - }, - "task_config": { - "db": "ElasticCloud", - "db_config": { - "db_label": "upTo2.5c8g", - "cloud_id": "**********", - "password": "**********" - }, - "db_case_config": { - "element_type": "float", - "index": "hnsw", - "metric_type": "COSINE", - "efConstruction": 360, - "M": 30, - "num_candidates": 100 - }, - "case_config": { - "case_id": 7, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 26412.1028, - "qps": 27.6181, - "serial_latency_p99": 0.3055, - "recall": 0.9999 - }, - "task_config": { - "db": "ElasticCloud", - "db_config": { - "db_label": "upTo2.5c8g", - "cloud_id": "**********", - "password": "**********" - }, - "db_case_config": { - "element_type": "float", - "index": "hnsw", - "metric_type": "COSINE", - "efConstruction": 360, - "M": 30, - "num_candidates": 100 - }, - "case_config": { - "case_id": 9, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "ElasticCloud", - "db_config": { - "db_label": "upTo2.5c8g", - "cloud_id": "**********", - "password": "**********" - }, - "db_case_config": { - "element_type": "float", - "index": "hnsw", - "metric_type": "COSINE", - "efConstruction": 360, - "M": 30, - "num_candidates": 100 - }, - "case_config": { - "case_id": 4, - "custom_case": {} - } - }, - "label": "?" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "ElasticCloud", - "db_config": { - "db_label": "upTo2.5c8g", - "cloud_id": "**********", - "password": "**********" - }, - "db_case_config": { - "element_type": "float", - "index": "hnsw", - "metric_type": "COSINE", - "efConstruction": 360, - "M": 30, - "num_candidates": 100 - }, - "case_config": { - "case_id": 6, - "custom_case": {} - } - }, - "label": "?" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "ElasticCloud", - "db_config": { - "db_label": "upTo2.5c8g", - "cloud_id": "**********", - "password": "**********" - }, - "db_case_config": { - "element_type": "float", - "index": "hnsw", - "metric_type": "COSINE", - "efConstruction": 360, - "M": 30, - "num_candidates": 100 - }, - "case_config": { - "case_id": 8, - "custom_case": {} - } - }, - "label": "?" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 6003.7507, - "qps": 100.6667, - "serial_latency_p99": 0.0211, - "recall": 0.9909 - }, - "task_config": { - "db": "Milvus", - "db_config": { - "db_label": "2c8g-disk", - "uri": "**********" - }, - "db_case_config": { - "index": "DISKANN", - "metric_type": "COSINE", - "search_list": 100 - }, - "case_config": { - "case_id": 5, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 6003.7507, - "qps": 101.1399, - "serial_latency_p99": 0.0197, - "recall": 0.9907 - }, - "task_config": { - "db": "Milvus", - "db_config": { - "db_label": "2c8g-disk", - "uri": "**********" - }, - "db_case_config": { - "index": "DISKANN", - "metric_type": "COSINE", - "search_list": 100 - }, - "case_config": { - "case_id": 7, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 6003.7507, - "qps": 52.2606, - "serial_latency_p99": 0.0183, - "recall": 1 - }, - "task_config": { - "db": "Milvus", - "db_config": { - "db_label": "2c8g-disk", - "uri": "**********" - }, - "db_case_config": { - "index": "DISKANN", - "metric_type": "COSINE", - "search_list": 100 - }, - "case_config": { - "case_id": 9, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "Milvus", - "db_config": { - "db_label": "2c8g-disk", - "uri": "**********" - }, - "db_case_config": { - "index": "DISKANN", - "metric_type": "COSINE", - "search_list": 100 - }, - "case_config": { - "case_id": 4, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "Milvus", - "db_config": { - "db_label": "2c8g-disk", - "uri": "**********" - }, - "db_case_config": { - "index": "DISKANN", - "metric_type": "COSINE", - "search_list": 100 - }, - "case_config": { - "case_id": 6, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "Milvus", - "db_config": { - "db_label": "2c8g-disk", - "uri": "**********" - }, - "db_case_config": { - "index": "DISKANN", - "metric_type": "COSINE", - "search_list": 100 - }, - "case_config": { - "case_id": 8, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 18790.8759, - "qps": 61.0661, - "serial_latency_p99": 0.0498, - "recall": 0.9911 - }, - "task_config": { - "db": "Milvus", - "db_config": { - "db_label": "4c16g-disk", - "uri": "**********" - }, - "db_case_config": { - "index": "DISKANN", - "metric_type": "COSINE", - "search_list": 100 - }, - "case_config": { - "case_id": 4, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 18790.8759, - "qps": 58.9326, - "serial_latency_p99": 0.0446, - "recall": 0.9911 - }, - "task_config": { - "db": "Milvus", - "db_config": { - "db_label": "4c16g-disk", - "uri": "**********" - }, - "db_case_config": { - "index": "DISKANN", - "metric_type": "COSINE", - "search_list": 100 - }, - "case_config": { - "case_id": 6, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 18790.8759, - "qps": 42.5977, - "serial_latency_p99": 0.0549, - "recall": 1 - }, - "task_config": { - "db": "Milvus", - "db_config": { - "db_label": "4c16g-disk", - "uri": "**********" - }, - "db_case_config": { - "index": "DISKANN", - "metric_type": "COSINE", - "search_list": 100 - }, - "case_config": { - "case_id": 8, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 1782.0061, - "qps": 536.0726, - "serial_latency_p99": 0.0082, - "recall": 0.9728 - }, - "task_config": { - "db": "Milvus", - "db_config": { - "db_label": "4c16g-disk", - "uri": "**********" - }, - "db_case_config": { - "index": "HNSW", - "metric_type": "COSINE", - "M": 30, - "efConstruction": 360, - "ef": 100 - }, - "case_config": { - "case_id": 5, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 1782.0061, - "qps": 467.179, - "serial_latency_p99": 0.007, - "recall": 0.9697 - }, - "task_config": { - "db": "Milvus", - "db_config": { - "db_label": "4c16g-disk", - "uri": "**********" - }, - "db_case_config": { - "index": "HNSW", - "metric_type": "COSINE", - "M": 30, - "efConstruction": 360, - "ef": 100 - }, - "case_config": { - "case_id": 7, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 1455.573, - "qps": 431.7512, - "serial_latency_p99": 0.0083, - "recall": 1 - }, - "task_config": { - "db": "Milvus", - "db_config": { - "db_label": "4c16g-disk", - "uri": "**********" - }, - "db_case_config": { - "index": "HNSW", - "metric_type": "COSINE", - "M": 30, - "efConstruction": 360, - "ef": 100 - }, - "case_config": { - "case_id": 9, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 6700000, - "load_duration": 0, - "qps": 0, - "serial_latency_p99": 0, - "recall": 0 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-cap", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "L2" - }, - "case_config": { - "case_id": 2, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 12700000, - "load_duration": 0, - "qps": 0, - "serial_latency_p99": 0, - "recall": 0 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-cap", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "L2" - }, - "case_config": { - "case_id": 1, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 2412.0463, - "qps": 330.0144, - "serial_latency_p99": 0.009, - "recall": 0.9507 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-cap", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 5, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 2412.0463, - "qps": 271.6585, - "serial_latency_p99": 0.0101, - "recall": 0.9678 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-cap", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 7, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 2412.0463, - "qps": 216.5226, - "serial_latency_p99": 0.0129, - "recall": 1 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-cap", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 9, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-cap", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 4, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-cap", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 6, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-cap", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 8, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 14521.4008, - "qps": 123.9553, - "serial_latency_p99": 0.023, - "recall": 0.971 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "2cu-cap", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 4, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 14521.4008, - "qps": 59.1479, - "serial_latency_p99": 0.0445, - "recall": 0.9906 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "2cu-cap", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 6, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 14521.4008, - "qps": 40.999, - "serial_latency_p99": 0.0553, - "recall": 1 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "2cu-cap", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 8, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 2340.5671, - "qps": 579.9416, - "serial_latency_p99": 0.0094, - "recall": 0.9213 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "2cu-cap", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 5, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 2340.5671, - "qps": 425.2529, - "serial_latency_p99": 0.0113, - "recall": 0.9686 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "2cu-cap", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 7, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 2340.5671, - "qps": 397.0539, - "serial_latency_p99": 0.0138, - "recall": 1 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "2cu-cap", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 9, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 5800000, - "load_duration": 0, - "qps": 0, - "serial_latency_p99": 0, - "recall": 0 - }, - "task_config": { - "db": "WeaviateCloud", - "db_config": { - "db_label": "bus_crit", - "url": "**********", - "api_key": "**********" - }, - "db_case_config": { - "metric_type": "COSINE", - "ef": -1, - "efConstruction": null, - "maxConnections": null - }, - "case_config": { - "case_id": 1, - "custom_case": null - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 1800000, - "load_duration": 0, - "qps": 0, - "serial_latency_p99": 0, - "recall": 0 - }, - "task_config": { - "db": "WeaviateCloud", - "db_config": { - "db_label": "bus_crit", - "url": "**********", - "api_key": "**********" - }, - "db_case_config": { - "metric_type": "COSINE", - "ef": -1, - "efConstruction": null, - "maxConnections": null - }, - "case_config": { - "case_id": 2, - "custom_case": null - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0, - "qps": 0, - "serial_latency_p99": 0, - "recall": 0 - }, - "task_config": { - "db": "WeaviateCloud", - "db_config": { - "db_label": "bus_crit", - "url": "**********", - "api_key": "**********" - }, - "db_case_config": { - "metric_type": "COSINE", - "ef": -1, - "efConstruction": null, - "maxConnections": null - }, - "case_config": { - "case_id": 4, - "custom_case": null - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0, - "qps": 0, - "serial_latency_p99": 0, - "recall": 0 - }, - "task_config": { - "db": "WeaviateCloud", - "db_config": { - "db_label": "bus_crit", - "url": "**********", - "api_key": "**********" - }, - "db_case_config": { - "metric_type": "COSINE", - "ef": -1, - "efConstruction": null, - "maxConnections": null - }, - "case_config": { - "case_id": 6, - "custom_case": null - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0, - "qps": 0, - "serial_latency_p99": 0, - "recall": 0 - }, - "task_config": { - "db": "WeaviateCloud", - "db_config": { - "db_label": "bus_crit", - "url": "**********", - "api_key": "**********" - }, - "db_case_config": { - "metric_type": "COSINE", - "ef": -1, - "efConstruction": null, - "maxConnections": null - }, - "case_config": { - "case_id": 8, - "custom_case": null - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 3674.0953, - "qps": 67.9121, - "serial_latency_p99": 0.1795, - "recall": 0.9909 - }, - "task_config": { - "db": "WeaviateCloud", - "db_config": { - "db_label": "bus_crit", - "url": "**********", - "api_key": "**********" - }, - "db_case_config": { - "metric_type": "COSINE", - "ef": -1, - "efConstruction": null, - "maxConnections": null - }, - "case_config": { - "case_id": 5, - "custom_case": null - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 3674.0953, - "qps": 0.7636, - "serial_latency_p99": 1.9213, - "recall": 0.9908 - }, - "task_config": { - "db": "WeaviateCloud", - "db_config": { - "db_label": "bus_crit", - "url": "**********", - "api_key": "**********" - }, - "db_case_config": { - "metric_type": "COSINE", - "ef": -1, - "efConstruction": null, - "maxConnections": null - }, - "case_config": { - "case_id": 7, - "custom_case": null - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 3674.0953, - "qps": 32, - "serial_latency_p99": 0.1245, - "recall": 1.0 - }, - "task_config": { - "db": "WeaviateCloud", - "db_config": { - "db_label": "bus_crit", - "url": "**********", - "api_key": "**********" - }, - "db_case_config": { - "metric_type": "COSINE", - "ef": -1, - "efConstruction": null, - "maxConnections": null - }, - "case_config": { - "case_id": 9, - "custom_case": null - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 1480000, - "load_duration": 0, - "qps": 0, - "serial_latency_p99": 0, - "recall": 0 - }, - "task_config": { - "db": "WeaviateCloud", - "db_config": { - "db_label": "sandbox", - "url": "**********", - "api_key": "**********" - }, - "db_case_config": { - "metric_type": "COSINE", - "ef": -1, - "efConstruction": null, - "maxConnections": null - }, - "case_config": { - "case_id": 1, - "custom_case": null - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 455000, - "load_duration": 0, - "qps": 0, - "serial_latency_p99": 0, - "recall": 0 - }, - "task_config": { - "db": "WeaviateCloud", - "db_config": { - "db_label": "sandbox", - "url": "**********", - "api_key": "**********" - }, - "db_case_config": { - "metric_type": "COSINE", - "ef": -1, - "efConstruction": null, - "maxConnections": null - }, - "case_config": { - "case_id": 2, - "custom_case": null - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0, - "qps": 0, - "serial_latency_p99": 0, - "recall": 0 - }, - "task_config": { - "db": "WeaviateCloud", - "db_config": { - "db_label": "sandbox", - "url": "**********", - "api_key": "**********" - }, - "db_case_config": { - "metric_type": "COSINE", - "ef": -1, - "efConstruction": null, - "maxConnections": null - }, - "case_config": { - "case_id": 5, - "custom_case": null - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0, - "qps": 0, - "serial_latency_p99": 0, - "recall": 0 - }, - "task_config": { - "db": "WeaviateCloud", - "db_config": { - "db_label": "sandbox", - "url": "**********", - "api_key": "**********" - }, - "db_case_config": { - "metric_type": "COSINE", - "ef": -1, - "efConstruction": null, - "maxConnections": null - }, - "case_config": { - "case_id": 7, - "custom_case": null - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0, - "qps": 0, - "serial_latency_p99": 0, - "recall": 0 - }, - "task_config": { - "db": "WeaviateCloud", - "db_config": { - "db_label": "sandbox", - "url": "**********", - "api_key": "**********" - }, - "db_case_config": { - "metric_type": "COSINE", - "ef": -1, - "efConstruction": null, - "maxConnections": null - }, - "case_config": { - "case_id": 9, - "custom_case": null - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0, - "qps": 0, - "serial_latency_p99": 0, - "recall": 0 - }, - "task_config": { - "db": "WeaviateCloud", - "db_config": { - "db_label": "sandbox", - "url": "**********", - "api_key": "**********" - }, - "db_case_config": { - "metric_type": "COSINE", - "ef": -1, - "efConstruction": null, - "maxConnections": null - }, - "case_config": { - "case_id": 4, - "custom_case": null - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0, - "qps": 0, - "serial_latency_p99": 0, - "recall": 0 - }, - "task_config": { - "db": "WeaviateCloud", - "db_config": { - "db_label": "sandbox", - "url": "**********", - "api_key": "**********" - }, - "db_case_config": { - "metric_type": "COSINE", - "ef": -1, - "efConstruction": null, - "maxConnections": null - }, - "case_config": { - "case_id": 6, - "custom_case": null - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0, - "qps": 0, - "serial_latency_p99": 0, - "recall": 0 - }, - "task_config": { - "db": "WeaviateCloud", - "db_config": { - "db_label": "sandbox", - "url": "**********", - "api_key": "**********" - }, - "db_case_config": { - "metric_type": "COSINE", - "ef": -1, - "efConstruction": null, - "maxConnections": null - }, - "case_config": { - "case_id": 8, - "custom_case": null - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 5500000, - "load_duration": 0, - "qps": 0, - "serial_latency_p99": 0, - "recall": 0 - }, - "task_config": { - "db": "WeaviateCloud", - "db_config": { - "db_label": "standard", - "url": "**********", - "api_key": "**********" - }, - "db_case_config": { - "metric_type": "COSINE", - "ef": -1, - "efConstruction": null, - "maxConnections": null - }, - "case_config": { - "case_id": 1, - "custom_case": null - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 1800000, - "load_duration": 0, - "qps": 0, - "serial_latency_p99": 0, - "recall": 0 - }, - "task_config": { - "db": "WeaviateCloud", - "db_config": { - "db_label": "standard", - "url": "**********", - "api_key": "**********" - }, - "db_case_config": { - "metric_type": "COSINE", - "ef": -1, - "efConstruction": null, - "maxConnections": null - }, - "case_config": { - "case_id": 2, - "custom_case": null - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 3580.7827, - "qps": 63.1365, - "serial_latency_p99": 0.1457, - "recall": 0.991 - }, - "task_config": { - "db": "WeaviateCloud", - "db_config": { - "db_label": "standard", - "url": "**********", - "api_key": "**********" - }, - "db_case_config": { - "metric_type": "COSINE", - "ef": -1, - "efConstruction": null, - "maxConnections": null - }, - "case_config": { - "case_id": 5, - "custom_case": null - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 3580.12, - "qps": 0.7512, - "serial_latency_p99": 1.9838, - "recall": 0.9908 - }, - "task_config": { - "db": "WeaviateCloud", - "db_config": { - "db_label": "standard", - "url": "**********", - "api_key": "**********" - }, - "db_case_config": { - "metric_type": "COSINE", - "ef": -1, - "efConstruction": 128, - "maxConnections": 64 - }, - "case_config": { - "case_id": 7, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0, - "qps": 30.1358, - "serial_latency_p99": 0.1298, - "recall": 1 - }, - "task_config": { - "db": "WeaviateCloud", - "db_config": { - "db_label": "standard", - "url": "**********", - "api_key": "**********" - }, - "db_case_config": { - "metric_type": "COSINE", - "ef": -1, - "efConstruction": 128, - "maxConnections": 64 - }, - "case_config": { - "case_id": 9, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0, - "qps": 0, - "serial_latency_p99": 0, - "recall": 0 - }, - "task_config": { - "db": "WeaviateCloud", - "db_config": { - "db_label": "standard", - "url": "**********", - "api_key": "**********" - }, - "db_case_config": { - "metric_type": "COSINE", - "ef": -1, - "efConstruction": null, - "maxConnections": null - }, - "case_config": { - "case_id": 4, - "custom_case": null - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0, - "qps": 0, - "serial_latency_p99": 0, - "recall": 0 - }, - "task_config": { - "db": "WeaviateCloud", - "db_config": { - "db_label": "standard", - "url": "**********", - "api_key": "**********" - }, - "db_case_config": { - "metric_type": "COSINE", - "ef": -1, - "efConstruction": null, - "maxConnections": null - }, - "case_config": { - "case_id": 6, - "custom_case": null - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0, - "qps": 0, - "serial_latency_p99": 0, - "recall": 0 - }, - "task_config": { - "db": "WeaviateCloud", - "db_config": { - "db_label": "standard", - "url": "**********", - "api_key": "**********" - }, - "db_case_config": { - "metric_type": "COSINE", - "ef": -1, - "efConstruction": null, - "maxConnections": null - }, - "case_config": { - "case_id": 8, - "custom_case": null - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 5000000, - "load_duration": 0, - "qps": 0, - "serial_latency_p99": 0, - "recall": 0 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-perf", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 1, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 1200000, - "load_duration": 0, - "qps": 0, - "serial_latency_p99": 0, - "recall": 0 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-perf", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 2, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 1100.6681, - "qps": 516.27, - "serial_latency_p99": 0.007, - "recall": 0.9463 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-perf", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 5, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 1100.6681, - "qps": 354.8416, - "serial_latency_p99": 0.01, - "recall": 0.9802 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-perf", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 7, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 1100.6681, - "qps": 427.5229, - "serial_latency_p99": 0.0087, - "recall": 1 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-perf", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 9, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-perf", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 4, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-perf", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 6, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-perf", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 8, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 1575.253, - "qps": 2884.689, - "serial_latency_p99": 0.0053, - "recall": 0.8801 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "8cu-perf", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 5, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 1853.989, - "qps": 1689.5799, - "serial_latency_p99": 0.0066, - "recall": 0.9493 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "8cu-perf", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 7, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 1575.253, - "qps": 1517.6792, - "serial_latency_p99": 0.01, - "recall": 1 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "8cu-perf", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 9, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 7635.0576, - "qps": 822.5318, - "serial_latency_p99": 0.0056, - "recall": 0.9294 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "8cu-perf", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 4, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 7635.0576, - "qps": 378.9146, - "serial_latency_p99": 0.0103, - "recall": 0.9758 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "8cu-perf", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 6, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 7635.0576, - "qps": 218.6854, - "serial_latency_p99": 0.0162, - "recall": 1 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "8cu-perf", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 8, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 9100000, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "Milvus", - "db_config": { - "db_label": "2c8g-hnsw", - "uri": "**********" - }, - "db_case_config": { - "index": "HNSW", - "metric_type": "COSINE", - "M": 30, - "efConstruction": 360, - "ef": 100 - }, - "case_config": { - "case_id": 1, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 1000000, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "Milvus", - "db_config": { - "db_label": "2c8g-hnsw", - "uri": "**********" - }, - "db_case_config": { - "index": "HNSW", - "metric_type": "COSINE", - "M": 30, - "efConstruction": 360, - "ef": 100 - }, - "case_config": { - "case_id": 2, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 2379.7118, - "qps": 274.5407, - "serial_latency_p99": 0.0049, - "recall": 0.9807 - }, - "task_config": { - "db": "Milvus", - "db_config": { - "db_label": "2c8g-hnsw", - "uri": "**********" - }, - "db_case_config": { - "index": "HNSW", - "metric_type": "COSINE", - "M": 30, - "efConstruction": 360, - "ef": 100 - }, - "case_config": { - "case_id": 5, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 2379.7118, - "qps": 236.5672, - "serial_latency_p99": 0.0103, - "recall": 0.981 - }, - "task_config": { - "db": "Milvus", - "db_config": { - "db_label": "2c8g-hnsw", - "uri": "**********" - }, - "db_case_config": { - "index": "HNSW", - "metric_type": "COSINE", - "M": 30, - "efConstruction": 360, - "ef": 100 - }, - "case_config": { - "case_id": 7, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 2379.7118, - "qps": 309.4833, - "serial_latency_p99": 0.0043, - "recall": 1 - }, - "task_config": { - "db": "Milvus", - "db_config": { - "db_label": "2c8g-hnsw", - "uri": "**********" - }, - "db_case_config": { - "index": "HNSW", - "metric_type": "COSINE", - "M": 30, - "efConstruction": 360, - "ef": 100 - }, - "case_config": { - "case_id": 9, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "Milvus", - "db_config": { - "db_label": "2c8g-hnsw", - "uri": "**********" - }, - "db_case_config": { - "index": "HNSW", - "metric_type": "COSINE", - "M": 30, - "efConstruction": 360, - "ef": 100 - }, - "case_config": { - "case_id": 4, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "Milvus", - "db_config": { - "db_label": "2c8g-hnsw", - "uri": "**********" - }, - "db_case_config": { - "index": "HNSW", - "metric_type": "COSINE", - "M": 30, - "efConstruction": 360, - "ef": 100 - }, - "case_config": { - "case_id": 6, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "Milvus", - "db_config": { - "db_label": "2c8g-hnsw", - "uri": "**********" - }, - "db_case_config": { - "index": "HNSW", - "metric_type": "COSINE", - "M": 30, - "efConstruction": 360, - "ef": 100 - }, - "case_config": { - "case_id": 8, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 5713.7159, - "qps": 178.6585, - "serial_latency_p99": 0.0137, - "recall": 0.9843 - }, - "task_config": { - "db": "Milvus", - "db_config": { - "db_label": "16c64g-hnsw", - "uri": "**********" - }, - "db_case_config": { - "index": "HNSW", - "metric_type": "COSINE", - "M": 30, - "efConstruction": 360, - "ef": 100 - }, - "case_config": { - "case_id": 4, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 5713.7159, - "qps": 178.3732, - "serial_latency_p99": 0.015, - "recall": 0.9844 - }, - "task_config": { - "db": "Milvus", - "db_config": { - "db_label": "16c64g-hnsw", - "uri": "**********" - }, - "db_case_config": { - "index": "HNSW", - "metric_type": "COSINE", - "M": 30, - "efConstruction": 360, - "ef": 100 - }, - "case_config": { - "case_id": 6, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 5713.7159, - "qps": 229.3526, - "serial_latency_p99": 0.0125, - "recall": 1 - }, - "task_config": { - "db": "Milvus", - "db_config": { - "db_label": "16c64g-hnsw", - "uri": "**********" - }, - "db_case_config": { - "index": "HNSW", - "metric_type": "COSINE", - "M": 30, - "efConstruction": 360, - "ef": 100 - }, - "case_config": { - "case_id": 8, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 581.8141, - "qps": 1258.7043, - "serial_latency_p99": 0.0049, - "recall": 0.9799 - }, - "task_config": { - "db": "Milvus", - "db_config": { - "db_label": "16c64g-hnsw", - "uri": "**********" - }, - "db_case_config": { - "index": "HNSW", - "metric_type": "COSINE", - "M": 30, - "efConstruction": 360, - "ef": 100 - }, - "case_config": { - "case_id": 5, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 581.8141, - "qps": 1075.8776, - "serial_latency_p99": 0.0053, - "recall": 0.98 - }, - "task_config": { - "db": "Milvus", - "db_config": { - "db_label": "16c64g-hnsw", - "uri": "**********" - }, - "db_case_config": { - "index": "HNSW", - "metric_type": "COSINE", - "M": 30, - "efConstruction": 360, - "ef": 100 - }, - "case_config": { - "case_id": 7, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 581.8141, - "qps": 1494.8493, - "serial_latency_p99": 0.0047, - "recall": 1 - }, - "task_config": { - "db": "Milvus", - "db_config": { - "db_label": "16c64g-hnsw", - "uri": "**********" - }, - "db_case_config": { - "index": "HNSW", - "metric_type": "COSINE", - "M": 30, - "efConstruction": 360, - "ef": 100 - }, - "case_config": { - "case_id": 9, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 4000000, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "QdrantCloud", - "db_config": { - "db_label": "2c8g-1node", - "url": "**********", - "api_key": "**********", - "prefer_grpc": true - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 1, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 900000, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "QdrantCloud", - "db_config": { - "db_label": "2c8g-1node", - "url": "**********", - "api_key": "**********", - "prefer_grpc": true - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 2, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "QdrantCloud", - "db_config": { - "db_label": "2c8g-1node", - "url": "**********", - "api_key": "**********", - "prefer_grpc": true - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 5, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "QdrantCloud", - "db_config": { - "db_label": "2c8g-1node", - "url": "**********", - "api_key": "**********", - "prefer_grpc": true - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 7, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "QdrantCloud", - "db_config": { - "db_label": "2c8g-1node", - "url": "**********", - "api_key": "**********", - "prefer_grpc": true - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 9, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "QdrantCloud", - "db_config": { - "db_label": "2c8g-1node", - "url": "**********", - "api_key": "**********", - "prefer_grpc": true - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 4, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "QdrantCloud", - "db_config": { - "db_label": "2c8g-1node", - "url": "**********", - "api_key": "**********", - "prefer_grpc": true - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 6, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "QdrantCloud", - "db_config": { - "db_label": "2c8g-1node", - "url": "**********", - "api_key": "**********", - "prefer_grpc": true - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 8, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 1647.857, - "qps": 537.4975, - "serial_latency_p99":0.0189, - "recall": 0.8903 - }, - "task_config": { - "db": "QdrantCloud", - "db_config": { - "db_label": "4c16g-5node", - "url": "**********", - "api_key": "**********" - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 5, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 1647.857, - "qps": 372.0466, - "serial_latency_p99": 0.0178, - "recall": 0.8904 - }, - "task_config": { - "db": "QdrantCloud", - "db_config": { - "db_label": "4c16g-5node", - "url": "**********", - "api_key": "**********" - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 7, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 1647.857, - "qps": 1156.2898, - "serial_latency_p99": 0.0144, - "recall": 0.9989 - }, - "task_config": { - "db": "QdrantCloud", - "db_config": { - "db_label": "4c16g-5node", - "url": "**********", - "api_key": "**********" - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 9, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 14623.578, - "qps": 110.248, - "serial_latency_p99": 0.069, - "recall": 0.898 - }, - "task_config": { - "db": "QdrantCloud", - "db_config": { - "db_label": "4c16g-5node", - "url": "**********", - "api_key": "**********", - "prefer_grpc": true - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 4, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 14623.578, - "qps": 87.2601, - "serial_latency_p99": 0.0278, - "recall":0.898 - }, - "task_config": { - "db": "QdrantCloud", - "db_config": { - "db_label": "4c16g-5node", - "url": "**********", - "api_key": "**********", - "prefer_grpc": true - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 6, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 14623.578, - "qps": 125.7846, - "serial_latency_p99": 0.0231, - "recall": 0.975 - }, - "task_config": { - "db": "QdrantCloud", - "db_config": { - "db_label": "4c16g-5node", - "url": "**********", - "api_key": "**********", - "prefer_grpc": true - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 8, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 1503.1147, - "qps": 240.7209, - "serial_latency_p99": 0.0174, - "recall": 0.8887 - }, - "task_config": { - "db": "QdrantCloud", - "db_config": { - "db_label": "4c16g-1node", - "url": "**********", - "api_key": "**********", - "prefer_grpc": true - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 5, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 1503.1147, - "qps": 189.4399, - "serial_latency_p99": 0.0175, - "recall": 0.8889 - }, - "task_config": { - "db": "QdrantCloud", - "db_config": { - "db_label": "4c16g-1node", - "url": "**********", - "api_key": "**********", - "prefer_grpc": true - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 7, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 1503.1147, - "qps": 313.5116, - "serial_latency_p99": 0.0161, - "recall": 0.9999 - }, - "task_config": { - "db": "QdrantCloud", - "db_config": { - "db_label": "4c16g-1node", - "url": "**********", - "api_key": "**********", - "prefer_grpc": true - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 9, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "QdrantCloud", - "db_config": { - "db_label": "4c16g-1node", - "url": "**********", - "api_key": "**********", - "prefer_grpc": true - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 4, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "QdrantCloud", - "db_config": { - "db_label": "4c16g-1node", - "url": "**********", - "api_key": "**********", - "prefer_grpc": true - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 6, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "QdrantCloud", - "db_config": { - "db_label": "4c16g-1node", - "url": "**********", - "api_key": "**********", - "prefer_grpc": true - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 8, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 5500000, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "Pinecone", - "db_config": { - "db_label": "s1.x1", - "api_key": "**********", - "environment": "**********", - "index_name": "test_index" - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 2, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 8300000, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "Pinecone", - "db_config": { - "db_label": "s1.x1", - "api_key": "**********", - "environment": "**********", - "index_name": "test_index" - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 1, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 1129.4487, - "qps": 18.7634, - "serial_latency_p99": 0.1537, - "recall": 0.8737 - }, - "task_config": { - "db": "Pinecone", - "db_config": { - "db_label": "s1.x1", - "api_key": "**********", - "environment": "**********", - "index_name": "test_index" - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 5, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 1129.4487, - "qps": 18.3619, - "serial_latency_p99": 0.0798, - "recall": 0.8741 - }, - "task_config": { - "db": "Pinecone", - "db_config": { - "db_label": "s1.x1", - "api_key": "**********", - "environment": "**********", - "index_name": "test_index" - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 7, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 1129.4487, - "qps": 25.2744, - "serial_latency_p99": 0.0612, - "recall": 0.9979 - }, - "task_config": { - "db": "Pinecone", - "db_config": { - "db_label": "s1.x1", - "api_key": "**********", - "environment": "**********", - "index_name": "test_index" - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 9, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "Pinecone", - "db_config": { - "db_label": "s1.x1", - "api_key": "**********", - "environment": "**********", - "index_name": "test_index" - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 4, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "Pinecone", - "db_config": { - "db_label": "s1.x1", - "api_key": "**********", - "environment": "**********", - "index_name": "test_index" - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 6, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "Pinecone", - "db_config": { - "db_label": "s1.x1", - "api_key": "**********", - "environment": "**********", - "index_name": "test_index" - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 8, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 1600000, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "Pinecone", - "db_config": { - "db_label": "p2.x1", - "api_key": "**********", - "environment": "**********", - "index_name": "test_index" - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 2, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 2700000, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "Pinecone", - "db_config": { - "db_label": "p2.x1", - "api_key": "**********", - "environment": "**********", - "index_name": "test_index" - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 1, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 1582.9409, - "qps": 261.798, - "serial_latency_p99": 0.0231, - "recall": 0.9262 - }, - "task_config": { - "db": "Pinecone", - "db_config": { - "db_label": "p2.x1", - "api_key": "**********", - "environment": "**********", - "index_name": "test_index" - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 5, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 1582.9409, - "qps": 166.1851, - "serial_latency_p99": 0.0239, - "recall": 0.9264 - }, - "task_config": { - "db": "Pinecone", - "db_config": { - "db_label": "p2.x1", - "api_key": "**********", - "environment": "**********", - "index_name": "test_index" - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 7, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 1582.9409, - "qps": 121.7169, - "serial_latency_p99": 0.029, - "recall": 0.9693 - }, - "task_config": { - "db": "Pinecone", - "db_config": { - "db_label": "p2.x1", - "api_key": "**********", - "environment": "**********", - "index_name": "test_index" - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 9, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "Pinecone", - "db_config": { - "db_label": "p2.x1", - "api_key": "**********", - "environment": "**********", - "index_name": "test_index" - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 4, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "Pinecone", - "db_config": { - "db_label": "p2.x1", - "api_key": "**********", - "environment": "**********", - "index_name": "test_index" - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 6, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "Pinecone", - "db_config": { - "db_label": "p2.x1", - "api_key": "**********", - "environment": "**********", - "index_name": "test_index" - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 8, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 700000, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "Pinecone", - "db_config": { - "db_label": "p1.x1", - "api_key": "**********", - "environment": "**********", - "index_name": "test_index" - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 2, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 4100000, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "Pinecone", - "db_config": { - "db_label": "p1.x1", - "api_key": "**********", - "environment": "**********", - "index_name": "test_index" - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 1, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 1123.9889, - "qps": 46.6189, - "serial_latency_p99": 0.0431, - "recall": 0.8737 - }, - "task_config": { - "db": "Pinecone", - "db_config": { - "db_label": "p1.x1", - "api_key": "**********", - "environment": "**********", - "index_name": "test_index" - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 5, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 1123.9889, - "qps": 42.4856, - "serial_latency_p99": 0.044, - "recall": 0.8741 - }, - "task_config": { - "db": "Pinecone", - "db_config": { - "db_label": "p1.x1", - "api_key": "**********", - "environment": "**********", - "index_name": "test_index" - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 7, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 1123.9889, - "qps": 138.9479, - "serial_latency_p99": 0.0262, - "recall": 0.9979 - }, - "task_config": { - "db": "Pinecone", - "db_config": { - "db_label": "p1.x1", - "api_key": "**********", - "environment": "**********", - "index_name": "test_index" - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 9, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "Pinecone", - "db_config": { - "db_label": "p1.x1", - "api_key": "**********", - "environment": "**********", - "index_name": "test_index" - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 4, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "Pinecone", - "db_config": { - "db_label": "p1.x1", - "api_key": "**********", - "environment": "**********", - "index_name": "test_index" - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 6, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "Pinecone", - "db_config": { - "db_label": "p1.x1", - "api_key": "**********", - "environment": "**********", - "index_name": "test_index" - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 8, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 1450.6205, - "qps": 20.7437, - "serial_latency_p99": 0.0758, - "recall": 0.9291 - }, - "task_config": { - "db": "Pinecone", - "db_config": { - "db_label": "s1.x1-2node", - "api_key": "**********", - "environment": "**********", - "index_name": "benchmark-test" - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 5, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 1450.6205, - "qps": 20.2993, - "serial_latency_p99": 0.0765, - "recall": 0.9293 - }, - "task_config": { - "db": "Pinecone", - "db_config": { - "db_label": "s1.x1-2node", - "api_key": "**********", - "environment": "**********", - "index_name": "benchmark-test" - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 7, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 1450.6205, - "qps": 26.4719, - "serial_latency_p99": 0.067, - "recall": 1.0 - }, - "task_config": { - "db": "Pinecone", - "db_config": { - "db_label": "s1.x1-2node", - "api_key": "**********", - "environment": "**********", - "index_name": "benchmark-test" - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 9, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 11374.7189, - "qps": 8.6675, - "serial_latency_p99": 0.1802, - "recall": 0.8369 - }, - "task_config": { - "db": "Pinecone", - "db_config": { - "db_label": "s1.x1-2node", - "api_key": "**********", - "environment": "**********", - "index_name": "benchmark-test" - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 4, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 11374.7189, - "qps": 7.8121, - "serial_latency_p99": 0.1677, - "recall": 0.8369 - }, - "task_config": { - "db": "Pinecone", - "db_config": { - "db_label": "s1.x1-2node", - "api_key": "**********", - "environment": "**********", - "index_name": "benchmark-test" - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 6, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 11374.7189, - "qps": 16.869, - "serial_latency_p99": 0.0878, - "recall": 0.9814 - }, - "task_config": { - "db": "Pinecone", - "db_config": { - "db_label": "s1.x1-2node", - "api_key": "**********", - "environment": "**********", - "index_name": "benchmark-test" - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 8, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 1210.2139, - "qps": 365.0835, - "serial_latency_p99": 0.0236, - "recall": 0.945 - }, - "task_config": { - "db": "Pinecone", - "db_config": { - "db_label": "p2.x1-8node", - "api_key": "**********", - "environment": "**********", - "index_name": "benchmark-test" - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 5, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 1210.2139, - "qps": 325.5271, - "serial_latency_p99": 0.0251, - "recall": 0.9452 - }, - "task_config": { - "db": "Pinecone", - "db_config": { - "db_label": "p2.x1-8node", - "api_key": "**********", - "environment": "**********", - "index_name": "benchmark-test" - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 7, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 1210.2139, - "qps": 596.7942, - "serial_latency_p99": 0.0242, - "recall": 0.9693 - }, - "task_config": { - "db": "Pinecone", - "db_config": { - "db_label": "p2.x1-8node", - "api_key": "**********", - "environment": "**********", - "index_name": "benchmark-test" - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 9, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 14165.6504, - "qps": 303.2044, - "serial_latency_p99": 0.0274, - "recall": 0.9246 - }, - "task_config": { - "db": "Pinecone", - "db_config": { - "db_label": "p2.x1-8node", - "api_key": "**********", - "environment": "**********", - "index_name": "benchmark-test" - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 4, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 14165.6504, - "qps": 136.0345, - "serial_latency_p99": 0.0319, - "recall": 0.9244 - }, - "task_config": { - "db": "Pinecone", - "db_config": { - "db_label": "p2.x1-8node", - "api_key": "**********", - "environment": "**********", - "index_name": "benchmark-test" - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 6, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 14165.6504, - "qps": 66.7221, - "serial_latency_p99": 0.0421, - "recall": 0.963 - }, - "task_config": { - "db": "Pinecone", - "db_config": { - "db_label": "p2.x1-8node", - "api_key": "**********", - "environment": "**********", - "index_name": "benchmark-test" - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 8, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 1211.8883, - "qps": 131.2549, - "serial_latency_p99": 0.0302, - "recall": 0.9867 - }, - "task_config": { - "db": "Pinecone", - "db_config": { - "db_label": "p1.x1-8node", - "api_key": "**********", - "environment": "**********", - "index_name": "benchmark-test" - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 5, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 1211.8883, - "qps": 127.9337, - "serial_latency_p99": 0.0301, - "recall": 0.9869 - }, - "task_config": { - "db": "Pinecone", - "db_config": { - "db_label": "p1.x1-8node", - "api_key": "**********", - "environment": "**********", - "index_name": "benchmark-test" - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 7, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 1211.8883, - "qps": 595.8462, - "serial_latency_p99": 0.0234, - "recall": 1.0 - }, - "task_config": { - "db": "Pinecone", - "db_config": { - "db_label": "p1.x1-8node", - "api_key": "**********", - "environment": "**********", - "index_name": "benchmark-test" - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 9, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "Pinecone", - "db_config": { - "db_label": "p1.x1-8node", - "api_key": "**********", - "environment": "**********", - "index_name": "benchmark-test" - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 4, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "Pinecone", - "db_config": { - "db_label": "p1.x1-8node", - "api_key": "**********", - "environment": "**********", - "index_name": "benchmark-test" - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 6, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "Pinecone", - "db_config": { - "db_label": "p1.x1-8node", - "api_key": "**********", - "environment": "**********", - "index_name": "benchmark-test" - }, - "db_case_config": { - "null": null - }, - "case_config": { - "case_id": 8, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "PgVector", - "db_config":{ - "db_label":"2c8g-1node", - "user_name":"**********", - "password":"**********", - "url":"**********", - "db_name":"**********" - }, - "db_case_config":{ - "metric_type":"L2", - "lists":10, - "probes":2 - }, - "case_config":{ - "case_id":1, - "custom_case":{ - - } - } - }, - "label": "?" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "PgVector", - "db_config":{ - "db_label":"2c8g-1node", - "user_name":"**********", - "password":"**********", - "url":"**********", - "db_name":"**********" - }, - "db_case_config":{ - "metric_type":"L2", - "lists":10, - "probes":2 - }, - "case_config":{ - "case_id":2, - "custom_case":{ - - } - } - }, - "label": "?" - }, - { - "metrics":{ - "max_load_count":0, - "load_duration":10247.5034, - "qps":10.6271, - "serial_latency_p99":0.7307, - "recall":0.8898 - }, - "task_config":{ - "db":"PgVector", - "db_config":{ - "db_label":"2c8g-1node", - "user_name":"**********", - "password":"**********", - "url":"**********", - "db_name":"**********" - }, - "db_case_config":{ - "metric_type":"COSINE", - "lists":10, - "probes":2 - }, - "case_config":{ - "case_id":5, - "custom_case":{ - - } - } - }, - "label":":)" - }, - { - "metrics":{ - "max_load_count":0, - "load_duration":10247.5034, - "qps":10.8507, - "serial_latency_p99":0.7332, - "recall":0.8897 - }, - "task_config":{ - "db":"PgVector", - "db_config":{ - "db_label":"2c8g-1node", - "user_name":"**********", - "password":"**********", - "url":"**********", - "db_name":"**********" - }, - "db_case_config":{ - "metric_type":"COSINE", - "lists":10, - "probes":2 - }, - "case_config":{ - "case_id":7, - "custom_case":{ - - } - } - }, - "label":":)" - }, - { - "metrics":{ - "max_load_count":0, - "load_duration":10247.5034, - "qps":75.7055, - "serial_latency_p99":0.1212, - "recall":0.9999 - }, - "task_config":{ - "db":"PgVector", - "db_config":{ - "db_label":"2c8g-1node", - "user_name":"**********", - "password":"**********", - "url":"**********", - "db_name":"**********" - }, - "db_case_config":{ - "metric_type":"COSINE", - "lists":10, - "probes":2 - }, - "case_config":{ - "case_id":9, - "custom_case":{ - - } - } - }, - "label":":)" - }, - { - "metrics":{ - "max_load_count":0, - "load_duration":0, - "qps":0, - "serial_latency_p99":0, - "recall":0 - }, - "task_config":{ - "db":"PgVector", - "db_config":{ - "db_label":"2c8g-1node", - "user_name":"**********", - "password":"**********", - "url":"**********", - "db_name":"**********" - }, - "db_case_config":{ - "metric_type":"COSINE", - "lists":10, - "probes":2 - }, - "case_config":{ - "case_id":4, - "custom_case":{ - - } - } - }, - "label":"x" - }, - { - "metrics":{ - "max_load_count":0, - "load_duration":0, - "qps":0, - "serial_latency_p99":0, - "recall":0 - }, - "task_config":{ - "db":"PgVector", - "db_config":{ - "db_label":"2c8g-1node", - "user_name":"**********", - "password":"**********", - "url":"**********", - "db_name":"**********" - }, - "db_case_config":{ - "metric_type":"COSINE", - "lists":10, - "probes":2 - }, - "case_config":{ - "case_id":6, - "custom_case":{ - - } - } - }, - "label":"x" - }, - { - "metrics":{ - "max_load_count":0, - "load_duration":0, - "qps":0, - "serial_latency_p99":0, - "recall":0 - }, - "task_config":{ - "db":"PgVector", - "db_config":{ - "db_label":"2c8g-1node", - "user_name":"**********", - "password":"**********", - "url":"**********", - "db_name":"**********" - }, - "db_case_config":{ - "metric_type":"COSINE", - "lists":10, - "probes":2 - }, - "case_config":{ - "case_id":8, - "custom_case":{ - - } - } - }, - "label":"x" - } - ] -} \ No newline at end of file From 34ce4d3c6680d5b12f5f0bd4f1c003a82eb62d1f Mon Sep 17 00:00:00 2001 From: zhuwenxing Date: Fri, 30 Jun 2023 17:59:58 +0800 Subject: [PATCH 008/327] add dataset with dim 1536 and case for it Signed-off-by: zhuwenxing --- vectordb_bench/__init__.py | 2 + vectordb_bench/backend/cases.py | 85 ++++++++++++++++++- vectordb_bench/backend/dataset.py | 12 +++ .../components/run_test/generateTasks.py | 30 +++---- .../frontend/const/dbCaseConfigs.py | 33 +++++++ 5 files changed, 145 insertions(+), 17 deletions(-) diff --git a/vectordb_bench/__init__.py b/vectordb_bench/__init__.py index 2d7d1ad2c..25febc22d 100644 --- a/vectordb_bench/__init__.py +++ b/vectordb_bench/__init__.py @@ -20,10 +20,12 @@ class config: RESULTS_LOCAL_DIR = pathlib.Path(__file__).parent.joinpath("results") CAPACITY_TIMEOUT_IN_SECONDS = 24 * 3600 # 24h + LOAD_TIMEOUT_DEFAULT = 2.5 * 3600 # 2.5h LOAD_TIMEOUT_1M = 2.5 * 3600 # 2.5h LOAD_TIMEOUT_10M = 25 * 3600 # 25h LOAD_TIMEOUT_100M = 250 * 3600 # 10.41d + OPTIMIZE_TIMEOUT_DEFAULT = 15 * 60 # 15min OPTIMIZE_TIMEOUT_1M = 15 * 60 # 15min OPTIMIZE_TIMEOUT_10M = 2.5 * 3600 # 2.5h OPTIMIZE_TIMEOUT_100M = 25 * 3600 # 1.04d diff --git a/vectordb_bench/backend/cases.py b/vectordb_bench/backend/cases.py index 56b38e3f4..c5d4cc3f4 100644 --- a/vectordb_bench/backend/cases.py +++ b/vectordb_bench/backend/cases.py @@ -34,6 +34,14 @@ class CaseType(Enum): Performance10M99P = 8 Performance1M99P = 9 + Performance500K = 10 + Performance5M = 11 + + Performance500K1P = 12 + Performance5M1P = 13 + Performance500K99P = 14 + Performance5M99P = 15 + Custom = 100 @property @@ -104,7 +112,8 @@ class CapacityCase(Case, BaseModel): class PerformanceCase(Case, BaseModel): label: CaseLabel = CaseLabel.Performance filter_rate: float | None = None - + load_timeout: float | int = config.LOAD_TIMEOUT_DEFAULT + optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_DEFAULT class CapacityDim960(CapacityCase): case_id: CaseType = CaseType.CapacityDim960 @@ -186,7 +195,6 @@ class Performance1M99P(PerformanceCase): optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_1M - class Performance100M(PerformanceCase): case_id: CaseType = CaseType.Performance100M filter_rate: float | int | None = None @@ -198,6 +206,69 @@ class Performance100M(PerformanceCase): optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_100M +class Performance500K(PerformanceCase): + case_id: CaseType = CaseType.Performance500K + filter_rate: float | int | None = None + dataset: DatasetManager = Dataset.OPENAI.manager(500_000) + name: str = "Search Performance Test (500K Dataset, 1536 Dim)" + description: str = """This case tests the search performance of a vector database with a medium 500K dataset (OpenAI 500K vectors, 1536 dimensions), at varying parallel levels. +Results will show index building time, recall, and maximum QPS.""" + load_timeout: float | int = config.LOAD_TIMEOUT_1M + optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_1M + + +class Performance5M(PerformanceCase): + case_id: CaseType = CaseType.Performance5M + filter_rate: float | int | None = None + dataset: DatasetManager = Dataset.OPENAI.manager(5_000_000) + name: str = "Search Performance Test (5M Dataset, 1536 Dim)" + description: str = """This case tests the search performance of a vector database with a medium 5M dataset (OpenAI 5M vectors, 1536 dimensions), at varying parallel levels. +Results will show index building time, recall, and maximum QPS.""" + load_timeout: float | int = config.LOAD_TIMEOUT_10M + optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_10M + + +class Performance500K1P(PerformanceCase): + case_id: CaseType = CaseType.Performance500K1P + filter_rate: float | int | None = 0.01 + dataset: DatasetManager = Dataset.OPENAI.manager(500_000) + name: str = "Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)" + description: str = """This case tests the search performance of a vector database with a large dataset (OpenAI 500K vectors, 1536 dimensions) under a low filtering rate (1% vectors), at varying parallel levels. +Results will show index building time, recall, and maximum QPS.""" + load_timeout: float | int = config.LOAD_TIMEOUT_1M + optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_1M + +class Performance5M1P(PerformanceCase): + case_id: CaseType = CaseType.Performance5M1P + filter_rate: float | int | None = 0.01 + dataset: DatasetManager = Dataset.OPENAI.manager(5_000_000) + name: str = "Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)" + description: str = """This case tests the search performance of a vector database with a large dataset (OpenAI 5M vectors, 1536 dimensions) under a low filtering rate (1% vectors), at varying parallel levels. +Results will show index building time, recall, and maximum QPS.""" + load_timeout: float | int = config.LOAD_TIMEOUT_10M + optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_10M + +class Performance500K99P(PerformanceCase): + case_id: CaseType = CaseType.Performance500K99P + filter_rate: float | int | None = 0.99 + dataset: DatasetManager = Dataset.OPENAI.manager(500_000) + name: str = "Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)" + description: str = """This case tests the search performance of a vector database with a medium dataset (OpenAI 500K vectors, 1536 dimensions) under a high filtering rate (99% vectors), at varying parallel levels. +Results will show index building time, recall, and maximum QPS.""" + load_timeout: float | int = config.LOAD_TIMEOUT_1M + optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_1M + +class Performance5M99P(PerformanceCase): + case_id: CaseType = CaseType.Performance5M99P + filter_rate: float | int | None = 0.99 + dataset: DatasetManager = Dataset.OPENAI.manager(5_000_000) + name: str = "Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)" + description: str = """This case tests the search performance of a vector database with a medium dataset (OpenAI 5M vectors, 1536 dimensions) under a high filtering rate (99% vectors), at varying parallel levels. +Results will show index building time, recall, and maximum QPS.""" + load_timeout: float | int = config.LOAD_TIMEOUT_10M + optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_10M + + type2case = { CaseType.CapacityDim960: CapacityDim960, CaseType.CapacityDim128: CapacityDim128, @@ -210,4 +281,14 @@ class Performance100M(PerformanceCase): CaseType.Performance1M1P: Performance1M1P, CaseType.Performance10M99P: Performance10M99P, CaseType.Performance1M99P: Performance1M99P, + + CaseType.Performance500K: Performance500K, + CaseType.Performance5M: Performance5M, + + CaseType.Performance500K1P: Performance500K1P, + CaseType.Performance5M1P: Performance5M1P, + + CaseType.Performance500K99P: Performance500K99P, + CaseType.Performance5M99P: Performance5M99P, + } diff --git a/vectordb_bench/backend/dataset.py b/vectordb_bench/backend/dataset.py index eafecd1b7..86b6a1402 100644 --- a/vectordb_bench/backend/dataset.py +++ b/vectordb_bench/backend/dataset.py @@ -97,6 +97,17 @@ class SIFT(BaseDataset): 50_000_000: "LARGE", } +class OpenAI(BaseDataset): + name: str = "OpenAI" + dim: int = 1536 + metric_type: MetricType = MetricType.COSINE + use_shuffled: bool = False + _size_label: dict = { + 50_000: "SMALL", + 500_000: "MEDIUM", + 5_000_000: "LARGE", + } + class DatasetManager(BaseModel): """Download dataset if not int the local directory. Provide data for cases. @@ -319,6 +330,7 @@ class Dataset(Enum): COHERE = Cohere GLOVE = Glove SIFT = SIFT + OPENAI = OpenAI def get(self, size: int) -> BaseDataset: return self.value(size=size) diff --git a/vectordb_bench/frontend/components/run_test/generateTasks.py b/vectordb_bench/frontend/components/run_test/generateTasks.py index d8422a61f..0c847d2cc 100644 --- a/vectordb_bench/frontend/components/run_test/generateTasks.py +++ b/vectordb_bench/frontend/components/run_test/generateTasks.py @@ -2,20 +2,20 @@ def generate_tasks(activedDbList, dbConfigs, activedCaseList, allCaseConfigs): - tasks = [ - TaskConfig( - db=db.value, - db_config=dbConfigs[db], - case_config=CaseConfig( - case_id=case.value, - custom_case={}, - ), - db_case_config=db.init_cls.case_config_cls( - allCaseConfigs[db][case].get(CaseConfigParamType.IndexType, None) - )(**{key.value: value for key, value in allCaseConfigs[db][case].items()}), - ) - for case in activedCaseList - for db in activedDbList - ] + tasks = [] + for db in activedDbList: + for case in activedCaseList: + task = TaskConfig( + db=db.value, + db_config=dbConfigs[db], + case_config=CaseConfig( + case_id=case.value, + custom_case={}, + ), + db_case_config=db.init_cls.case_config_cls( + allCaseConfigs[db][case].get(CaseConfigParamType.IndexType, None) + )(**{key.value: value for key, value in allCaseConfigs[db][case].items()}), + ) + tasks.append(task) return tasks diff --git a/vectordb_bench/frontend/const/dbCaseConfigs.py b/vectordb_bench/frontend/const/dbCaseConfigs.py index 6bba020e6..f496b2743 100644 --- a/vectordb_bench/frontend/const/dbCaseConfigs.py +++ b/vectordb_bench/frontend/const/dbCaseConfigs.py @@ -17,12 +17,21 @@ CaseType.Performance10M, CaseType.Performance1M, DIVIDER, + CaseType.Performance5M, + CaseType.Performance500K, + DIVIDER, CaseType.Performance10M1P, CaseType.Performance1M1P, DIVIDER, + CaseType.Performance5M1P, + CaseType.Performance500K1P, + DIVIDER, CaseType.Performance10M99P, CaseType.Performance1M99P, DIVIDER, + CaseType.Performance5M99P, + CaseType.Performance500K99P, + DIVIDER, CaseType.CapacityDim960, CaseType.CapacityDim128, ] @@ -254,6 +263,12 @@ class CaseConfigInput(BaseModel): CaseType.Performance1M1P: MilvusPerformanceConfig, CaseType.Performance10M99P: MilvusPerformanceConfig, CaseType.Performance1M99P: MilvusPerformanceConfig, + CaseType.Performance5M: MilvusPerformanceConfig, + CaseType.Performance500K: MilvusPerformanceConfig, + CaseType.Performance5M1P: MilvusPerformanceConfig, + CaseType.Performance500K1P: MilvusPerformanceConfig, + CaseType.Performance5M99P: MilvusPerformanceConfig, + CaseType.Performance500K99P: MilvusPerformanceConfig, }, DB.WeaviateCloud: { CaseType.CapacityDim960: WeaviateLoadConfig, @@ -265,6 +280,12 @@ class CaseConfigInput(BaseModel): CaseType.Performance1M1P: WeaviatePerformanceConfig, CaseType.Performance10M99P: WeaviatePerformanceConfig, CaseType.Performance1M99P: WeaviatePerformanceConfig, + CaseType.Performance5M: WeaviatePerformanceConfig, + CaseType.Performance500K: WeaviatePerformanceConfig, + CaseType.Performance5M1P: WeaviatePerformanceConfig, + CaseType.Performance500K1P: WeaviatePerformanceConfig, + CaseType.Performance5M99P: WeaviatePerformanceConfig, + CaseType.Performance500K99P: WeaviatePerformanceConfig, }, DB.ElasticCloud: { CaseType.CapacityDim960: ESLoadingConfig, @@ -276,6 +297,12 @@ class CaseConfigInput(BaseModel): CaseType.Performance1M1P: ESPerformanceConfig, CaseType.Performance10M99P: ESPerformanceConfig, CaseType.Performance1M99P: ESPerformanceConfig, + CaseType.Performance5M: ESPerformanceConfig, + CaseType.Performance500K: ESPerformanceConfig, + CaseType.Performance5M1P: ESPerformanceConfig, + CaseType.Performance500K1P: ESPerformanceConfig, + CaseType.Performance5M99P: ESPerformanceConfig, + CaseType.Performance500K99P: ESPerformanceConfig, }, DB.PgVector: { CaseType.CapacityDim960: PgVectorLoadingConfig, @@ -287,5 +314,11 @@ class CaseConfigInput(BaseModel): CaseType.Performance1M1P: PgVectorPerformanceConfig, CaseType.Performance10M99P: PgVectorPerformanceConfig, CaseType.Performance1M99P: PgVectorPerformanceConfig, + CaseType.Performance5M: PgVectorPerformanceConfig, + CaseType.Performance500K: PgVectorPerformanceConfig, + CaseType.Performance5M1P: PgVectorPerformanceConfig, + CaseType.Performance500K1P: PgVectorPerformanceConfig, + CaseType.Performance5M99P: PgVectorPerformanceConfig, + CaseType.Performance500K99P: PgVectorPerformanceConfig, }, } From c2a882bc0f5ba499f371237e685eb637b927f6bd Mon Sep 17 00:00:00 2001 From: "min.tian" Date: Tue, 18 Jul 2023 19:06:59 +0800 Subject: [PATCH 009/327] [dataset] openai will use_shuffle_train data Signed-off-by: min.tian --- vectordb_bench/backend/dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vectordb_bench/backend/dataset.py b/vectordb_bench/backend/dataset.py index 86b6a1402..8c2b0ceef 100644 --- a/vectordb_bench/backend/dataset.py +++ b/vectordb_bench/backend/dataset.py @@ -101,7 +101,7 @@ class OpenAI(BaseDataset): name: str = "OpenAI" dim: int = 1536 metric_type: MetricType = MetricType.COSINE - use_shuffled: bool = False + use_shuffled: bool = config.USE_SHUFFLED_DATA _size_label: dict = { 50_000: "SMALL", 500_000: "MEDIUM", From f136d66d351594d727bc957cf2cc31b16b714a65 Mon Sep 17 00:00:00 2001 From: Li Liu Date: Fri, 11 Aug 2023 18:42:53 +0800 Subject: [PATCH 010/327] Add 1536 data Signed-off-by: Li Liu --- ...result_20230808_standard_elasticcloud.json | 187 ++++ .../result_20230808_standard_milvus.json | 655 ++++++++++++ .../result_20230808_standard_pgvector.json | 181 ++++ .../result_20230808_standard_pinecone.json | 943 ++++++++++++++++++ .../result_20230808_standard_qdrantcloud.json | 457 +++++++++ ...esult_20230808_standard_weaviatecloud.json | 343 +++++++ .../result_20230808_standard_zillizcloud.json | 655 ++++++++++++ 7 files changed, 3421 insertions(+) create mode 100644 vectordb_bench/results/ElasticCloud/result_20230808_standard_elasticcloud.json create mode 100644 vectordb_bench/results/Milvus/result_20230808_standard_milvus.json create mode 100644 vectordb_bench/results/PgVector/result_20230808_standard_pgvector.json create mode 100644 vectordb_bench/results/Pinecone/result_20230808_standard_pinecone.json create mode 100644 vectordb_bench/results/QdrantCloud/result_20230808_standard_qdrantcloud.json create mode 100644 vectordb_bench/results/WeaviateCloud/result_20230808_standard_weaviatecloud.json create mode 100644 vectordb_bench/results/ZillizCloud/result_20230808_standard_zillizcloud.json diff --git a/vectordb_bench/results/ElasticCloud/result_20230808_standard_elasticcloud.json b/vectordb_bench/results/ElasticCloud/result_20230808_standard_elasticcloud.json new file mode 100644 index 000000000..53dc3e6a1 --- /dev/null +++ b/vectordb_bench/results/ElasticCloud/result_20230808_standard_elasticcloud.json @@ -0,0 +1,187 @@ +{ + "run_id": "5c1e8bd468224ffda1b39b08cdc342c3", + "task_label": "standard", + "results": [ + { + "metrics": { + "max_load_count": 0, + "load_duration": 8671.2705, + "qps": 11.2945, + "serial_latency_p99": 3.6112, + "recall": 0.996 + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "upTo2.5c8g", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "metric_type": "COSINE", + "efConstruction": 360, + "M": 30, + "num_candidates": null + }, + "case_config": { + "case_id": 10, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "upTo2.5c8g", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "metric_type": "COSINE", + "efConstruction": 360, + "M": 30, + "num_candidates": null + }, + "case_config": { + "case_id": 11, + "custom_case": {} + } + }, + "label": "?" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 8671.2705, + "qps": 17.3271, + "serial_latency_p99": 3.7748, + "recall": 0.9961 + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "upTo2.5c8g", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "metric_type": "COSINE", + "efConstruction": 360, + "M": 30, + "num_candidates": null + }, + "case_config": { + "case_id": 12, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "upTo2.5c8g", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "metric_type": "COSINE", + "efConstruction": 360, + "M": 30, + "num_candidates": null + }, + "case_config": { + "case_id": 13, + "custom_case": {} + } + }, + "label": "?" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 8671.2705, + "qps": 26.26, + "serial_latency_p99": 0.5561, + "recall": 0.9999 + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "upTo2.5c8g", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "metric_type": "COSINE", + "efConstruction": 360, + "M": 30, + "num_candidates": null + }, + "case_config": { + "case_id": 14, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "upTo2.5c8g", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "metric_type": "COSINE", + "efConstruction": 360, + "M": 30, + "num_candidates": null + }, + "case_config": { + "case_id": 15, + "custom_case": {} + } + }, + "label": "?" + } + ], + "file_fmt": "result_{}_{}_{}.json" +} \ No newline at end of file diff --git a/vectordb_bench/results/Milvus/result_20230808_standard_milvus.json b/vectordb_bench/results/Milvus/result_20230808_standard_milvus.json new file mode 100644 index 000000000..bfc48ac58 --- /dev/null +++ b/vectordb_bench/results/Milvus/result_20230808_standard_milvus.json @@ -0,0 +1,655 @@ +{ + "run_id": "5c1e8bd468224ffda1b39b08cdc342c3", + "task_label": "standard", + "results": [ + { + "metrics": { + "max_load_count": 0, + "load_duration": 6548.0874, + "qps": 37.432, + "serial_latency_p99": 0.075, + "recall": 0.9975 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "2c8g-disk", + "uri": "**********" + }, + "db_case_config": { + "index": "DISKANN", + "metric_type": "COSINE", + "search_list": 100 + }, + "case_config": { + "case_id": 10, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "2c8g-disk", + "uri": "**********" + }, + "db_case_config": { + "index": "DISKANN", + "metric_type": "COSINE", + "search_list": 100 + }, + "case_config": { + "case_id": 11, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 6548.0874, + "qps": 37.0696, + "serial_latency_p99": 0.0735, + "recall": 0.9976 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "2c8g-disk", + "uri": "**********" + }, + "db_case_config": { + "index": "DISKANN", + "metric_type": "COSINE", + "search_list": 100 + }, + "case_config": { + "case_id": 12, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "2c8g-disk", + "uri": "**********" + }, + "db_case_config": { + "index": "DISKANN", + "metric_type": "COSINE", + "search_list": 100 + }, + "case_config": { + "case_id": 13, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 6548.0874, + "qps": 81.1915, + "serial_latency_p99": 0.053, + "recall": 1.0 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "2c8g-disk", + "uri": "**********" + }, + "db_case_config": { + "index": "DISKANN", + "metric_type": "COSINE", + "search_list": 100 + }, + "case_config": { + "case_id": 14, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "2c8g-disk", + "uri": "**********" + }, + "db_case_config": { + "index": "DISKANN", + "metric_type": "COSINE", + "search_list": 100 + }, + "case_config": { + "case_id": 15, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 5548.1138, + "qps": 321.6048, + "serial_latency_p99": 0.0134, + "recall": 0.989 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "4c16g-disk", + "uri": "**********" + }, + "db_case_config": { + "index": "DISKANN", + "metric_type": "COSINE", + "search_list": 100 + }, + "case_config": { + "case_id": 10, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 51197.1096, + "qps": 22.1467, + "serial_latency_p99": 0.0868, + "recall": 0.9972 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "4c16g-disk", + "uri": "**********" + }, + "db_case_config": { + "index": "DISKANN", + "metric_type": "COSINE", + "search_list": 100 + }, + "case_config": { + "case_id": 11, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 5548.1138, + "qps": 303.2551, + "serial_latency_p99": 0.0109, + "recall": 0.9876 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "4c16g-disk", + "uri": "**********" + }, + "db_case_config": { + "index": "DISKANN", + "metric_type": "COSINE", + "search_list": 100 + }, + "case_config": { + "case_id": 12, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 51197.1096, + "qps": 21.5388, + "serial_latency_p99": 0.0817, + "recall": 0.997 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "4c16g-disk", + "uri": "**********" + }, + "db_case_config": { + "index": "DISKANN", + "metric_type": "COSINE", + "search_list": 100 + }, + "case_config": { + "case_id": 13, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 5548.1138, + "qps": 394.5418, + "serial_latency_p99": 0.0083, + "recall": 1.0 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "4c16g-disk", + "uri": "**********" + }, + "db_case_config": { + "index": "DISKANN", + "metric_type": "COSINE", + "search_list": 100 + }, + "case_config": { + "case_id": 14, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 51197.1096, + "qps": 37.878, + "serial_latency_p99": 0.045, + "recall": 1.0 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "4c16g-disk", + "uri": "**********" + }, + "db_case_config": { + "index": "DISKANN", + "metric_type": "COSINE", + "search_list": 100 + }, + "case_config": { + "case_id": 15, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 3268.311, + "qps": 180.2757, + "serial_latency_p99": 0.006, + "recall": 0.9942 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "2c8g-hnsw", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 10, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "2c8g-hnsw", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 11, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 3268.311, + "qps": 179.0033, + "serial_latency_p99": 0.0064, + "recall": 0.9943 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "2c8g-hnsw", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 12, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "2c8g-hnsw", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 13, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 3268.311, + "qps": 526.8846, + "serial_latency_p99": 0.0036, + "recall": 1.0 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "2c8g-hnsw", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 14, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "2c8g-hnsw", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 15, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 622.1326, + "qps": 626.5243, + "serial_latency_p99": 0.0062, + "recall": 0.9954 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-hnsw", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 10, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 6533.9989, + "qps": 78.4227, + "serial_latency_p99": 0.0253, + "recall": 0.9982 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-hnsw", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 11, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 622.1326, + "qps": 599.4213, + "serial_latency_p99": 0.0066, + "recall": 0.9955 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-hnsw", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 12, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 6533.9989, + "qps": 78.5351, + "serial_latency_p99": 0.0263, + "recall": 0.9982 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-hnsw", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 13, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 622.1326, + "qps": 2098.2113, + "serial_latency_p99": 0.0034, + "recall": 1.0 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-hnsw", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 14, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 6533.9989, + "qps": 275.6292, + "serial_latency_p99": 0.01, + "recall": 1.0 + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-hnsw", + "uri": "**********" + }, + "db_case_config": { + "index": "HNSW", + "metric_type": "COSINE", + "M": 30, + "efConstruction": 360, + "ef": 100 + }, + "case_config": { + "case_id": 15, + "custom_case": {} + } + }, + "label": ":)" + } + ], + "file_fmt": "result_{}_{}_{}.json" +} \ No newline at end of file diff --git a/vectordb_bench/results/PgVector/result_20230808_standard_pgvector.json b/vectordb_bench/results/PgVector/result_20230808_standard_pgvector.json new file mode 100644 index 000000000..16bc838c0 --- /dev/null +++ b/vectordb_bench/results/PgVector/result_20230808_standard_pgvector.json @@ -0,0 +1,181 @@ +{ + "run_id": "5c1e8bd468224ffda1b39b08cdc342c3", + "task_label": "standard", + "results": [ + { + "metrics": { + "max_load_count": 0, + "load_duration": 1380.9471, + "qps": 0.8836, + "serial_latency_p99": 2.523, + "recall": 0.8528 + }, + "task_config": { + "db": "PgVector", + "db_config": { + "db_label": "2c8g-60gdisk", + "user_name": "**********", + "password": "**********", + "url": "**********", + "db_name": "**********" + }, + "db_case_config": { + "metric_type": "L2", + "lists": 10, + "probes": 2 + }, + "case_config": { + "case_id": 10, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "PgVector", + "db_config": { + "db_label": "2c8g-60gdisk", + "user_name": "**********", + "password": "**********", + "url": "**********", + "db_name": "**********" + }, + "db_case_config": { + "metric_type": "L2", + "lists": 10, + "probes": 2 + }, + "case_config": { + "case_id": 11, + "custom_case": {} + } + }, + "label": "?" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1380.9471, + "qps": 0.8937, + "serial_latency_p99": 3.7202, + "recall": 0.8525 + }, + "task_config": { + "db": "PgVector", + "db_config": { + "db_label": "2c8g-60gdisk", + "user_name": "**********", + "password": "**********", + "url": "**********", + "db_name": "**********" + }, + "db_case_config": { + "metric_type": "L2", + "lists": 10, + "probes": 2 + }, + "case_config": { + "case_id": 12, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "PgVector", + "db_config": { + "db_label": "2c8g-60gdisk", + "user_name": "**********", + "password": "**********", + "url": "**********", + "db_name": "**********" + }, + "db_case_config": { + "metric_type": "L2", + "lists": 10, + "probes": 2 + }, + "case_config": { + "case_id": 13, + "custom_case": {} + } + }, + "label": "?" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1372.7522, + "qps": 1.2145, + "serial_latency_p99": 3.6224, + "recall": 0.7487 + }, + "task_config": { + "db": "PgVector", + "db_config": { + "db_label": "2c8g-60gdisk", + "user_name": "**********", + "password": "**********", + "url": "**********", + "db_name": "**********" + }, + "db_case_config": { + "metric_type": "L2", + "lists": 10, + "probes": 2 + }, + "case_config": { + "case_id": 14, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "PgVector", + "db_config": { + "db_label": "2c8g-60gdisk", + "user_name": "**********", + "password": "**********", + "url": "**********", + "db_name": "**********" + }, + "db_case_config": { + "metric_type": "L2", + "lists": 10, + "probes": 2 + }, + "case_config": { + "case_id": 15, + "custom_case": {} + } + }, + "label": "?" + } + ], + "file_fmt": "result_{}_{}_{}.json" +} \ No newline at end of file diff --git a/vectordb_bench/results/Pinecone/result_20230808_standard_pinecone.json b/vectordb_bench/results/Pinecone/result_20230808_standard_pinecone.json new file mode 100644 index 000000000..9ec826447 --- /dev/null +++ b/vectordb_bench/results/Pinecone/result_20230808_standard_pinecone.json @@ -0,0 +1,943 @@ +{ + "run_id": "5c1e8bd468224ffda1b39b08cdc342c3", + "task_label": "standard", + "results": [ + { + "metrics": { + "max_load_count": 0, + "load_duration": 1439.0, + "qps": 228.4, + "serial_latency_p99": 0.0222, + "recall": 0.9348 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p2.x1", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 10, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p2.x1", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 11, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1439.0, + "qps": 181.5, + "serial_latency_p99": 0.0261, + "recall": 0.9345 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p2.x1", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 12, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p2.x1", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 13, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1439.0, + "qps": 205.7, + "serial_latency_p99": 0.0242, + "recall": 0.9586 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p2.x1", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 14, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p2.x1", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 15, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1409.0, + "qps": 67.63, + "serial_latency_p99": 0.036, + "recall": 0.8064 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p1.x1", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 10, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p1.x1", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 11, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1409.0, + "qps": 63.35, + "serial_latency_p99": 0.0384, + "recall": 0.8065 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p1.x1", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 12, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p1.x1", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 13, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1409.0, + "qps": 176.7, + "serial_latency_p99": 0.0276, + "recall": 1.0 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p1.x1", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 14, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p1.x1", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 15, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1125.0, + "qps": 15.33, + "serial_latency_p99": 0.0849, + "recall": 0.8064 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "s1.x1", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 10, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "s1.x1", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 11, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1125.0, + "qps": 15.13, + "serial_latency_p99": 0.0867, + "recall": 0.8065 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "s1.x1", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 12, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "s1.x1", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 13, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1125.0, + "qps": 17.41, + "serial_latency_p99": 0.0743, + "recall": 1.0 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "s1.x1", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 14, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "s1.x1", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 15, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1383.0, + "qps": 16.34, + "serial_latency_p99": 0.0887, + "recall": 0.879 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "s1.x1-2node", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 10, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 13960.0, + "qps": 10.45, + "serial_latency_p99": 0.1268, + "recall": 0.8208 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "s1.x1-2node", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 11, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1383.0, + "qps": 16.18, + "serial_latency_p99": 0.0875, + "recall": 0.8793 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "s1.x1-2node", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 12, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 13960.0, + "qps": 9.8, + "serial_latency_p99": 0.1309, + "recall": 0.8212 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "s1.x1-2node", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 13, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1383.0, + "qps": 36.11, + "serial_latency_p99": 0.0551, + "recall": 1.0 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "s1.x1-2node", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 14, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 13960.0, + "qps": 14.84, + "serial_latency_p99": 0.0927, + "recall": 0.96 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "s1.x1-2node", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 15, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1132.0, + "qps": 322.7, + "serial_latency_p99": 0.0264, + "recall": 0.9478 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p2.x1-8node", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 10, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 11430.0, + "qps": 265.5, + "serial_latency_p99": 0.0269, + "recall": 0.9332 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p2.x1-8node", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 11, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1132.0, + "qps": 303.8, + "serial_latency_p99": 0.0273, + "recall": 0.9478 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p2.x1-8node", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 12, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 11430.0, + "qps": 180.2, + "serial_latency_p99": 0.0282, + "recall": 0.9335 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p2.x1-8node", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 13, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1132.0, + "qps": 730.7, + "serial_latency_p99": 0.0246, + "recall": 0.9586 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p2.x1-8node", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 14, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 11430.0, + "qps": 104.3, + "serial_latency_p99": 0.0317, + "recall": 0.9563 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p2.x1-8node", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 15, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p1.x1-8node", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 10, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p1.x1-8node", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 11, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1396.0, + "qps": 147.7, + "serial_latency_p99": 0.0353, + "recall": 0.9707 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p1.x1-8node", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 12, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p1.x1-8node", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 13, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1396.0, + "qps": 782.5, + "serial_latency_p99": 0.0259, + "recall": 1.0 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p1.x1-8node", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 14, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p1.x1-8node", + "api_key": "**********", + "environment": "**********", + "index_name": "test_index" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 15, + "custom_case": {} + } + }, + "label": "x" + } + ], + "file_fmt": "result_{}_{}_{}.json" +} \ No newline at end of file diff --git a/vectordb_bench/results/QdrantCloud/result_20230808_standard_qdrantcloud.json b/vectordb_bench/results/QdrantCloud/result_20230808_standard_qdrantcloud.json new file mode 100644 index 000000000..af41537a8 --- /dev/null +++ b/vectordb_bench/results/QdrantCloud/result_20230808_standard_qdrantcloud.json @@ -0,0 +1,457 @@ +{ + "run_id": "5c1e8bd468224ffda1b39b08cdc342c3", + "task_label": "standard", + "results": [ + { + "metrics": { + "max_load_count": 0, + "load_duration": 1817.9087, + "qps": 78.7196, + "serial_latency_p99": 0.0494, + "recall": 0.9203 + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "2c8g-1node", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": null + }, + "case_config": { + "case_id": 10, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "2c8g-1node", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": null + }, + "case_config": { + "case_id": 11, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1817.9087, + "qps": 68.3111, + "serial_latency_p99": 0.0355, + "recall": 0.9202 + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "2c8g-1node", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": null + }, + "case_config": { + "case_id": 12, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "2c8g-1node", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": null + }, + "case_config": { + "case_id": 13, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1817.9087, + "qps": 210.2147, + "serial_latency_p99": 0.0268, + "recall": 0.9996 + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "2c8g-1node", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": null + }, + "case_config": { + "case_id": 14, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "2c8g-1node", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": null + }, + "case_config": { + "case_id": 15, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1398.8057, + "qps": 188.6436, + "serial_latency_p99": 0.9175, + "recall": 0.9175 + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "4c16g-1node", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": null + }, + "case_config": { + "case_id": 10, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "4c16g-1node", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": null + }, + "case_config": { + "case_id": 11, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1398.8057, + "qps": 155.6991, + "serial_latency_p99": 0.9171, + "recall": 0.9171 + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "4c16g-1node", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": null + }, + "case_config": { + "case_id": 12, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "4c16g-1node", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": null + }, + "case_config": { + "case_id": 13, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1398.8057, + "qps": 445.3289, + "serial_latency_p99": 0.0141, + "recall": 0.9999 + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "4c16g-1node", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": null + }, + "case_config": { + "case_id": 14, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "4c16g-1node", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": null + }, + "case_config": { + "case_id": 15, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1523.7192, + "qps": 633.6033, + "serial_latency_p99": 0.0246, + "recall": 0.919 + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "4c16g-5node", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": null + }, + "case_config": { + "case_id": 10, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 13191.326, + "qps": 95.5682, + "serial_latency_p99": 0.0587, + "recall": 0.9463 + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "4c16g-5node", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": null + }, + "case_config": { + "case_id": 11, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1523.7192, + "qps": 434.4062, + "serial_latency_p99": 0.0174, + "recall": 0.9181 + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "4c16g-5node", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": null + }, + "case_config": { + "case_id": 12, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 13191.326, + "qps": 80.3192, + "serial_latency_p99": 0.0266, + "recall": 0.9462 + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "4c16g-5node", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": null + }, + "case_config": { + "case_id": 13, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1523.7192, + "qps": 1509.3293, + "serial_latency_p99": 0.0185, + "recall": 0.9995 + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "4c16g-5node", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": null + }, + "case_config": { + "case_id": 14, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 13191.326, + "qps": 166.7252, + "serial_latency_p99": 0.0196, + "recall": 0.9988 + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "4c16g-5node", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": null + }, + "case_config": { + "case_id": 15, + "custom_case": {} + } + }, + "label": ":)" + } + ], + "file_fmt": "result_{}_{}_{}.json" +} \ No newline at end of file diff --git a/vectordb_bench/results/WeaviateCloud/result_20230808_standard_weaviatecloud.json b/vectordb_bench/results/WeaviateCloud/result_20230808_standard_weaviatecloud.json new file mode 100644 index 000000000..ec89d5129 --- /dev/null +++ b/vectordb_bench/results/WeaviateCloud/result_20230808_standard_weaviatecloud.json @@ -0,0 +1,343 @@ +{ + "run_id": "5c1e8bd468224ffda1b39b08cdc342c3", + "task_label": "standard", + "results": [ + { + "metrics": { + "max_load_count": 0, + "load_duration": 2973.0, + "qps": 46.8622, + "serial_latency_p99": 0.123, + "recall": 0.9957 + }, + "task_config": { + "db": "WeaviateCloud", + "db_config": { + "db_label": "standard", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "ef": -1, + "efConstruction": 128, + "maxConnections": 64 + }, + "case_config": { + "case_id": 10, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "WeaviateCloud", + "db_config": { + "db_label": "standard", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "ef": -1, + "efConstruction": 128, + "maxConnections": 64 + }, + "case_config": { + "case_id": 11, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 2973.0, + "qps": 1.8388, + "serial_latency_p99": 1.0635, + "recall": 0.9957 + }, + "task_config": { + "db": "WeaviateCloud", + "db_config": { + "db_label": "standard", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "ef": -1, + "efConstruction": 128, + "maxConnections": 64 + }, + "case_config": { + "case_id": 12, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "WeaviateCloud", + "db_config": { + "db_label": "standard", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "ef": -1, + "efConstruction": 128, + "maxConnections": 64 + }, + "case_config": { + "case_id": 13, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 2973.0, + "qps": 45.0666, + "serial_latency_p99": 0.1092, + "recall": 1.0 + }, + "task_config": { + "db": "WeaviateCloud", + "db_config": { + "db_label": "standard", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "ef": -1, + "efConstruction": 128, + "maxConnections": 64 + }, + "case_config": { + "case_id": 14, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "WeaviateCloud", + "db_config": { + "db_label": "standard", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "ef": -1, + "efConstruction": 128, + "maxConnections": 64 + }, + "case_config": { + "case_id": 15, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 3292.0, + "qps": 43.5017, + "serial_latency_p99": 0.2287, + "recall": 0.9957 + }, + "task_config": { + "db": "WeaviateCloud", + "db_config": { + "db_label": "bus_crit", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "ef": -1, + "efConstruction": 128, + "maxConnections": 64 + }, + "case_config": { + "case_id": 10, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "WeaviateCloud", + "db_config": { + "db_label": "bus_crit", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "ef": -1, + "efConstruction": 128, + "maxConnections": 64 + }, + "case_config": { + "case_id": 11, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 3292.0, + "qps": 1.5668, + "serial_latency_p99": 1.1144, + "recall": 0.9957 + }, + "task_config": { + "db": "WeaviateCloud", + "db_config": { + "db_label": "bus_crit", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "ef": -1, + "efConstruction": 128, + "maxConnections": 64 + }, + "case_config": { + "case_id": 12, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "WeaviateCloud", + "db_config": { + "db_label": "bus_crit", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "ef": -1, + "efConstruction": 128, + "maxConnections": 64 + }, + "case_config": { + "case_id": 13, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 3292.0, + "qps": 41.5443, + "serial_latency_p99": 0.159, + "recall": 1.0 + }, + "task_config": { + "db": "WeaviateCloud", + "db_config": { + "db_label": "bus_crit", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "ef": -1, + "efConstruction": 128, + "maxConnections": 64 + }, + "case_config": { + "case_id": 14, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "WeaviateCloud", + "db_config": { + "db_label": "bus_crit", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "ef": -1, + "efConstruction": 128, + "maxConnections": 64 + }, + "case_config": { + "case_id": 15, + "custom_case": {} + } + }, + "label": "x" + } + ], + "file_fmt": "result_{}_{}_{}.json" +} \ No newline at end of file diff --git a/vectordb_bench/results/ZillizCloud/result_20230808_standard_zillizcloud.json b/vectordb_bench/results/ZillizCloud/result_20230808_standard_zillizcloud.json new file mode 100644 index 000000000..8d992d523 --- /dev/null +++ b/vectordb_bench/results/ZillizCloud/result_20230808_standard_zillizcloud.json @@ -0,0 +1,655 @@ +{ + "run_id": "5c1e8bd468224ffda1b39b08cdc342c3", + "task_label": "standard", + "results": [ + { + "metrics": { + "max_load_count": 0, + "load_duration": 1079.0, + "qps": 297.5, + "serial_latency_p99": 0.0072, + "recall": 0.974 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-perf", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 10, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-perf", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 11, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1060.0, + "qps": 228.3, + "serial_latency_p99": 0.0106, + "recall": 0.994 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-perf", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 12, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-perf", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 13, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1067.0, + "qps": 584.0, + "serial_latency_p99": 0.0046, + "recall": 1.0 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-perf", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 14, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-perf", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 15, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1786.0, + "qps": 1871.0, + "serial_latency_p99": 0.007, + "recall": 0.9602 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 10, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 6848.0, + "qps": 556.7, + "serial_latency_p99": 0.0067, + "recall": 0.9723 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 11, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1786.0, + "qps": 1583.0, + "serial_latency_p99": 0.0068, + "recall": 0.9836 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 12, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 6976.0, + "qps": 294.3, + "serial_latency_p99": 0.0109, + "recall": 0.9939 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 13, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1786.0, + "qps": 2345.0, + "serial_latency_p99": 0.0089, + "recall": 1.0 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 14, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 6976.0, + "qps": 295.6, + "serial_latency_p99": 0.0123, + "recall": 1.0 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 15, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 2408.0, + "qps": 143.0, + "serial_latency_p99": 0.0335, + "recall": 0.9818 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-cap", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 10, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-cap", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 11, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 2408.0, + "qps": 106.0, + "serial_latency_p99": 0.0207, + "recall": 0.9887 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-cap", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 12, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-cap", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 13, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 2408.0, + "qps": 189.0, + "serial_latency_p99": 0.0116, + "recall": 1.0 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-cap", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 14, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-cap", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 15, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "2cu-cap", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 10, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 15050.0, + "qps": 71.74, + "serial_latency_p99": 0.0508, + "recall": 0.9883 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "2cu-cap", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 11, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 2586.0, + "qps": 287.0, + "serial_latency_p99": 0.0149, + "recall": 0.9865 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "2cu-cap", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 12, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 13793.0, + "qps": 34.6654, + "serial_latency_p99": 0.0647, + "recall": 0.9961 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "2cu-cap", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 13, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 2586.0, + "qps": 412.0, + "serial_latency_p99": 0.0103, + "recall": 1.0 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "2cu-cap", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 14, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 13793.0, + "qps": 42.169, + "serial_latency_p99": 0.0468, + "recall": 1.0 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "2cu-cap", + "uri": "**********", + "user": "root", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 15, + "custom_case": {} + } + }, + "label": ":)" + } + ], + "file_fmt": "result_{}_{}_{}.json" +} \ No newline at end of file From a6f9eaf69912365923565eeb08843226d1f1e759 Mon Sep 17 00:00:00 2001 From: Li Liu Date: Mon, 14 Aug 2023 11:40:05 +0800 Subject: [PATCH 011/327] Fix variables' name to fit different dim Signed-off-by: Li Liu --- vectordb_bench/backend/cases.py | 156 +++++++++--------- .../frontend/const/dbCaseConfigs.py | 130 +++++++-------- 2 files changed, 143 insertions(+), 143 deletions(-) diff --git a/vectordb_bench/backend/cases.py b/vectordb_bench/backend/cases.py index c5d4cc3f4..7f40ae4f8 100644 --- a/vectordb_bench/backend/cases.py +++ b/vectordb_bench/backend/cases.py @@ -25,22 +25,22 @@ class CaseType(Enum): CapacityDim128 = 1 CapacityDim960 = 2 - Performance100M = 3 - Performance10M = 4 - Performance1M = 5 + Performance768D100M = 3 + Performance768D10M = 4 + Performance768D1M = 5 - Performance10M1P = 6 - Performance1M1P = 7 - Performance10M99P = 8 - Performance1M99P = 9 + Performance768D10M1P = 6 + Performance768D1M1P = 7 + Performance768D10M99P = 8 + Performance768D1M99P = 9 - Performance500K = 10 - Performance5M = 11 + Performance1536D500K = 10 + Performance1536D5M = 11 - Performance500K1P = 12 - Performance5M1P = 13 - Performance500K99P = 14 - Performance5M99P = 15 + Performance1536D500K1P = 12 + Performance1536D5M1P = 13 + Performance1536D500K99P = 14 + Performance1536D5M99P = 15 Custom = 100 @@ -131,164 +131,164 @@ class CapacityDim128(CapacityCase): Number of inserted vectors will be reported.""" -class Performance10M(PerformanceCase): - case_id: CaseType = CaseType.Performance10M +class Performance768D10M(PerformanceCase): + case_id: CaseType = CaseType.Performance768D10M dataset: DatasetManager = Dataset.COHERE.manager(10_000_000) name: str = "Search Performance Test (10M Dataset, 768 Dim)" description: str = """This case tests the search performance of a vector database with a large dataset (Cohere 10M vectors, 768 dimensions) at varying parallel levels. Results will show index building time, recall, and maximum QPS.""" - load_timeout: float | int = config.LOAD_TIMEOUT_10M - optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_10M + load_timeout: float | int = config.LOAD_TIMEOUT_768D_10M + optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_768D_10M -class Performance1M(PerformanceCase): - case_id: CaseType = CaseType.Performance1M +class Performance768D1M(PerformanceCase): + case_id: CaseType = CaseType.Performance768D1M dataset: DatasetManager = Dataset.COHERE.manager(1_000_000) name: str = "Search Performance Test (1M Dataset, 768 Dim)" description: str = """This case tests the search performance of a vector database with a medium dataset (Cohere 1M vectors, 768 dimensions) at varying parallel levels. Results will show index building time, recall, and maximum QPS.""" - load_timeout: float | int = config.LOAD_TIMEOUT_1M - optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_1M + load_timeout: float | int = config.LOAD_TIMEOUT_768D_1M + optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_768D_1M -class Performance10M1P(PerformanceCase): - case_id: CaseType = CaseType.Performance10M1P +class Performance768D10M1P(PerformanceCase): + case_id: CaseType = CaseType.Performance768D10M1P filter_rate: float | int | None = 0.01 dataset: DatasetManager = Dataset.COHERE.manager(10_000_000) name: str = "Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)" description: str = """This case tests the search performance of a vector database with a large dataset (Cohere 10M vectors, 768 dimensions) under a low filtering rate (1% vectors), at varying parallel levels. Results will show index building time, recall, and maximum QPS.""" - load_timeout: float | int = config.LOAD_TIMEOUT_10M - optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_10M + load_timeout: float | int = config.LOAD_TIMEOUT_768D_10M + optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_768D_10M -class Performance1M1P(PerformanceCase): - case_id: CaseType = CaseType.Performance1M1P +class Performance768D1M1P(PerformanceCase): + case_id: CaseType = CaseType.Performance768D1M1P filter_rate: float | int | None = 0.01 dataset: DatasetManager = Dataset.COHERE.manager(1_000_000) name: str = "Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)" description: str = """This case tests the search performance of a vector database with a medium dataset (Cohere 1M vectors, 768 dimensions) under a low filtering rate (1% vectors), at varying parallel levels. Results will show index building time, recall, and maximum QPS.""" - load_timeout: float | int = config.LOAD_TIMEOUT_1M - optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_1M + load_timeout: float | int = config.LOAD_TIMEOUT_768D_1M + optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_768D_1M -class Performance10M99P(PerformanceCase): - case_id: CaseType = CaseType.Performance10M99P +class Performance768D10M99P(PerformanceCase): + case_id: CaseType = CaseType.Performance768D10M99P filter_rate: float | int | None = 0.99 dataset: DatasetManager = Dataset.COHERE.manager(10_000_000) name: str = "Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)" description: str = """This case tests the search performance of a vector database with a large dataset (Cohere 10M vectors, 768 dimensions) under a high filtering rate (99% vectors), at varying parallel levels. Results will show index building time, recall, and maximum QPS.""" - load_timeout: float | int = config.LOAD_TIMEOUT_10M - optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_10M + load_timeout: float | int = config.LOAD_TIMEOUT_768D_10M + optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_768D_10M -class Performance1M99P(PerformanceCase): - case_id: CaseType = CaseType.Performance1M99P +class Performance768D1M99P(PerformanceCase): + case_id: CaseType = CaseType.Performance768D1M99P filter_rate: float | int | None = 0.99 dataset: DatasetManager = Dataset.COHERE.manager(1_000_000) name: str = "Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)" description: str = """This case tests the search performance of a vector database with a medium dataset (Cohere 1M vectors, 768 dimensions) under a high filtering rate (99% vectors), at varying parallel levels. Results will show index building time, recall, and maximum QPS.""" - load_timeout: float | int = config.LOAD_TIMEOUT_1M - optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_1M + load_timeout: float | int = config.LOAD_TIMEOUT_768D_1M + optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_768D_1M -class Performance100M(PerformanceCase): - case_id: CaseType = CaseType.Performance100M +class Performance768D100M(PerformanceCase): + case_id: CaseType = CaseType.Performance768D100M filter_rate: float | int | None = None dataset: DatasetManager = Dataset.LAION.manager(100_000_000) name: str = "Search Performance Test (100M Dataset, 768 Dim)" description: str = """This case tests the search performance of a vector database with a large 100M dataset (LAION 100M vectors, 768 dimensions), at varying parallel levels. Results will show index building time, recall, and maximum QPS.""" - load_timeout: float | int = config.LOAD_TIMEOUT_100M - optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_100M + load_timeout: float | int = config.LOAD_TIMEOUT_768D_100M + optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_768D_100M -class Performance500K(PerformanceCase): - case_id: CaseType = CaseType.Performance500K +class Performance1536D500K(PerformanceCase): + case_id: CaseType = CaseType.Performance1536D500K filter_rate: float | int | None = None dataset: DatasetManager = Dataset.OPENAI.manager(500_000) name: str = "Search Performance Test (500K Dataset, 1536 Dim)" description: str = """This case tests the search performance of a vector database with a medium 500K dataset (OpenAI 500K vectors, 1536 dimensions), at varying parallel levels. Results will show index building time, recall, and maximum QPS.""" - load_timeout: float | int = config.LOAD_TIMEOUT_1M - optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_1M + load_timeout: float | int = config.LOAD_TIMEOUT_1536D_500K + optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_1536D_500K -class Performance5M(PerformanceCase): - case_id: CaseType = CaseType.Performance5M +class Performance1536D5M(PerformanceCase): + case_id: CaseType = CaseType.Performance1536D5M filter_rate: float | int | None = None dataset: DatasetManager = Dataset.OPENAI.manager(5_000_000) name: str = "Search Performance Test (5M Dataset, 1536 Dim)" description: str = """This case tests the search performance of a vector database with a medium 5M dataset (OpenAI 5M vectors, 1536 dimensions), at varying parallel levels. Results will show index building time, recall, and maximum QPS.""" - load_timeout: float | int = config.LOAD_TIMEOUT_10M - optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_10M + load_timeout: float | int = config.LOAD_TIMEOUT_1536D_5M + optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_1536D_5M -class Performance500K1P(PerformanceCase): - case_id: CaseType = CaseType.Performance500K1P +class Performance1536D500K1P(PerformanceCase): + case_id: CaseType = CaseType.Performance1536D500K1P filter_rate: float | int | None = 0.01 dataset: DatasetManager = Dataset.OPENAI.manager(500_000) name: str = "Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)" description: str = """This case tests the search performance of a vector database with a large dataset (OpenAI 500K vectors, 1536 dimensions) under a low filtering rate (1% vectors), at varying parallel levels. Results will show index building time, recall, and maximum QPS.""" - load_timeout: float | int = config.LOAD_TIMEOUT_1M - optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_1M + load_timeout: float | int = config.LOAD_TIMEOUT_1536D_500K + optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_1536D_500K -class Performance5M1P(PerformanceCase): - case_id: CaseType = CaseType.Performance5M1P +class Performance1536D5M1P(PerformanceCase): + case_id: CaseType = CaseType.Performance1536D5M1P filter_rate: float | int | None = 0.01 dataset: DatasetManager = Dataset.OPENAI.manager(5_000_000) name: str = "Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)" description: str = """This case tests the search performance of a vector database with a large dataset (OpenAI 5M vectors, 1536 dimensions) under a low filtering rate (1% vectors), at varying parallel levels. Results will show index building time, recall, and maximum QPS.""" - load_timeout: float | int = config.LOAD_TIMEOUT_10M - optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_10M + load_timeout: float | int = config.LOAD_TIMEOUT_1536D_5M + optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_1536D_5M -class Performance500K99P(PerformanceCase): - case_id: CaseType = CaseType.Performance500K99P +class Performance1536D500K99P(PerformanceCase): + case_id: CaseType = CaseType.Performance1536D500K99P filter_rate: float | int | None = 0.99 dataset: DatasetManager = Dataset.OPENAI.manager(500_000) name: str = "Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)" description: str = """This case tests the search performance of a vector database with a medium dataset (OpenAI 500K vectors, 1536 dimensions) under a high filtering rate (99% vectors), at varying parallel levels. Results will show index building time, recall, and maximum QPS.""" - load_timeout: float | int = config.LOAD_TIMEOUT_1M - optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_1M + load_timeout: float | int = config.LOAD_TIMEOUT_1536D_500K + optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_1536D_500K -class Performance5M99P(PerformanceCase): - case_id: CaseType = CaseType.Performance5M99P +class Performance1536D5M99P(PerformanceCase): + case_id: CaseType = CaseType.Performance1536D5M99P filter_rate: float | int | None = 0.99 dataset: DatasetManager = Dataset.OPENAI.manager(5_000_000) name: str = "Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)" description: str = """This case tests the search performance of a vector database with a medium dataset (OpenAI 5M vectors, 1536 dimensions) under a high filtering rate (99% vectors), at varying parallel levels. Results will show index building time, recall, and maximum QPS.""" - load_timeout: float | int = config.LOAD_TIMEOUT_10M - optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_10M + load_timeout: float | int = config.LOAD_TIMEOUT_1536D_5M + optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_1536D_5M type2case = { CaseType.CapacityDim960: CapacityDim960, CaseType.CapacityDim128: CapacityDim128, - CaseType.Performance100M: Performance100M, - CaseType.Performance10M: Performance10M, - CaseType.Performance1M: Performance1M, + CaseType.Performance768D100M: Performance768D100M, + CaseType.Performance768D10M: Performance768D10M, + CaseType.Performance768D1M: Performance768D1M, - CaseType.Performance10M1P: Performance10M1P, - CaseType.Performance1M1P: Performance1M1P, - CaseType.Performance10M99P: Performance10M99P, - CaseType.Performance1M99P: Performance1M99P, + CaseType.Performance768D10M1P: Performance768D10M1P, + CaseType.Performance768D1M1P: Performance768D1M1P, + CaseType.Performance768D10M99P: Performance768D10M99P, + CaseType.Performance768D1M99P: Performance768D1M99P, - CaseType.Performance500K: Performance500K, - CaseType.Performance5M: Performance5M, + CaseType.Performance1536D500K: Performance1536D500K, + CaseType.Performance1536D5M: Performance1536D5M, - CaseType.Performance500K1P: Performance500K1P, - CaseType.Performance5M1P: Performance5M1P, + CaseType.Performance1536D500K1P: Performance1536D500K1P, + CaseType.Performance1536D5M1P: Performance1536D5M1P, - CaseType.Performance500K99P: Performance500K99P, - CaseType.Performance5M99P: Performance5M99P, + CaseType.Performance1536D500K99P: Performance1536D500K99P, + CaseType.Performance1536D5M99P: Performance1536D5M99P, } diff --git a/vectordb_bench/frontend/const/dbCaseConfigs.py b/vectordb_bench/frontend/const/dbCaseConfigs.py index f496b2743..bff623bb0 100644 --- a/vectordb_bench/frontend/const/dbCaseConfigs.py +++ b/vectordb_bench/frontend/const/dbCaseConfigs.py @@ -13,24 +13,24 @@ DIVIDER = "DIVIDER" CASE_LIST_WITH_DIVIDER = [ - CaseType.Performance100M, - CaseType.Performance10M, - CaseType.Performance1M, + CaseType.Performance768D100M, + CaseType.Performance768D10M, + CaseType.Performance768D1M, DIVIDER, - CaseType.Performance5M, - CaseType.Performance500K, + CaseType.Performance1536D5M, + CaseType.Performance1536D500K, DIVIDER, - CaseType.Performance10M1P, - CaseType.Performance1M1P, + CaseType.Performance768D10M1P, + CaseType.Performance768D1M1P, DIVIDER, - CaseType.Performance5M1P, - CaseType.Performance500K1P, + CaseType.Performance1536D5M1P, + CaseType.Performance1536D500K1P, DIVIDER, - CaseType.Performance10M99P, - CaseType.Performance1M99P, + CaseType.Performance768D10M99P, + CaseType.Performance768D1M99P, DIVIDER, - CaseType.Performance5M99P, - CaseType.Performance500K99P, + CaseType.Performance1536D5M99P, + CaseType.Performance1536D500K99P, DIVIDER, CaseType.CapacityDim960, CaseType.CapacityDim128, @@ -256,69 +256,69 @@ class CaseConfigInput(BaseModel): DB.Milvus: { CaseType.CapacityDim960: MilvusLoadConfig, CaseType.CapacityDim128: MilvusLoadConfig, - CaseType.Performance100M: MilvusPerformanceConfig, - CaseType.Performance10M: MilvusPerformanceConfig, - CaseType.Performance1M: MilvusPerformanceConfig, - CaseType.Performance10M1P: MilvusPerformanceConfig, - CaseType.Performance1M1P: MilvusPerformanceConfig, - CaseType.Performance10M99P: MilvusPerformanceConfig, - CaseType.Performance1M99P: MilvusPerformanceConfig, - CaseType.Performance5M: MilvusPerformanceConfig, - CaseType.Performance500K: MilvusPerformanceConfig, - CaseType.Performance5M1P: MilvusPerformanceConfig, - CaseType.Performance500K1P: MilvusPerformanceConfig, - CaseType.Performance5M99P: MilvusPerformanceConfig, - CaseType.Performance500K99P: MilvusPerformanceConfig, + CaseType.Performance768D100M: MilvusPerformanceConfig, + CaseType.Performance768D10M: MilvusPerformanceConfig, + CaseType.Performance768D1M: MilvusPerformanceConfig, + CaseType.Performance768D10M1P: MilvusPerformanceConfig, + CaseType.Performance768D1M1P: MilvusPerformanceConfig, + CaseType.Performance768D10M99P: MilvusPerformanceConfig, + CaseType.Performance768D1M99P: MilvusPerformanceConfig, + CaseType.Performance1536D5M: MilvusPerformanceConfig, + CaseType.Performance1536D500K: MilvusPerformanceConfig, + CaseType.Performance1536D5M1P: MilvusPerformanceConfig, + CaseType.Performance1536D500K1P: MilvusPerformanceConfig, + CaseType.Performance1536D5M99P: MilvusPerformanceConfig, + CaseType.Performance1536D500K99P: MilvusPerformanceConfig, }, DB.WeaviateCloud: { CaseType.CapacityDim960: WeaviateLoadConfig, CaseType.CapacityDim128: WeaviateLoadConfig, - CaseType.Performance100M: WeaviatePerformanceConfig, - CaseType.Performance10M: WeaviatePerformanceConfig, - CaseType.Performance1M: WeaviatePerformanceConfig, - CaseType.Performance10M1P: WeaviatePerformanceConfig, - CaseType.Performance1M1P: WeaviatePerformanceConfig, - CaseType.Performance10M99P: WeaviatePerformanceConfig, - CaseType.Performance1M99P: WeaviatePerformanceConfig, - CaseType.Performance5M: WeaviatePerformanceConfig, - CaseType.Performance500K: WeaviatePerformanceConfig, - CaseType.Performance5M1P: WeaviatePerformanceConfig, - CaseType.Performance500K1P: WeaviatePerformanceConfig, - CaseType.Performance5M99P: WeaviatePerformanceConfig, - CaseType.Performance500K99P: WeaviatePerformanceConfig, + CaseType.Performance768D100M: WeaviatePerformanceConfig, + CaseType.Performance768D10M: WeaviatePerformanceConfig, + CaseType.Performance768D1M: WeaviatePerformanceConfig, + CaseType.Performance768D10M1P: WeaviatePerformanceConfig, + CaseType.Performance768D1M1P: WeaviatePerformanceConfig, + CaseType.Performance768D10M99P: WeaviatePerformanceConfig, + CaseType.Performance768D1M99P: WeaviatePerformanceConfig, + CaseType.Performance1536D5M: WeaviatePerformanceConfig, + CaseType.Performance1536D500K: WeaviatePerformanceConfig, + CaseType.Performance1536D5M1P: WeaviatePerformanceConfig, + CaseType.Performance1536D500K1P: WeaviatePerformanceConfig, + CaseType.Performance1536D5M99P: WeaviatePerformanceConfig, + CaseType.Performance1536D500K99P: WeaviatePerformanceConfig, }, DB.ElasticCloud: { CaseType.CapacityDim960: ESLoadingConfig, CaseType.CapacityDim128: ESLoadingConfig, - CaseType.Performance100M: ESPerformanceConfig, - CaseType.Performance10M: ESPerformanceConfig, - CaseType.Performance1M: ESPerformanceConfig, - CaseType.Performance10M1P: ESPerformanceConfig, - CaseType.Performance1M1P: ESPerformanceConfig, - CaseType.Performance10M99P: ESPerformanceConfig, - CaseType.Performance1M99P: ESPerformanceConfig, - CaseType.Performance5M: ESPerformanceConfig, - CaseType.Performance500K: ESPerformanceConfig, - CaseType.Performance5M1P: ESPerformanceConfig, - CaseType.Performance500K1P: ESPerformanceConfig, - CaseType.Performance5M99P: ESPerformanceConfig, - CaseType.Performance500K99P: ESPerformanceConfig, + CaseType.Performance768D100M: ESPerformanceConfig, + CaseType.Performance768D10M: ESPerformanceConfig, + CaseType.Performance768D1M: ESPerformanceConfig, + CaseType.Performance768D10M1P: ESPerformanceConfig, + CaseType.Performance768D1M1P: ESPerformanceConfig, + CaseType.Performance768D10M99P: ESPerformanceConfig, + CaseType.Performance768D1M99P: ESPerformanceConfig, + CaseType.Performance1536D5M: ESPerformanceConfig, + CaseType.Performance1536D500K: ESPerformanceConfig, + CaseType.Performance1536D5M1P: ESPerformanceConfig, + CaseType.Performance1536D500K1P: ESPerformanceConfig, + CaseType.Performance1536D5M99P: ESPerformanceConfig, + CaseType.Performance1536D500K99P: ESPerformanceConfig, }, DB.PgVector: { CaseType.CapacityDim960: PgVectorLoadingConfig, CaseType.CapacityDim128: PgVectorLoadingConfig, - CaseType.Performance100M: PgVectorPerformanceConfig, - CaseType.Performance10M: PgVectorPerformanceConfig, - CaseType.Performance1M: PgVectorPerformanceConfig, - CaseType.Performance10M1P: PgVectorPerformanceConfig, - CaseType.Performance1M1P: PgVectorPerformanceConfig, - CaseType.Performance10M99P: PgVectorPerformanceConfig, - CaseType.Performance1M99P: PgVectorPerformanceConfig, - CaseType.Performance5M: PgVectorPerformanceConfig, - CaseType.Performance500K: PgVectorPerformanceConfig, - CaseType.Performance5M1P: PgVectorPerformanceConfig, - CaseType.Performance500K1P: PgVectorPerformanceConfig, - CaseType.Performance5M99P: PgVectorPerformanceConfig, - CaseType.Performance500K99P: PgVectorPerformanceConfig, + CaseType.Performance768D100M: PgVectorPerformanceConfig, + CaseType.Performance768D10M: PgVectorPerformanceConfig, + CaseType.Performance768D1M: PgVectorPerformanceConfig, + CaseType.Performance768D10M1P: PgVectorPerformanceConfig, + CaseType.Performance768D1M1P: PgVectorPerformanceConfig, + CaseType.Performance768D10M99P: PgVectorPerformanceConfig, + CaseType.Performance768D1M99P: PgVectorPerformanceConfig, + CaseType.Performance1536D5M: PgVectorPerformanceConfig, + CaseType.Performance1536D500K: PgVectorPerformanceConfig, + CaseType.Performance1536D5M1P: PgVectorPerformanceConfig, + CaseType.Performance1536D500K1P: PgVectorPerformanceConfig, + CaseType.Performance1536D5M99P: PgVectorPerformanceConfig, + CaseType.Performance1536D500K99P: PgVectorPerformanceConfig, }, } From 5cff0d793ea0c762b3bd71a91151dbbbc2011e8e Mon Sep 17 00:00:00 2001 From: Li Liu Date: Mon, 14 Aug 2023 11:40:05 +0800 Subject: [PATCH 012/327] Fix variables' name to fit different dim Signed-off-by: Li Liu --- vectordb_bench/__init__.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/vectordb_bench/__init__.py b/vectordb_bench/__init__.py index 25febc22d..4bc5488a5 100644 --- a/vectordb_bench/__init__.py +++ b/vectordb_bench/__init__.py @@ -19,18 +19,23 @@ class config: RESULTS_LOCAL_DIR = pathlib.Path(__file__).parent.joinpath("results") - CAPACITY_TIMEOUT_IN_SECONDS = 24 * 3600 # 24h + CAPACITY_TIMEOUT_IN_SECONDS = 24 * 3600 # 24h LOAD_TIMEOUT_DEFAULT = 2.5 * 3600 # 2.5h - LOAD_TIMEOUT_1M = 2.5 * 3600 # 2.5h - LOAD_TIMEOUT_10M = 25 * 3600 # 25h - LOAD_TIMEOUT_100M = 250 * 3600 # 10.41d + LOAD_TIMEOUT_768D_1M = 2.5 * 3600 # 2.5h + LOAD_TIMEOUT_768D_10M = 25 * 3600 # 25h + LOAD_TIMEOUT_768D_100M = 250 * 3600 # 10.41d + + LOAD_TIMEOUT_1536D_500K = 2.5 * 3600 # 2.5h + LOAD_TIMEOUT_1536D_5M = 25 * 3600 # 25h OPTIMIZE_TIMEOUT_DEFAULT = 15 * 60 # 15min - OPTIMIZE_TIMEOUT_1M = 15 * 60 # 15min - OPTIMIZE_TIMEOUT_10M = 2.5 * 3600 # 2.5h - OPTIMIZE_TIMEOUT_100M = 25 * 3600 # 1.04d + OPTIMIZE_TIMEOUT_768D_1M = 15 * 60 # 15min + OPTIMIZE_TIMEOUT_768D_10M = 2.5 * 3600 # 2.5h + OPTIMIZE_TIMEOUT_768D_100M = 25 * 3600 # 1.04d + OPTIMIZE_TIMEOUT_1536D_500K = 15 * 60 # 15min + OPTIMIZE_TIMEOUT_1536D_5M = 2.5 * 3600 # 2.5h def display(self) -> str: tmp = [ i for i in inspect.getmembers(self) From be9ed8290b5cdc62659b5da7a9941e120ef2a262 Mon Sep 17 00:00:00 2001 From: Li Liu Date: Mon, 14 Aug 2023 13:18:29 +0800 Subject: [PATCH 013/327] Update ReadMe for 1536 support Signed-off-by: Li Liu --- README.md | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index d3adb77e0..ef33cb8d8 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ VectorDBBench is not just an offering of benchmark results for mainstream vector Understanding the importance of user experience, we provide an intuitive visual interface. This not only empowers users to initiate benchmarks at ease, but also to view comparative result reports, thereby reproducing benchmark results effortlessly. To add more relevance and practicality, we provide cost-effectiveness reports particularly for cloud services. This allows for a more realistic and applicable benchmarking process. -Closely mimicking real-world production environments, we've set up diverse testing scenarios including insertion, searching, and filtered searching. To provide you with credible and reliable data, we've included public datasets from actual production scenarios, such as [SIFT](http://corpus-texmex.irisa.fr/), [GIST](http://corpus-texmex.irisa.fr/), [Cohere](https://huggingface.co/datasets/Cohere/wikipedia-22-12/tree/main/en), and more. It's fascinating to discover how a relatively unknown open-source database might excel in certain circumstances! +Closely mimicking real-world production environments, we've set up diverse testing scenarios including insertion, searching, and filtered searching. To provide you with credible and reliable data, we've included public datasets from actual production scenarios, such as [SIFT](http://corpus-texmex.irisa.fr/), [GIST](http://corpus-texmex.irisa.fr/), [Cohere](https://huggingface.co/datasets/Cohere/wikipedia-22-12/tree/main/en), and a dataset generated by OpenAI from an opensource [raw dataset](https://huggingface.co/datasets/allenai/c4). It's fascinating to discover how a relatively unknown open-source database might excel in certain circumstances! Prepare to delve into the world of VectorDBBench, and let it guide you in uncovering your perfect vector database match. @@ -77,7 +77,7 @@ $ ruff check vectordb_bench --fix ![image](https://github.com/zilliztech/VectorDBBench/assets/105927039/7f5cdae7-f9f2-4a81-b2e0-e5c6268cd970) This is the main page of VectorDBBench, which displays the standard benchmark results we provide. Additionally, results of all tests performed by users themselves will also be shown here. We also offer the ability to select and compare results from multiple tests simultaneously. -The standard benchmark results displayed here include all 9 cases that we currently support for all our clients (Milvus, Zilliz Cloud, Elastic Search, Qdrant Cloud, and Weaviate Cloud). However, as some systems may not be able to complete all the tests successfully due to issues like Out of Memory (OOM) or timeouts, not all clients are included in every case. +The standard benchmark results displayed here include all 15 cases that we currently support for 6 of our clients (Milvus, Zilliz Cloud, Elastic Search, Qdrant Cloud, Weaviate Cloud and PgVector). However, as some systems may not be able to complete all the tests successfully due to issues like Out of Memory (OOM) or timeouts, not all clients are included in every case. All standard benchmark results are generated by a client running on an 8 core, 32 GB host, which is located in the same region as the server being tested. The client host is equipped with an `Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz` processor. Also all the servers for the open-source systems tested in our benchmarks run on hosts with the same type of processor. ### Run Test Page @@ -92,27 +92,27 @@ Now we can only run one task at the same time. ### Code Structure ![image](https://github.com/zilliztech/VectorDBBench/assets/105927039/8c06512e-5419-4381-b084-9c93aed59639) ### Client -Our client module is designed with flexibility and extensibility in mind, aiming to integrate APIs from different systems seamlessly. As of now, it supports Milvus, Zilliz Cloud, Elastic Search, Pinecone, Qdrant, and Weaviate. Stay tuned for more options, as we are consistently working on extending our reach to other systems. +Our client module is designed with flexibility and extensibility in mind, aiming to integrate APIs from different systems seamlessly. As of now, it supports Milvus, Zilliz Cloud, Elastic Search, Pinecone, Qdrant Cloud, Weaviate Cloud, PgVector, Redis, and Chroma. Stay tuned for more options, as we are consistently working on extending our reach to other systems. ### Benchmark Cases -We've developed an array of 9 comprehensive benchmark cases to test vector databases' various capabilities, each designed to give you a different piece of the puzzle. These cases are categorized into three main types: +We've developed an array of 15 comprehensive benchmark cases to test vector databases' various capabilities, each designed to give you a different piece of the puzzle. These cases are categorized into three main types: #### Capacity Case - **Large Dim:** Tests the database's loading capacity by inserting large-dimension vectors (GIST 100K vectors, 960 dimensions) until fully loaded. The final number of inserted vectors is reported. - **Small Dim:** Similar to the Large Dim case but uses small-dimension vectors (SIFT 100K vectors, 128 dimensions). #### Search Performance Case - **XLarge Dataset:** Measures search performance with a massive dataset (LAION 100M vectors, 768 dimensions) at varying parallel levels. The results include index building time, recall, latency, and maximum QPS. -- **Large Dataset:** Similar to the XLarge Dataset case, but uses a slightly smaller dataset (Cohere 10M vectors, 768 dimensions). -- **Medium Dataset:** A case using a medium dataset (Cohere 1M vectors, 768 dimensions). +- **Large Dataset:** Similar to the XLarge Dataset case, but uses a slightly smaller dataset (10M-768dim, 5M-1536dim). +- **Medium Dataset:** A case using a medium dataset (1M-768dim, 500K-1536dim). #### Filtering Search Performance Case -- **Large Dataset, Low Filtering Rate:** Evaluates search performance with a large dataset (Cohere 10M vectors, 768 dimensions) under a low filtering rate (1% vectors) at different parallel levels. -- **Medium Dataset, Low Filtering Rate:** This case uses a medium dataset (Cohere 1M vectors, 768 dimensions) with a similar low filtering rate. -- **Large Dataset, High Filtering Rate:** It tests with a large dataset (Cohere 10M vectors, 768 dimensions) but under a high filtering rate (99% vectors). -- **Medium Dataset, High Filtering Rate:** This case uses a medium dataset (Cohere 1M vectors, 768 dimensions) with a high filtering rate. +- **Large Dataset, Low Filtering Rate:** Evaluates search performance with a large dataset (10M-768dim, 5M-1536dim) under a low filtering rate (1% vectors) at different parallel levels. +- **Medium Dataset, Low Filtering Rate:** This case uses a medium dataset (1M-768dim, 500K-1536dim) with a similar low filtering rate. +- **Large Dataset, High Filtering Rate:** It tests with a large dataset (10M-768dim, 5M-1536dim) but under a high filtering rate (99% vectors). +- **Medium Dataset, High Filtering Rate:** This case uses a medium dataset (1M-768dim, 500K-1536dim) with a high filtering rate. For a quick reference, here is a table summarizing the key aspects of each case: Case No. | Case Type | Dataset Size | Filtering Rate | Results | |----------|-----------|--------------|----------------|---------| -1 | Capacity Case | GIST 100K vectors, 960 dimensions | N/A | Number of inserted vectors | -2 | Capacity Case | SIFT 100K vectors, 128 dimensions | N/A | Number of inserted vectors | +1 | Capacity Case | SIFT 100K vectors, 128 dimensions | N/A | Number of inserted vectors | +2 | Capacity Case | GIST 100K vectors, 960 dimensions | N/A | Number of inserted vectors | 3 | Search Performance Case | LAION 100M vectors, 768 dimensions | N/A | Index building time, recall, latency, maximum QPS | 4 | Search Performance Case | Cohere 10M vectors, 768 dimensions | N/A | Index building time, recall, latency, maximum QPS | 5 | Search Performance Case | Cohere 1M vectors, 768 dimensions | N/A | Index building time, recall, latency, maximum QPS | @@ -120,6 +120,13 @@ Case No. | Case Type | Dataset Size | Filtering Rate | Results | 7 | Filtering Search Performance Case | Cohere 1M vectors, 768 dimensions | 1% vectors | Index building time, recall, latency, maximum QPS | 8 | Filtering Search Performance Case | Cohere 10M vectors, 768 dimensions | 99% vectors | Index building time, recall, latency, maximum QPS | 9 | Filtering Search Performance Case | Cohere 1M vectors, 768 dimensions | 99% vectors | Index building time, recall, latency, maximum QPS | +10 | Search Performance Case | OpenAI generated 500K vectors, 1536 dimensions | N/A | Index building time, recall, latency, maximum QPS | +11 | Search Performance Case | OpenAI generated 5M vectors, 1536 dimensions | N/A | Index building time, recall, latency, maximum QPS | +12 | Filtering Search Performance Case | OpenAI generated 500K vectors, 1536 dimensions | 1% vectors | Index building time, recall, latency, maximum QPS | +13 | Filtering Search Performance Case | OpenAI generated 5M vectors, 1536 dimensions | 1% vectors | Index building time, recall, latency, maximum QPS | +14 | Filtering Search Performance Case | OpenAI generated 500K vectors, 1536 dimensions | 99% vectors | Index building time, recall, latency, maximum QPS | +15 | Filtering Search Performance Case | OpenAI generated 5M vectors, 1536 dimensions | 99% vectors | Index building time, recall, latency, maximum QPS | + Each case provides an in-depth examination of a vector database's abilities, providing you a comprehensive view of the database's performance. @@ -237,7 +244,7 @@ This multi-tiered timeout approach allows our benchmark to be more representativ Other Cases - 1M vectors, 768 dimensions + 1M vectors, 768 dimensions
500K vectors, 1536 dimensions Loading timeout 2.5 hours @@ -247,7 +254,7 @@ This multi-tiered timeout approach allows our benchmark to be more representativ Other Cases - 10M vectors, 768 dimensions + 10M vectors, 768 dimensions
5M vectors, 1536 dimensions Loading timeout 25 hours From ec3d494627fe2f7b08cf911470957cd7c4a36dbd Mon Sep 17 00:00:00 2001 From: liliu-z <105927039+liliu-z@users.noreply.github.com> Date: Mon, 14 Aug 2023 13:24:42 +0800 Subject: [PATCH 014/327] Update README.md (#208) * Update README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index ef33cb8d8..cfaa86215 100644 --- a/README.md +++ b/README.md @@ -74,14 +74,14 @@ $ ruff check vectordb_bench --fix ## How does it work? ### Result Page -![image](https://github.com/zilliztech/VectorDBBench/assets/105927039/7f5cdae7-f9f2-4a81-b2e0-e5c6268cd970) +![image](https://github.com/zilliztech/VectorDBBench/assets/105927039/8a981327-c1c6-4796-8a85-c86154cb5472) This is the main page of VectorDBBench, which displays the standard benchmark results we provide. Additionally, results of all tests performed by users themselves will also be shown here. We also offer the ability to select and compare results from multiple tests simultaneously. The standard benchmark results displayed here include all 15 cases that we currently support for 6 of our clients (Milvus, Zilliz Cloud, Elastic Search, Qdrant Cloud, Weaviate Cloud and PgVector). However, as some systems may not be able to complete all the tests successfully due to issues like Out of Memory (OOM) or timeouts, not all clients are included in every case. All standard benchmark results are generated by a client running on an 8 core, 32 GB host, which is located in the same region as the server being tested. The client host is equipped with an `Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz` processor. Also all the servers for the open-source systems tested in our benchmarks run on hosts with the same type of processor. ### Run Test Page -![image](https://github.com/zilliztech/VectorDBBench/assets/105927039/a789099a-3707-4214-8052-b73463b8f2c6) +![image](https://github.com/zilliztech/VectorDBBench/assets/105927039/f3135a29-8f12-4aac-bbb3-f2f55e2a2ff0) This is the page to run a test: 1. Initially, you select the systems to be tested - multiple selections are allowed. Once selected, corresponding forms will pop up to gather necessary information for using the chosen databases. The db_label is used to differentiate different instances of the same system. We recommend filling in the host size or instance type here (as we do in our standard results). 2. The next step is to select the test cases you want to perform. You can select multiple cases at once, and a form to collect corresponding parameters will appear. @@ -274,4 +274,4 @@ This multi-tiered timeout approach allows our benchmark to be more representativ -**Note:** Some datapoints in the standard benchmark results that voilate this timeout will be kept for now for reference. We will remove them in the future. \ No newline at end of file +**Note:** Some datapoints in the standard benchmark results that voilate this timeout will be kept for now for reference. We will remove them in the future. From c8b4e156d075480da7c2ba0250233ee6340f355d Mon Sep 17 00:00:00 2001 From: "min.tian" Date: Mon, 14 Aug 2023 18:05:12 +0800 Subject: [PATCH 015/327] update leaderboard data Signed-off-by: min.tian --- vectordb_bench/results/leaderboard.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vectordb_bench/results/leaderboard.json b/vectordb_bench/results/leaderboard.json index 9bbbacc0c..f86dd8724 100644 --- a/vectordb_bench/results/leaderboard.json +++ b/vectordb_bench/results/leaderboard.json @@ -1 +1 @@ -[{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":15.2269,"latency":861.8,"recall":0.9888,"label":1,"qp$":31.76903818068016},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":15.1749,"latency":774.3,"recall":0.989,"label":1,"qp$":31.660546630502814},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":27.6181,"latency":305.5,"recall":0.9999,"label":1,"qp$":57.621740037554765},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":100.6667,"latency":21.1,"recall":0.9909,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":101.1399,"latency":19.7,"recall":0.9907,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":52.2606,"latency":18.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":61.0661,"latency":49.8,"recall":0.9911,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":58.9326,"latency":44.6,"recall":0.9911,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":42.5977,"latency":54.9,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":536.0726,"latency":8.200000000000001,"recall":0.9728,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":467.179,"latency":7.0,"recall":0.9697,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":431.7512,"latency":8.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":330.0144,"latency":9.0,"recall":0.9507,"label":1,"qp$":2075.5622641509435},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":271.6585,"latency":10.1,"recall":0.9678,"label":1,"qp$":1708.5440251572327},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":216.5226,"latency":12.9,"recall":1.0,"label":1,"qp$":1361.777358490566},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":123.9553,"latency":23.0,"recall":0.971,"label":1,"qp$":389.79654088050313},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":59.1479,"latency":44.5,"recall":0.9906,"label":1,"qp$":185.99968553459118},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":40.999,"latency":55.300000000000004,"recall":1.0,"label":1,"qp$":128.92767295597486},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":579.9416,"latency":9.4,"recall":0.9213,"label":1,"qp$":1823.71572327044},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":425.2529,"latency":11.299999999999999,"recall":0.9686,"label":1,"qp$":1337.2732704402515},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":397.0539,"latency":13.799999999999999,"recall":1.0,"label":1,"qp$":1248.5971698113208},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":67.9121,"latency":179.5,"recall":0.9909,"label":1,"qp$":2.083193251533742},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.7636,"latency":1921.3,"recall":0.9908,"label":1,"qp$":0.02342331288343558},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":32.0,"latency":124.5,"recall":1.0,"label":1,"qp$":0.9815950920245399},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":63.1365,"latency":145.7,"recall":0.991,"label":1,"qp$":6.251138613861386},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.7512,"latency":1983.8,"recall":0.9908,"label":1,"qp$":0.07437623762376237},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":30.1358,"latency":129.8,"recall":1.0,"label":1,"qp$":2.983742574257426},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":516.27,"latency":7.0,"recall":0.9463,"label":1,"qp$":3246.9811320754716},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":354.8416,"latency":10.0,"recall":0.9802,"label":1,"qp$":2231.708176100629},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":427.5229,"latency":8.7,"recall":1.0,"label":1,"qp$":2688.8232704402517},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":2884.689,"latency":5.3,"recall":0.8801,"label":1,"qp$":2267.837264150943},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":1689.5799,"latency":6.6,"recall":0.9493,"label":1,"qp$":1328.2860849056603},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1517.6792,"latency":10.0,"recall":1.0,"label":1,"qp$":1193.1440251572328},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":822.5318,"latency":5.6,"recall":0.9294,"label":1,"qp$":646.6444968553459},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":378.9146,"latency":10.3,"recall":0.9758,"label":1,"qp$":297.8888364779874},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":218.6854,"latency":16.2,"recall":1.0,"label":1,"qp$":171.92248427672953},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":274.5407,"latency":4.8999999999999995,"recall":0.9807,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":236.5672,"latency":10.3,"recall":0.981,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":309.4833,"latency":4.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":178.6585,"latency":13.700000000000001,"recall":0.9843,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":178.3732,"latency":15.0,"recall":0.9844,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":229.3526,"latency":12.5,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":1258.7043,"latency":4.8999999999999995,"recall":0.9799,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":1075.8776,"latency":5.3,"recall":0.98,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1494.8493,"latency":4.7,"recall":1.0,"label":1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":537.4975,"latency":18.9,"recall":0.8903,"label":1,"qp$":376.9267180925666},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":372.0466,"latency":17.8,"recall":0.8904,"label":1,"qp$":260.9022440392707},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1156.2898,"latency":14.4,"recall":0.9989,"label":1,"qp$":810.8624123422161},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":110.248,"latency":69.0,"recall":0.898,"label":1,"qp$":77.31276297335204},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":87.2601,"latency":27.799999999999997,"recall":0.898,"label":1,"qp$":61.1922159887798},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":125.7846,"latency":23.099999999999998,"recall":0.975,"label":1,"qp$":88.20799438990183},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":240.7209,"latency":17.4,"recall":0.8887,"label":1,"qp$":844.0424263674614},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":189.4399,"latency":17.5,"recall":0.8889,"label":1,"qp$":664.2352734922861},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":313.5116,"latency":16.1,"recall":0.9999,"label":1,"qp$":1099.2692847124824},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":18.7634,"latency":153.70000000000002,"recall":0.8737,"label":1,"qp$":192.84069886947586},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":18.3619,"latency":79.8,"recall":0.8741,"label":1,"qp$":188.7142857142857},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":25.2744,"latency":61.199999999999996,"recall":0.9979,"label":1,"qp$":259.7574511819116},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":261.798,"latency":23.099999999999998,"recall":0.9262,"label":1,"qp$":1793.13698630137},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":166.1851,"latency":23.900000000000002,"recall":0.9264,"label":1,"qp$":1138.2541095890413},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":121.7169,"latency":29.0,"recall":0.9693,"label":1,"qp$":833.677397260274},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":46.6189,"latency":43.1,"recall":0.8737,"label":1,"qp$":479.1253854059609},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":42.4856,"latency":44.0,"recall":0.8741,"label":1,"qp$":436.6454265159301},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":138.9479,"latency":26.200000000000003,"recall":0.9979,"label":1,"qp$":1428.0359712230218},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":20.7437,"latency":75.80000000000001,"recall":0.9291,"label":1,"qp$":106.37794871794871},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":20.2993,"latency":76.5,"recall":0.9293,"label":1,"qp$":104.09897435897435},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":26.4719,"latency":67.0,"recall":1.0,"label":1,"qp$":135.75333333333333},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":8.6675,"latency":180.2,"recall":0.8369,"label":1,"qp$":44.44871794871795},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":7.8121,"latency":167.7,"recall":0.8369,"label":1,"qp$":40.06205128205128},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":16.869,"latency":87.8,"recall":0.9814,"label":1,"qp$":86.50769230769231},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":365.0835,"latency":23.599999999999998,"recall":0.945,"label":1,"qp$":312.57148972602744},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":325.5271,"latency":25.1,"recall":0.9452,"label":1,"qp$":278.7047089041096},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":596.7942,"latency":24.2,"recall":0.9693,"label":1,"qp$":510.95393835616443},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":303.2044,"latency":27.400000000000002,"recall":0.9246,"label":1,"qp$":259.5928082191781},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":136.0345,"latency":31.9,"recall":0.9244,"label":1,"qp$":116.46789383561645},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":66.7221,"latency":42.1,"recall":0.963,"label":1,"qp$":57.125085616438355},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":131.2549,"latency":30.200000000000003,"recall":0.9867,"label":1,"qp$":168.49152759948652},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":127.9337,"latency":30.099999999999998,"recall":0.9869,"label":1,"qp$":164.2281129653402},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":595.8462,"latency":23.400000000000002,"recall":1.0,"label":1,"qp$":764.8860077021822},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-1node","db_name":"PgVector-2c8g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":10.6271,"latency":730.7,"recall":0.8898,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-1node","db_name":"PgVector-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":10.8507,"latency":733.1999999999999,"recall":0.8897,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-1node","db_name":"PgVector-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":75.7055,"latency":121.2,"recall":0.9999,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-1node","db_name":"PgVector-2c8g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-1node","db_name":"PgVector-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-1node","db_name":"PgVector-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0}] \ No newline at end of file +[{"db":"PgVector","db_label":"2c8g-60gdisk","db_name":"PgVector-2c8g-60gdisk","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":0.8836,"latency":2523.0,"recall":0.8528,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-60gdisk","db_name":"PgVector-2c8g-60gdisk","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-60gdisk","db_name":"PgVector-2c8g-60gdisk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":0.8937,"latency":3720.2000000000003,"recall":0.8525,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-60gdisk","db_name":"PgVector-2c8g-60gdisk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-60gdisk","db_name":"PgVector-2c8g-60gdisk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":1.2145,"latency":3622.3999999999996,"recall":0.7487,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-60gdisk","db_name":"PgVector-2c8g-60gdisk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-1node","db_name":"PgVector-2c8g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":10.6271,"latency":730.7,"recall":0.8898,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-1node","db_name":"PgVector-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":10.8507,"latency":733.1999999999999,"recall":0.8897,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-1node","db_name":"PgVector-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":75.7055,"latency":121.2,"recall":0.9999,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-1node","db_name":"PgVector-2c8g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-1node","db_name":"PgVector-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-1node","db_name":"PgVector-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":11.2945,"latency":3611.2000000000003,"recall":0.996,"label":1,"qp$":23.564573336115167},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":17.3271,"latency":3774.7999999999997,"recall":0.9961,"label":1,"qp$":36.15084498226581},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":26.26,"latency":556.1,"recall":0.9999,"label":1,"qp$":54.78823283955769},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":15.2269,"latency":861.8,"recall":0.9888,"label":1,"qp$":31.76903818068016},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":15.1749,"latency":774.3,"recall":0.989,"label":1,"qp$":31.660546630502814},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":27.6181,"latency":305.5,"recall":0.9999,"label":1,"qp$":57.621740037554765},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":228.4,"latency":22.2,"recall":0.9348,"label":1,"qp$":1564.3835616438357},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":181.5,"latency":26.1,"recall":0.9345,"label":1,"qp$":1243.150684931507},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":205.7,"latency":24.2,"recall":0.9586,"label":1,"qp$":1408.9041095890411},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":67.63,"latency":36.0,"recall":0.8064,"label":1,"qp$":695.0668036998972},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":63.35,"latency":38.4,"recall":0.8065,"label":1,"qp$":651.0791366906475},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":176.7,"latency":27.599999999999998,"recall":1.0,"label":1,"qp$":1816.032887975334},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":15.33,"latency":84.9,"recall":0.8064,"label":1,"qp$":157.55395683453239},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":15.13,"latency":86.7,"recall":0.8065,"label":1,"qp$":155.49845837615624},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":17.41,"latency":74.30000000000001,"recall":1.0,"label":1,"qp$":178.9311408016444},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":16.34,"latency":88.7,"recall":0.879,"label":1,"qp$":83.7948717948718},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":10.45,"latency":126.8,"recall":0.8208,"label":1,"qp$":53.589743589743584},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":16.18,"latency":87.5,"recall":0.8793,"label":1,"qp$":82.97435897435896},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":9.8,"latency":130.89999999999998,"recall":0.8212,"label":1,"qp$":50.256410256410255},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":36.11,"latency":55.1,"recall":1.0,"label":1,"qp$":185.17948717948718},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":14.84,"latency":92.7,"recall":0.96,"label":1,"qp$":76.1025641025641},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":322.7,"latency":26.4,"recall":0.9478,"label":1,"qp$":276.2842465753425},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":265.5,"latency":26.9,"recall":0.9332,"label":1,"qp$":227.31164383561645},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":303.8,"latency":27.3,"recall":0.9478,"label":1,"qp$":260.10273972602744},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":180.2,"latency":28.2,"recall":0.9335,"label":1,"qp$":154.28082191780823},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":730.7,"latency":24.6,"recall":0.9586,"label":1,"qp$":625.5993150684932},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":104.3,"latency":31.7,"recall":0.9563,"label":1,"qp$":89.29794520547945},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":147.7,"latency":35.3,"recall":0.9707,"label":1,"qp$":189.60205391527597},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":782.5,"latency":25.9,"recall":1.0,"label":1,"qp$":1004.4929396662387},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":18.7634,"latency":153.70000000000002,"recall":0.8737,"label":1,"qp$":192.84069886947586},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":18.3619,"latency":79.8,"recall":0.8741,"label":1,"qp$":188.7142857142857},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":25.2744,"latency":61.199999999999996,"recall":0.9979,"label":1,"qp$":259.7574511819116},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":261.798,"latency":23.099999999999998,"recall":0.9262,"label":1,"qp$":1793.13698630137},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":166.1851,"latency":23.900000000000002,"recall":0.9264,"label":1,"qp$":1138.2541095890413},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":121.7169,"latency":29.0,"recall":0.9693,"label":1,"qp$":833.677397260274},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":46.6189,"latency":43.1,"recall":0.8737,"label":1,"qp$":479.1253854059609},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":42.4856,"latency":44.0,"recall":0.8741,"label":1,"qp$":436.6454265159301},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":138.9479,"latency":26.200000000000003,"recall":0.9979,"label":1,"qp$":1428.0359712230218},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":20.7437,"latency":75.80000000000001,"recall":0.9291,"label":1,"qp$":106.37794871794871},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":20.2993,"latency":76.5,"recall":0.9293,"label":1,"qp$":104.09897435897435},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":26.4719,"latency":67.0,"recall":1.0,"label":1,"qp$":135.75333333333333},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":8.6675,"latency":180.2,"recall":0.8369,"label":1,"qp$":44.44871794871795},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":7.8121,"latency":167.7,"recall":0.8369,"label":1,"qp$":40.06205128205128},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":16.869,"latency":87.8,"recall":0.9814,"label":1,"qp$":86.50769230769231},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":365.0835,"latency":23.599999999999998,"recall":0.945,"label":1,"qp$":312.57148972602744},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":325.5271,"latency":25.1,"recall":0.9452,"label":1,"qp$":278.7047089041096},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":596.7942,"latency":24.2,"recall":0.9693,"label":1,"qp$":510.95393835616443},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":303.2044,"latency":27.400000000000002,"recall":0.9246,"label":1,"qp$":259.5928082191781},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":136.0345,"latency":31.9,"recall":0.9244,"label":1,"qp$":116.46789383561645},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":66.7221,"latency":42.1,"recall":0.963,"label":1,"qp$":57.125085616438355},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":131.2549,"latency":30.200000000000003,"recall":0.9867,"label":1,"qp$":168.49152759948652},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":127.9337,"latency":30.099999999999998,"recall":0.9869,"label":1,"qp$":164.2281129653402},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":595.8462,"latency":23.400000000000002,"recall":1.0,"label":1,"qp$":764.8860077021822},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":330.0144,"latency":9.0,"recall":0.9507,"label":1,"qp$":2075.5622641509435},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":271.6585,"latency":10.1,"recall":0.9678,"label":1,"qp$":1708.5440251572327},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":216.5226,"latency":12.9,"recall":1.0,"label":1,"qp$":1361.777358490566},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":123.9553,"latency":23.0,"recall":0.971,"label":1,"qp$":389.79654088050313},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":59.1479,"latency":44.5,"recall":0.9906,"label":1,"qp$":185.99968553459118},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":40.999,"latency":55.300000000000004,"recall":1.0,"label":1,"qp$":128.92767295597486},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":579.9416,"latency":9.4,"recall":0.9213,"label":1,"qp$":1823.71572327044},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":425.2529,"latency":11.299999999999999,"recall":0.9686,"label":1,"qp$":1337.2732704402515},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":397.0539,"latency":13.799999999999999,"recall":1.0,"label":1,"qp$":1248.5971698113208},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":516.27,"latency":7.0,"recall":0.9463,"label":1,"qp$":3246.9811320754716},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":354.8416,"latency":10.0,"recall":0.9802,"label":1,"qp$":2231.708176100629},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":427.5229,"latency":8.7,"recall":1.0,"label":1,"qp$":2688.8232704402517},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":2884.689,"latency":5.3,"recall":0.8801,"label":1,"qp$":2267.837264150943},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":1689.5799,"latency":6.6,"recall":0.9493,"label":1,"qp$":1328.2860849056603},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1517.6792,"latency":10.0,"recall":1.0,"label":1,"qp$":1193.1440251572328},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":822.5318,"latency":5.6,"recall":0.9294,"label":1,"qp$":646.6444968553459},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":378.9146,"latency":10.3,"recall":0.9758,"label":1,"qp$":297.8888364779874},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":218.6854,"latency":16.2,"recall":1.0,"label":1,"qp$":171.92248427672953},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":297.5,"latency":7.2,"recall":0.974,"label":1,"qp$":1871.0691823899372},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":228.3,"latency":10.6,"recall":0.994,"label":1,"qp$":1435.8490566037735},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":584.0,"latency":4.6,"recall":1.0,"label":1,"qp$":3672.9559748427673},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":1871.0,"latency":7.0,"recall":0.9602,"label":1,"qp$":1470.9119496855346},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":556.7,"latency":6.7,"recall":0.9723,"label":1,"qp$":437.65723270440253},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1583.0,"latency":6.8,"recall":0.9836,"label":1,"qp$":1244.4968553459119},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":294.3,"latency":10.9,"recall":0.9939,"label":1,"qp$":231.3679245283019},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":2345.0,"latency":8.9,"recall":1.0,"label":1,"qp$":1843.553459119497},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":295.6,"latency":12.3,"recall":1.0,"label":1,"qp$":232.38993710691824},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":143.0,"latency":33.5,"recall":0.9818,"label":1,"qp$":899.3710691823899},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":106.0,"latency":20.7,"recall":0.9887,"label":1,"qp$":666.6666666666666},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":189.0,"latency":11.6,"recall":1.0,"label":1,"qp$":1188.6792452830189},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":71.74,"latency":50.8,"recall":0.9883,"label":1,"qp$":225.59748427672955},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":287.0,"latency":14.9,"recall":0.9865,"label":1,"qp$":902.5157232704403},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":34.6654,"latency":64.69999999999999,"recall":0.9961,"label":1,"qp$":109.01069182389936},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":412.0,"latency":10.3,"recall":1.0,"label":1,"qp$":1295.5974842767296},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":42.169,"latency":46.800000000000004,"recall":1.0,"label":1,"qp$":132.6069182389937},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":537.4975,"latency":18.9,"recall":0.8903,"label":1,"qp$":376.9267180925666},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":372.0466,"latency":17.8,"recall":0.8904,"label":1,"qp$":260.9022440392707},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1156.2898,"latency":14.4,"recall":0.9989,"label":1,"qp$":810.8624123422161},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":110.248,"latency":69.0,"recall":0.898,"label":1,"qp$":77.31276297335204},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":87.2601,"latency":27.799999999999997,"recall":0.898,"label":1,"qp$":61.1922159887798},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":125.7846,"latency":23.099999999999998,"recall":0.975,"label":1,"qp$":88.20799438990183},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":240.7209,"latency":17.4,"recall":0.8887,"label":1,"qp$":844.0424263674614},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":189.4399,"latency":17.5,"recall":0.8889,"label":1,"qp$":664.2352734922861},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":313.5116,"latency":16.1,"recall":0.9999,"label":1,"qp$":1099.2692847124824},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":78.7196,"latency":49.4,"recall":0.9203,"label":1,"qp$":474.21445783132526},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":68.3111,"latency":35.5,"recall":0.9202,"label":1,"qp$":411.5126506024096},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":210.2147,"latency":26.8,"recall":0.9996,"label":1,"qp$":1266.3536144578313},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":188.6436,"latency":917.5,"recall":0.9175,"label":1,"qp$":661.4431977559607},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":155.6991,"latency":917.1,"recall":0.9171,"label":1,"qp$":545.929523141655},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":445.3289,"latency":14.1,"recall":0.9999,"label":1,"qp$":1561.461781206171},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":633.6033,"latency":24.6,"recall":0.919,"label":1,"qp$":444.32208976157085},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":95.5682,"latency":58.7,"recall":0.9463,"label":1,"qp$":67.01837307152876},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":434.4062,"latency":17.4,"recall":0.9181,"label":1,"qp$":304.6326788218794},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":80.3192,"latency":26.599999999999998,"recall":0.9462,"label":1,"qp$":56.324824684431974},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":1509.3293,"latency":18.5,"recall":0.9995,"label":1,"qp$":1058.4356942496495},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":166.7252,"latency":19.599999999999998,"recall":0.9988,"label":1,"qp$":116.91809256661992},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":46.8622,"latency":123.0,"recall":0.9957,"label":1,"qp$":4.639821782178218},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1.8388,"latency":1063.5,"recall":0.9957,"label":1,"qp$":0.18205940594059405},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":45.0666,"latency":109.2,"recall":1.0,"label":1,"qp$":4.462039603960396},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":43.5017,"latency":228.7,"recall":0.9957,"label":1,"qp$":1.3344079754601226},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1.5668,"latency":1114.4,"recall":0.9957,"label":1,"qp$":0.04806134969325153},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":41.5443,"latency":159.0,"recall":1.0,"label":1,"qp$":1.2743650306748466},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":67.9121,"latency":179.5,"recall":0.9909,"label":1,"qp$":2.083193251533742},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.7636,"latency":1921.3,"recall":0.9908,"label":1,"qp$":0.02342331288343558},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":32.0,"latency":124.5,"recall":1.0,"label":1,"qp$":0.9815950920245399},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":63.1365,"latency":145.7,"recall":0.991,"label":1,"qp$":6.251138613861386},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.7512,"latency":1983.8,"recall":0.9908,"label":1,"qp$":0.07437623762376237},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":30.1358,"latency":129.8,"recall":1.0,"label":1,"qp$":2.983742574257426},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":37.432,"latency":75.0,"recall":0.9975,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":37.0696,"latency":73.5,"recall":0.9976,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":81.1915,"latency":53.0,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":321.6048,"latency":13.4,"recall":0.989,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":22.1467,"latency":86.8,"recall":0.9972,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":303.2551,"latency":10.9,"recall":0.9876,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":21.5388,"latency":81.69999999999999,"recall":0.997,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":394.5418,"latency":8.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":37.878,"latency":45.0,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":180.2757,"latency":6.0,"recall":0.9942,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":179.0033,"latency":6.4,"recall":0.9943,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":526.8846,"latency":3.6,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":626.5243,"latency":6.2,"recall":0.9954,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":78.4227,"latency":25.3,"recall":0.9982,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":599.4213,"latency":6.6,"recall":0.9955,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":78.5351,"latency":26.3,"recall":0.9982,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":2098.2113,"latency":3.4,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":275.6292,"latency":10.0,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":100.6667,"latency":21.1,"recall":0.9909,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":101.1399,"latency":19.7,"recall":0.9907,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":52.2606,"latency":18.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":61.0661,"latency":49.8,"recall":0.9911,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":58.9326,"latency":44.6,"recall":0.9911,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":42.5977,"latency":54.9,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":536.0726,"latency":8.200000000000001,"recall":0.9728,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":467.179,"latency":7.0,"recall":0.9697,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":431.7512,"latency":8.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":274.5407,"latency":4.8999999999999995,"recall":0.9807,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":236.5672,"latency":10.3,"recall":0.981,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":309.4833,"latency":4.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":178.6585,"latency":13.700000000000001,"recall":0.9843,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":178.3732,"latency":15.0,"recall":0.9844,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":229.3526,"latency":12.5,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":1258.7043,"latency":4.8999999999999995,"recall":0.9799,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":1075.8776,"latency":5.3,"recall":0.98,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1494.8493,"latency":4.7,"recall":1.0,"label":1,"qp$":0.0}] \ No newline at end of file From 378116d99c797536b3e6426c5aaabcdce60a6521 Mon Sep 17 00:00:00 2001 From: Li Liu Date: Mon, 14 Aug 2023 20:08:00 +0800 Subject: [PATCH 016/327] Add lost datapoint Signed-off-by: Li Liu --- .../result_20230808_standard_zillizcloud.json | 10 +++++----- vectordb_bench/results/leaderboard.json | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/vectordb_bench/results/ZillizCloud/result_20230808_standard_zillizcloud.json b/vectordb_bench/results/ZillizCloud/result_20230808_standard_zillizcloud.json index 8d992d523..68460e649 100644 --- a/vectordb_bench/results/ZillizCloud/result_20230808_standard_zillizcloud.json +++ b/vectordb_bench/results/ZillizCloud/result_20230808_standard_zillizcloud.json @@ -491,10 +491,10 @@ { "metrics": { "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 + "load_duration": 2586.0, + "qps": 379.9721, + "serial_latency_p99": 0.0124, + "recall": 0.982 }, "task_config": { "db": "ZillizCloud", @@ -513,7 +513,7 @@ "custom_case": {} } }, - "label": "x" + "label": ":)" }, { "metrics": { diff --git a/vectordb_bench/results/leaderboard.json b/vectordb_bench/results/leaderboard.json index f86dd8724..320ae4be8 100644 --- a/vectordb_bench/results/leaderboard.json +++ b/vectordb_bench/results/leaderboard.json @@ -1 +1 @@ -[{"db":"PgVector","db_label":"2c8g-60gdisk","db_name":"PgVector-2c8g-60gdisk","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":0.8836,"latency":2523.0,"recall":0.8528,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-60gdisk","db_name":"PgVector-2c8g-60gdisk","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-60gdisk","db_name":"PgVector-2c8g-60gdisk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":0.8937,"latency":3720.2000000000003,"recall":0.8525,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-60gdisk","db_name":"PgVector-2c8g-60gdisk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-60gdisk","db_name":"PgVector-2c8g-60gdisk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":1.2145,"latency":3622.3999999999996,"recall":0.7487,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-60gdisk","db_name":"PgVector-2c8g-60gdisk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-1node","db_name":"PgVector-2c8g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":10.6271,"latency":730.7,"recall":0.8898,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-1node","db_name":"PgVector-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":10.8507,"latency":733.1999999999999,"recall":0.8897,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-1node","db_name":"PgVector-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":75.7055,"latency":121.2,"recall":0.9999,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-1node","db_name":"PgVector-2c8g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-1node","db_name":"PgVector-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-1node","db_name":"PgVector-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":11.2945,"latency":3611.2000000000003,"recall":0.996,"label":1,"qp$":23.564573336115167},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":17.3271,"latency":3774.7999999999997,"recall":0.9961,"label":1,"qp$":36.15084498226581},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":26.26,"latency":556.1,"recall":0.9999,"label":1,"qp$":54.78823283955769},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":15.2269,"latency":861.8,"recall":0.9888,"label":1,"qp$":31.76903818068016},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":15.1749,"latency":774.3,"recall":0.989,"label":1,"qp$":31.660546630502814},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":27.6181,"latency":305.5,"recall":0.9999,"label":1,"qp$":57.621740037554765},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":228.4,"latency":22.2,"recall":0.9348,"label":1,"qp$":1564.3835616438357},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":181.5,"latency":26.1,"recall":0.9345,"label":1,"qp$":1243.150684931507},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":205.7,"latency":24.2,"recall":0.9586,"label":1,"qp$":1408.9041095890411},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":67.63,"latency":36.0,"recall":0.8064,"label":1,"qp$":695.0668036998972},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":63.35,"latency":38.4,"recall":0.8065,"label":1,"qp$":651.0791366906475},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":176.7,"latency":27.599999999999998,"recall":1.0,"label":1,"qp$":1816.032887975334},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":15.33,"latency":84.9,"recall":0.8064,"label":1,"qp$":157.55395683453239},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":15.13,"latency":86.7,"recall":0.8065,"label":1,"qp$":155.49845837615624},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":17.41,"latency":74.30000000000001,"recall":1.0,"label":1,"qp$":178.9311408016444},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":16.34,"latency":88.7,"recall":0.879,"label":1,"qp$":83.7948717948718},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":10.45,"latency":126.8,"recall":0.8208,"label":1,"qp$":53.589743589743584},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":16.18,"latency":87.5,"recall":0.8793,"label":1,"qp$":82.97435897435896},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":9.8,"latency":130.89999999999998,"recall":0.8212,"label":1,"qp$":50.256410256410255},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":36.11,"latency":55.1,"recall":1.0,"label":1,"qp$":185.17948717948718},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":14.84,"latency":92.7,"recall":0.96,"label":1,"qp$":76.1025641025641},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":322.7,"latency":26.4,"recall":0.9478,"label":1,"qp$":276.2842465753425},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":265.5,"latency":26.9,"recall":0.9332,"label":1,"qp$":227.31164383561645},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":303.8,"latency":27.3,"recall":0.9478,"label":1,"qp$":260.10273972602744},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":180.2,"latency":28.2,"recall":0.9335,"label":1,"qp$":154.28082191780823},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":730.7,"latency":24.6,"recall":0.9586,"label":1,"qp$":625.5993150684932},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":104.3,"latency":31.7,"recall":0.9563,"label":1,"qp$":89.29794520547945},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":147.7,"latency":35.3,"recall":0.9707,"label":1,"qp$":189.60205391527597},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":782.5,"latency":25.9,"recall":1.0,"label":1,"qp$":1004.4929396662387},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":18.7634,"latency":153.70000000000002,"recall":0.8737,"label":1,"qp$":192.84069886947586},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":18.3619,"latency":79.8,"recall":0.8741,"label":1,"qp$":188.7142857142857},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":25.2744,"latency":61.199999999999996,"recall":0.9979,"label":1,"qp$":259.7574511819116},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":261.798,"latency":23.099999999999998,"recall":0.9262,"label":1,"qp$":1793.13698630137},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":166.1851,"latency":23.900000000000002,"recall":0.9264,"label":1,"qp$":1138.2541095890413},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":121.7169,"latency":29.0,"recall":0.9693,"label":1,"qp$":833.677397260274},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":46.6189,"latency":43.1,"recall":0.8737,"label":1,"qp$":479.1253854059609},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":42.4856,"latency":44.0,"recall":0.8741,"label":1,"qp$":436.6454265159301},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":138.9479,"latency":26.200000000000003,"recall":0.9979,"label":1,"qp$":1428.0359712230218},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":20.7437,"latency":75.80000000000001,"recall":0.9291,"label":1,"qp$":106.37794871794871},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":20.2993,"latency":76.5,"recall":0.9293,"label":1,"qp$":104.09897435897435},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":26.4719,"latency":67.0,"recall":1.0,"label":1,"qp$":135.75333333333333},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":8.6675,"latency":180.2,"recall":0.8369,"label":1,"qp$":44.44871794871795},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":7.8121,"latency":167.7,"recall":0.8369,"label":1,"qp$":40.06205128205128},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":16.869,"latency":87.8,"recall":0.9814,"label":1,"qp$":86.50769230769231},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":365.0835,"latency":23.599999999999998,"recall":0.945,"label":1,"qp$":312.57148972602744},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":325.5271,"latency":25.1,"recall":0.9452,"label":1,"qp$":278.7047089041096},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":596.7942,"latency":24.2,"recall":0.9693,"label":1,"qp$":510.95393835616443},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":303.2044,"latency":27.400000000000002,"recall":0.9246,"label":1,"qp$":259.5928082191781},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":136.0345,"latency":31.9,"recall":0.9244,"label":1,"qp$":116.46789383561645},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":66.7221,"latency":42.1,"recall":0.963,"label":1,"qp$":57.125085616438355},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":131.2549,"latency":30.200000000000003,"recall":0.9867,"label":1,"qp$":168.49152759948652},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":127.9337,"latency":30.099999999999998,"recall":0.9869,"label":1,"qp$":164.2281129653402},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":595.8462,"latency":23.400000000000002,"recall":1.0,"label":1,"qp$":764.8860077021822},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":330.0144,"latency":9.0,"recall":0.9507,"label":1,"qp$":2075.5622641509435},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":271.6585,"latency":10.1,"recall":0.9678,"label":1,"qp$":1708.5440251572327},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":216.5226,"latency":12.9,"recall":1.0,"label":1,"qp$":1361.777358490566},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":123.9553,"latency":23.0,"recall":0.971,"label":1,"qp$":389.79654088050313},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":59.1479,"latency":44.5,"recall":0.9906,"label":1,"qp$":185.99968553459118},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":40.999,"latency":55.300000000000004,"recall":1.0,"label":1,"qp$":128.92767295597486},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":579.9416,"latency":9.4,"recall":0.9213,"label":1,"qp$":1823.71572327044},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":425.2529,"latency":11.299999999999999,"recall":0.9686,"label":1,"qp$":1337.2732704402515},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":397.0539,"latency":13.799999999999999,"recall":1.0,"label":1,"qp$":1248.5971698113208},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":516.27,"latency":7.0,"recall":0.9463,"label":1,"qp$":3246.9811320754716},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":354.8416,"latency":10.0,"recall":0.9802,"label":1,"qp$":2231.708176100629},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":427.5229,"latency":8.7,"recall":1.0,"label":1,"qp$":2688.8232704402517},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":2884.689,"latency":5.3,"recall":0.8801,"label":1,"qp$":2267.837264150943},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":1689.5799,"latency":6.6,"recall":0.9493,"label":1,"qp$":1328.2860849056603},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1517.6792,"latency":10.0,"recall":1.0,"label":1,"qp$":1193.1440251572328},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":822.5318,"latency":5.6,"recall":0.9294,"label":1,"qp$":646.6444968553459},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":378.9146,"latency":10.3,"recall":0.9758,"label":1,"qp$":297.8888364779874},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":218.6854,"latency":16.2,"recall":1.0,"label":1,"qp$":171.92248427672953},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":297.5,"latency":7.2,"recall":0.974,"label":1,"qp$":1871.0691823899372},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":228.3,"latency":10.6,"recall":0.994,"label":1,"qp$":1435.8490566037735},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":584.0,"latency":4.6,"recall":1.0,"label":1,"qp$":3672.9559748427673},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":1871.0,"latency":7.0,"recall":0.9602,"label":1,"qp$":1470.9119496855346},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":556.7,"latency":6.7,"recall":0.9723,"label":1,"qp$":437.65723270440253},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1583.0,"latency":6.8,"recall":0.9836,"label":1,"qp$":1244.4968553459119},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":294.3,"latency":10.9,"recall":0.9939,"label":1,"qp$":231.3679245283019},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":2345.0,"latency":8.9,"recall":1.0,"label":1,"qp$":1843.553459119497},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":295.6,"latency":12.3,"recall":1.0,"label":1,"qp$":232.38993710691824},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":143.0,"latency":33.5,"recall":0.9818,"label":1,"qp$":899.3710691823899},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":106.0,"latency":20.7,"recall":0.9887,"label":1,"qp$":666.6666666666666},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":189.0,"latency":11.6,"recall":1.0,"label":1,"qp$":1188.6792452830189},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":71.74,"latency":50.8,"recall":0.9883,"label":1,"qp$":225.59748427672955},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":287.0,"latency":14.9,"recall":0.9865,"label":1,"qp$":902.5157232704403},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":34.6654,"latency":64.69999999999999,"recall":0.9961,"label":1,"qp$":109.01069182389936},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":412.0,"latency":10.3,"recall":1.0,"label":1,"qp$":1295.5974842767296},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":42.169,"latency":46.800000000000004,"recall":1.0,"label":1,"qp$":132.6069182389937},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":537.4975,"latency":18.9,"recall":0.8903,"label":1,"qp$":376.9267180925666},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":372.0466,"latency":17.8,"recall":0.8904,"label":1,"qp$":260.9022440392707},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1156.2898,"latency":14.4,"recall":0.9989,"label":1,"qp$":810.8624123422161},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":110.248,"latency":69.0,"recall":0.898,"label":1,"qp$":77.31276297335204},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":87.2601,"latency":27.799999999999997,"recall":0.898,"label":1,"qp$":61.1922159887798},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":125.7846,"latency":23.099999999999998,"recall":0.975,"label":1,"qp$":88.20799438990183},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":240.7209,"latency":17.4,"recall":0.8887,"label":1,"qp$":844.0424263674614},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":189.4399,"latency":17.5,"recall":0.8889,"label":1,"qp$":664.2352734922861},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":313.5116,"latency":16.1,"recall":0.9999,"label":1,"qp$":1099.2692847124824},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":78.7196,"latency":49.4,"recall":0.9203,"label":1,"qp$":474.21445783132526},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":68.3111,"latency":35.5,"recall":0.9202,"label":1,"qp$":411.5126506024096},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":210.2147,"latency":26.8,"recall":0.9996,"label":1,"qp$":1266.3536144578313},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":188.6436,"latency":917.5,"recall":0.9175,"label":1,"qp$":661.4431977559607},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":155.6991,"latency":917.1,"recall":0.9171,"label":1,"qp$":545.929523141655},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":445.3289,"latency":14.1,"recall":0.9999,"label":1,"qp$":1561.461781206171},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":633.6033,"latency":24.6,"recall":0.919,"label":1,"qp$":444.32208976157085},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":95.5682,"latency":58.7,"recall":0.9463,"label":1,"qp$":67.01837307152876},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":434.4062,"latency":17.4,"recall":0.9181,"label":1,"qp$":304.6326788218794},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":80.3192,"latency":26.599999999999998,"recall":0.9462,"label":1,"qp$":56.324824684431974},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":1509.3293,"latency":18.5,"recall":0.9995,"label":1,"qp$":1058.4356942496495},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":166.7252,"latency":19.599999999999998,"recall":0.9988,"label":1,"qp$":116.91809256661992},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":46.8622,"latency":123.0,"recall":0.9957,"label":1,"qp$":4.639821782178218},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1.8388,"latency":1063.5,"recall":0.9957,"label":1,"qp$":0.18205940594059405},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":45.0666,"latency":109.2,"recall":1.0,"label":1,"qp$":4.462039603960396},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":43.5017,"latency":228.7,"recall":0.9957,"label":1,"qp$":1.3344079754601226},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1.5668,"latency":1114.4,"recall":0.9957,"label":1,"qp$":0.04806134969325153},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":41.5443,"latency":159.0,"recall":1.0,"label":1,"qp$":1.2743650306748466},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":67.9121,"latency":179.5,"recall":0.9909,"label":1,"qp$":2.083193251533742},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.7636,"latency":1921.3,"recall":0.9908,"label":1,"qp$":0.02342331288343558},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":32.0,"latency":124.5,"recall":1.0,"label":1,"qp$":0.9815950920245399},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":63.1365,"latency":145.7,"recall":0.991,"label":1,"qp$":6.251138613861386},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.7512,"latency":1983.8,"recall":0.9908,"label":1,"qp$":0.07437623762376237},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":30.1358,"latency":129.8,"recall":1.0,"label":1,"qp$":2.983742574257426},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":37.432,"latency":75.0,"recall":0.9975,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":37.0696,"latency":73.5,"recall":0.9976,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":81.1915,"latency":53.0,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":321.6048,"latency":13.4,"recall":0.989,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":22.1467,"latency":86.8,"recall":0.9972,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":303.2551,"latency":10.9,"recall":0.9876,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":21.5388,"latency":81.69999999999999,"recall":0.997,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":394.5418,"latency":8.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":37.878,"latency":45.0,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":180.2757,"latency":6.0,"recall":0.9942,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":179.0033,"latency":6.4,"recall":0.9943,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":526.8846,"latency":3.6,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":626.5243,"latency":6.2,"recall":0.9954,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":78.4227,"latency":25.3,"recall":0.9982,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":599.4213,"latency":6.6,"recall":0.9955,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":78.5351,"latency":26.3,"recall":0.9982,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":2098.2113,"latency":3.4,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":275.6292,"latency":10.0,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":100.6667,"latency":21.1,"recall":0.9909,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":101.1399,"latency":19.7,"recall":0.9907,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":52.2606,"latency":18.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":61.0661,"latency":49.8,"recall":0.9911,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":58.9326,"latency":44.6,"recall":0.9911,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":42.5977,"latency":54.9,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":536.0726,"latency":8.200000000000001,"recall":0.9728,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":467.179,"latency":7.0,"recall":0.9697,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":431.7512,"latency":8.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":274.5407,"latency":4.8999999999999995,"recall":0.9807,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":236.5672,"latency":10.3,"recall":0.981,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":309.4833,"latency":4.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":178.6585,"latency":13.700000000000001,"recall":0.9843,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":178.3732,"latency":15.0,"recall":0.9844,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":229.3526,"latency":12.5,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":1258.7043,"latency":4.8999999999999995,"recall":0.9799,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":1075.8776,"latency":5.3,"recall":0.98,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1494.8493,"latency":4.7,"recall":1.0,"label":1,"qp$":0.0}] \ No newline at end of file +[{"db":"PgVector","db_label":"2c8g-60gdisk","db_name":"PgVector-2c8g-60gdisk","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":0.8836,"latency":2523.0,"recall":0.8528,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-60gdisk","db_name":"PgVector-2c8g-60gdisk","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-60gdisk","db_name":"PgVector-2c8g-60gdisk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":0.8937,"latency":3720.2000000000003,"recall":0.8525,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-60gdisk","db_name":"PgVector-2c8g-60gdisk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-60gdisk","db_name":"PgVector-2c8g-60gdisk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":1.2145,"latency":3622.3999999999996,"recall":0.7487,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-60gdisk","db_name":"PgVector-2c8g-60gdisk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-1node","db_name":"PgVector-2c8g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":10.6271,"latency":730.7,"recall":0.8898,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-1node","db_name":"PgVector-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":10.8507,"latency":733.1999999999999,"recall":0.8897,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-1node","db_name":"PgVector-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":75.7055,"latency":121.2,"recall":0.9999,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-1node","db_name":"PgVector-2c8g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-1node","db_name":"PgVector-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-1node","db_name":"PgVector-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":11.2945,"latency":3611.2000000000003,"recall":0.996,"label":1,"qp$":23.564573336115167},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":17.3271,"latency":3774.7999999999997,"recall":0.9961,"label":1,"qp$":36.15084498226581},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":26.26,"latency":556.1,"recall":0.9999,"label":1,"qp$":54.78823283955769},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":15.2269,"latency":861.8,"recall":0.9888,"label":1,"qp$":31.76903818068016},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":15.1749,"latency":774.3,"recall":0.989,"label":1,"qp$":31.660546630502814},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":27.6181,"latency":305.5,"recall":0.9999,"label":1,"qp$":57.621740037554765},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":228.4,"latency":22.2,"recall":0.9348,"label":1,"qp$":1564.3835616438357},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":181.5,"latency":26.1,"recall":0.9345,"label":1,"qp$":1243.150684931507},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":205.7,"latency":24.2,"recall":0.9586,"label":1,"qp$":1408.9041095890411},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":67.63,"latency":36.0,"recall":0.8064,"label":1,"qp$":695.0668036998972},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":63.35,"latency":38.4,"recall":0.8065,"label":1,"qp$":651.0791366906475},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":176.7,"latency":27.599999999999998,"recall":1.0,"label":1,"qp$":1816.032887975334},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":15.33,"latency":84.9,"recall":0.8064,"label":1,"qp$":157.55395683453239},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":15.13,"latency":86.7,"recall":0.8065,"label":1,"qp$":155.49845837615624},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":17.41,"latency":74.30000000000001,"recall":1.0,"label":1,"qp$":178.9311408016444},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":16.34,"latency":88.7,"recall":0.879,"label":1,"qp$":83.7948717948718},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":10.45,"latency":126.8,"recall":0.8208,"label":1,"qp$":53.589743589743584},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":16.18,"latency":87.5,"recall":0.8793,"label":1,"qp$":82.97435897435896},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":9.8,"latency":130.89999999999998,"recall":0.8212,"label":1,"qp$":50.256410256410255},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":36.11,"latency":55.1,"recall":1.0,"label":1,"qp$":185.17948717948718},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":14.84,"latency":92.7,"recall":0.96,"label":1,"qp$":76.1025641025641},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":322.7,"latency":26.4,"recall":0.9478,"label":1,"qp$":276.2842465753425},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":265.5,"latency":26.9,"recall":0.9332,"label":1,"qp$":227.31164383561645},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":303.8,"latency":27.3,"recall":0.9478,"label":1,"qp$":260.10273972602744},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":180.2,"latency":28.2,"recall":0.9335,"label":1,"qp$":154.28082191780823},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":730.7,"latency":24.6,"recall":0.9586,"label":1,"qp$":625.5993150684932},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":104.3,"latency":31.7,"recall":0.9563,"label":1,"qp$":89.29794520547945},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":147.7,"latency":35.3,"recall":0.9707,"label":1,"qp$":189.60205391527597},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":782.5,"latency":25.9,"recall":1.0,"label":1,"qp$":1004.4929396662387},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":18.7634,"latency":153.70000000000002,"recall":0.8737,"label":1,"qp$":192.84069886947586},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":18.3619,"latency":79.8,"recall":0.8741,"label":1,"qp$":188.7142857142857},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":25.2744,"latency":61.199999999999996,"recall":0.9979,"label":1,"qp$":259.7574511819116},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":261.798,"latency":23.099999999999998,"recall":0.9262,"label":1,"qp$":1793.13698630137},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":166.1851,"latency":23.900000000000002,"recall":0.9264,"label":1,"qp$":1138.2541095890413},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":121.7169,"latency":29.0,"recall":0.9693,"label":1,"qp$":833.677397260274},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":46.6189,"latency":43.1,"recall":0.8737,"label":1,"qp$":479.1253854059609},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":42.4856,"latency":44.0,"recall":0.8741,"label":1,"qp$":436.6454265159301},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":138.9479,"latency":26.200000000000003,"recall":0.9979,"label":1,"qp$":1428.0359712230218},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":20.7437,"latency":75.80000000000001,"recall":0.9291,"label":1,"qp$":106.37794871794871},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":20.2993,"latency":76.5,"recall":0.9293,"label":1,"qp$":104.09897435897435},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":26.4719,"latency":67.0,"recall":1.0,"label":1,"qp$":135.75333333333333},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":8.6675,"latency":180.2,"recall":0.8369,"label":1,"qp$":44.44871794871795},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":7.8121,"latency":167.7,"recall":0.8369,"label":1,"qp$":40.06205128205128},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":16.869,"latency":87.8,"recall":0.9814,"label":1,"qp$":86.50769230769231},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":365.0835,"latency":23.599999999999998,"recall":0.945,"label":1,"qp$":312.57148972602744},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":325.5271,"latency":25.1,"recall":0.9452,"label":1,"qp$":278.7047089041096},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":596.7942,"latency":24.2,"recall":0.9693,"label":1,"qp$":510.95393835616443},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":303.2044,"latency":27.400000000000002,"recall":0.9246,"label":1,"qp$":259.5928082191781},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":136.0345,"latency":31.9,"recall":0.9244,"label":1,"qp$":116.46789383561645},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":66.7221,"latency":42.1,"recall":0.963,"label":1,"qp$":57.125085616438355},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":131.2549,"latency":30.200000000000003,"recall":0.9867,"label":1,"qp$":168.49152759948652},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":127.9337,"latency":30.099999999999998,"recall":0.9869,"label":1,"qp$":164.2281129653402},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":595.8462,"latency":23.400000000000002,"recall":1.0,"label":1,"qp$":764.8860077021822},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":330.0144,"latency":9.0,"recall":0.9507,"label":1,"qp$":2075.5622641509435},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":271.6585,"latency":10.1,"recall":0.9678,"label":1,"qp$":1708.5440251572327},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":216.5226,"latency":12.9,"recall":1.0,"label":1,"qp$":1361.777358490566},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":123.9553,"latency":23.0,"recall":0.971,"label":1,"qp$":389.79654088050313},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":59.1479,"latency":44.5,"recall":0.9906,"label":1,"qp$":185.99968553459118},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":40.999,"latency":55.300000000000004,"recall":1.0,"label":1,"qp$":128.92767295597486},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":579.9416,"latency":9.4,"recall":0.9213,"label":1,"qp$":1823.71572327044},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":425.2529,"latency":11.299999999999999,"recall":0.9686,"label":1,"qp$":1337.2732704402515},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":397.0539,"latency":13.799999999999999,"recall":1.0,"label":1,"qp$":1248.5971698113208},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":516.27,"latency":7.0,"recall":0.9463,"label":1,"qp$":3246.9811320754716},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":354.8416,"latency":10.0,"recall":0.9802,"label":1,"qp$":2231.708176100629},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":427.5229,"latency":8.7,"recall":1.0,"label":1,"qp$":2688.8232704402517},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":2884.689,"latency":5.3,"recall":0.8801,"label":1,"qp$":2267.837264150943},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":1689.5799,"latency":6.6,"recall":0.9493,"label":1,"qp$":1328.2860849056603},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1517.6792,"latency":10.0,"recall":1.0,"label":1,"qp$":1193.1440251572328},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":822.5318,"latency":5.6,"recall":0.9294,"label":1,"qp$":646.6444968553459},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":378.9146,"latency":10.3,"recall":0.9758,"label":1,"qp$":297.8888364779874},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":218.6854,"latency":16.2,"recall":1.0,"label":1,"qp$":171.92248427672953},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":297.5,"latency":7.2,"recall":0.974,"label":1,"qp$":1871.0691823899372},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":228.3,"latency":10.6,"recall":0.994,"label":1,"qp$":1435.8490566037735},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":584.0,"latency":4.6,"recall":1.0,"label":1,"qp$":3672.9559748427673},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":1871.0,"latency":7.0,"recall":0.9602,"label":1,"qp$":1470.9119496855346},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":556.7,"latency":6.7,"recall":0.9723,"label":1,"qp$":437.65723270440253},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1583.0,"latency":6.8,"recall":0.9836,"label":1,"qp$":1244.4968553459119},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":294.3,"latency":10.9,"recall":0.9939,"label":1,"qp$":231.3679245283019},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":2345.0,"latency":8.9,"recall":1.0,"label":1,"qp$":1843.553459119497},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":295.6,"latency":12.3,"recall":1.0,"label":1,"qp$":232.38993710691824},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":143.0,"latency":33.5,"recall":0.9818,"label":1,"qp$":899.3710691823899},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":106.0,"latency":20.7,"recall":0.9887,"label":1,"qp$":666.6666666666666},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":189.0,"latency":11.6,"recall":1.0,"label":1,"qp$":1188.6792452830189},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":379.9721,"latency":12.4,"recall":0.982,"label":1,"qp$":1194.8808176100629},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":71.74,"latency":50.8,"recall":0.9883,"label":1,"qp$":225.59748427672955},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":287.0,"latency":14.9,"recall":0.9865,"label":1,"qp$":902.5157232704403},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":34.6654,"latency":64.69999999999999,"recall":0.9961,"label":1,"qp$":109.01069182389936},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":412.0,"latency":10.3,"recall":1.0,"label":1,"qp$":1295.5974842767296},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":42.169,"latency":46.800000000000004,"recall":1.0,"label":1,"qp$":132.6069182389937},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":537.4975,"latency":18.9,"recall":0.8903,"label":1,"qp$":376.9267180925666},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":372.0466,"latency":17.8,"recall":0.8904,"label":1,"qp$":260.9022440392707},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1156.2898,"latency":14.4,"recall":0.9989,"label":1,"qp$":810.8624123422161},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":110.248,"latency":69.0,"recall":0.898,"label":1,"qp$":77.31276297335204},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":87.2601,"latency":27.799999999999997,"recall":0.898,"label":1,"qp$":61.1922159887798},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":125.7846,"latency":23.099999999999998,"recall":0.975,"label":1,"qp$":88.20799438990183},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":240.7209,"latency":17.4,"recall":0.8887,"label":1,"qp$":844.0424263674614},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":189.4399,"latency":17.5,"recall":0.8889,"label":1,"qp$":664.2352734922861},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":313.5116,"latency":16.1,"recall":0.9999,"label":1,"qp$":1099.2692847124824},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":78.7196,"latency":49.4,"recall":0.9203,"label":1,"qp$":474.21445783132526},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":68.3111,"latency":35.5,"recall":0.9202,"label":1,"qp$":411.5126506024096},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":210.2147,"latency":26.8,"recall":0.9996,"label":1,"qp$":1266.3536144578313},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":188.6436,"latency":917.5,"recall":0.9175,"label":1,"qp$":661.4431977559607},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":155.6991,"latency":917.1,"recall":0.9171,"label":1,"qp$":545.929523141655},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":445.3289,"latency":14.1,"recall":0.9999,"label":1,"qp$":1561.461781206171},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":633.6033,"latency":24.6,"recall":0.919,"label":1,"qp$":444.32208976157085},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":95.5682,"latency":58.7,"recall":0.9463,"label":1,"qp$":67.01837307152876},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":434.4062,"latency":17.4,"recall":0.9181,"label":1,"qp$":304.6326788218794},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":80.3192,"latency":26.599999999999998,"recall":0.9462,"label":1,"qp$":56.324824684431974},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":1509.3293,"latency":18.5,"recall":0.9995,"label":1,"qp$":1058.4356942496495},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":166.7252,"latency":19.599999999999998,"recall":0.9988,"label":1,"qp$":116.91809256661992},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":46.8622,"latency":123.0,"recall":0.9957,"label":1,"qp$":4.639821782178218},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1.8388,"latency":1063.5,"recall":0.9957,"label":1,"qp$":0.18205940594059405},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":45.0666,"latency":109.2,"recall":1.0,"label":1,"qp$":4.462039603960396},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":43.5017,"latency":228.7,"recall":0.9957,"label":1,"qp$":1.3344079754601226},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1.5668,"latency":1114.4,"recall":0.9957,"label":1,"qp$":0.04806134969325153},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":41.5443,"latency":159.0,"recall":1.0,"label":1,"qp$":1.2743650306748466},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":67.9121,"latency":179.5,"recall":0.9909,"label":1,"qp$":2.083193251533742},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.7636,"latency":1921.3,"recall":0.9908,"label":1,"qp$":0.02342331288343558},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":32.0,"latency":124.5,"recall":1.0,"label":1,"qp$":0.9815950920245399},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":63.1365,"latency":145.7,"recall":0.991,"label":1,"qp$":6.251138613861386},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.7512,"latency":1983.8,"recall":0.9908,"label":1,"qp$":0.07437623762376237},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":30.1358,"latency":129.8,"recall":1.0,"label":1,"qp$":2.983742574257426},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":37.432,"latency":75.0,"recall":0.9975,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":37.0696,"latency":73.5,"recall":0.9976,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":81.1915,"latency":53.0,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":321.6048,"latency":13.4,"recall":0.989,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":22.1467,"latency":86.8,"recall":0.9972,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":303.2551,"latency":10.9,"recall":0.9876,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":21.5388,"latency":81.69999999999999,"recall":0.997,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":394.5418,"latency":8.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":37.878,"latency":45.0,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":180.2757,"latency":6.0,"recall":0.9942,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":179.0033,"latency":6.4,"recall":0.9943,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":526.8846,"latency":3.6,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":626.5243,"latency":6.2,"recall":0.9954,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":78.4227,"latency":25.3,"recall":0.9982,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":599.4213,"latency":6.6,"recall":0.9955,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":78.5351,"latency":26.3,"recall":0.9982,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":2098.2113,"latency":3.4,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":275.6292,"latency":10.0,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":100.6667,"latency":21.1,"recall":0.9909,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":101.1399,"latency":19.7,"recall":0.9907,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":52.2606,"latency":18.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":61.0661,"latency":49.8,"recall":0.9911,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":58.9326,"latency":44.6,"recall":0.9911,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":42.5977,"latency":54.9,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":536.0726,"latency":8.200000000000001,"recall":0.9728,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":467.179,"latency":7.0,"recall":0.9697,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":431.7512,"latency":8.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":274.5407,"latency":4.8999999999999995,"recall":0.9807,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":236.5672,"latency":10.3,"recall":0.981,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":309.4833,"latency":4.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":178.6585,"latency":13.700000000000001,"recall":0.9843,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":178.3732,"latency":15.0,"recall":0.9844,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":229.3526,"latency":12.5,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":1258.7043,"latency":4.8999999999999995,"recall":0.9799,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":1075.8776,"latency":5.3,"recall":0.98,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1494.8493,"latency":4.7,"recall":1.0,"label":1,"qp$":0.0}] \ No newline at end of file From 3a8b715bae506b6670d6720af754d7279071a11b Mon Sep 17 00:00:00 2001 From: Li Liu Date: Tue, 15 Aug 2023 17:35:00 +0800 Subject: [PATCH 017/327] Sync pgvector's label name Signed-off-by: Li Liu --- .../result_20230727_standard_pgvector.json | 16 ++++++++-------- .../result_20230808_standard_pgvector.json | 12 ++++++------ vectordb_bench/results/leaderboard.json | 2 +- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/vectordb_bench/results/PgVector/result_20230727_standard_pgvector.json b/vectordb_bench/results/PgVector/result_20230727_standard_pgvector.json index 5385e1f5c..4a5b9d5fe 100644 --- a/vectordb_bench/results/PgVector/result_20230727_standard_pgvector.json +++ b/vectordb_bench/results/PgVector/result_20230727_standard_pgvector.json @@ -13,7 +13,7 @@ "task_config": { "db": "PgVector", "db_config": { - "db_label": "2c8g-1node", + "db_label": "2c8g", "user_name": "**********", "password": "**********", "url": "**********", @@ -42,7 +42,7 @@ "task_config": { "db": "PgVector", "db_config": { - "db_label": "2c8g-1node", + "db_label": "2c8g", "user_name": "**********", "password": "**********", "url": "**********", @@ -71,7 +71,7 @@ "task_config": { "db": "PgVector", "db_config": { - "db_label": "2c8g-1node", + "db_label": "2c8g", "user_name": "**********", "password": "**********", "url": "**********", @@ -100,7 +100,7 @@ "task_config": { "db": "PgVector", "db_config": { - "db_label": "2c8g-1node", + "db_label": "2c8g", "user_name": "**********", "password": "**********", "url": "**********", @@ -129,7 +129,7 @@ "task_config": { "db": "PgVector", "db_config": { - "db_label": "2c8g-1node", + "db_label": "2c8g", "user_name": "**********", "password": "**********", "url": "**********", @@ -158,7 +158,7 @@ "task_config": { "db": "PgVector", "db_config": { - "db_label": "2c8g-1node", + "db_label": "2c8g", "user_name": "**********", "password": "**********", "url": "**********", @@ -187,7 +187,7 @@ "task_config": { "db": "PgVector", "db_config": { - "db_label": "2c8g-1node", + "db_label": "2c8g", "user_name": "**********", "password": "**********", "url": "**********", @@ -216,7 +216,7 @@ "task_config": { "db": "PgVector", "db_config": { - "db_label": "2c8g-1node", + "db_label": "2c8g", "user_name": "**********", "password": "**********", "url": "**********", diff --git a/vectordb_bench/results/PgVector/result_20230808_standard_pgvector.json b/vectordb_bench/results/PgVector/result_20230808_standard_pgvector.json index 16bc838c0..5e8a15c98 100644 --- a/vectordb_bench/results/PgVector/result_20230808_standard_pgvector.json +++ b/vectordb_bench/results/PgVector/result_20230808_standard_pgvector.json @@ -13,7 +13,7 @@ "task_config": { "db": "PgVector", "db_config": { - "db_label": "2c8g-60gdisk", + "db_label": "2c8g", "user_name": "**********", "password": "**********", "url": "**********", @@ -42,7 +42,7 @@ "task_config": { "db": "PgVector", "db_config": { - "db_label": "2c8g-60gdisk", + "db_label": "2c8g", "user_name": "**********", "password": "**********", "url": "**********", @@ -71,7 +71,7 @@ "task_config": { "db": "PgVector", "db_config": { - "db_label": "2c8g-60gdisk", + "db_label": "2c8g", "user_name": "**********", "password": "**********", "url": "**********", @@ -100,7 +100,7 @@ "task_config": { "db": "PgVector", "db_config": { - "db_label": "2c8g-60gdisk", + "db_label": "2c8g", "user_name": "**********", "password": "**********", "url": "**********", @@ -129,7 +129,7 @@ "task_config": { "db": "PgVector", "db_config": { - "db_label": "2c8g-60gdisk", + "db_label": "2c8g", "user_name": "**********", "password": "**********", "url": "**********", @@ -158,7 +158,7 @@ "task_config": { "db": "PgVector", "db_config": { - "db_label": "2c8g-60gdisk", + "db_label": "2c8g", "user_name": "**********", "password": "**********", "url": "**********", diff --git a/vectordb_bench/results/leaderboard.json b/vectordb_bench/results/leaderboard.json index 320ae4be8..a09e4ba1b 100644 --- a/vectordb_bench/results/leaderboard.json +++ b/vectordb_bench/results/leaderboard.json @@ -1 +1 @@ -[{"db":"PgVector","db_label":"2c8g-60gdisk","db_name":"PgVector-2c8g-60gdisk","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":0.8836,"latency":2523.0,"recall":0.8528,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-60gdisk","db_name":"PgVector-2c8g-60gdisk","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-60gdisk","db_name":"PgVector-2c8g-60gdisk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":0.8937,"latency":3720.2000000000003,"recall":0.8525,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-60gdisk","db_name":"PgVector-2c8g-60gdisk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-60gdisk","db_name":"PgVector-2c8g-60gdisk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":1.2145,"latency":3622.3999999999996,"recall":0.7487,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-60gdisk","db_name":"PgVector-2c8g-60gdisk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-1node","db_name":"PgVector-2c8g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":10.6271,"latency":730.7,"recall":0.8898,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-1node","db_name":"PgVector-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":10.8507,"latency":733.1999999999999,"recall":0.8897,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-1node","db_name":"PgVector-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":75.7055,"latency":121.2,"recall":0.9999,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-1node","db_name":"PgVector-2c8g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-1node","db_name":"PgVector-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g-1node","db_name":"PgVector-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":11.2945,"latency":3611.2000000000003,"recall":0.996,"label":1,"qp$":23.564573336115167},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":17.3271,"latency":3774.7999999999997,"recall":0.9961,"label":1,"qp$":36.15084498226581},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":26.26,"latency":556.1,"recall":0.9999,"label":1,"qp$":54.78823283955769},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":15.2269,"latency":861.8,"recall":0.9888,"label":1,"qp$":31.76903818068016},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":15.1749,"latency":774.3,"recall":0.989,"label":1,"qp$":31.660546630502814},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":27.6181,"latency":305.5,"recall":0.9999,"label":1,"qp$":57.621740037554765},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":228.4,"latency":22.2,"recall":0.9348,"label":1,"qp$":1564.3835616438357},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":181.5,"latency":26.1,"recall":0.9345,"label":1,"qp$":1243.150684931507},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":205.7,"latency":24.2,"recall":0.9586,"label":1,"qp$":1408.9041095890411},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":67.63,"latency":36.0,"recall":0.8064,"label":1,"qp$":695.0668036998972},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":63.35,"latency":38.4,"recall":0.8065,"label":1,"qp$":651.0791366906475},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":176.7,"latency":27.599999999999998,"recall":1.0,"label":1,"qp$":1816.032887975334},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":15.33,"latency":84.9,"recall":0.8064,"label":1,"qp$":157.55395683453239},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":15.13,"latency":86.7,"recall":0.8065,"label":1,"qp$":155.49845837615624},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":17.41,"latency":74.30000000000001,"recall":1.0,"label":1,"qp$":178.9311408016444},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":16.34,"latency":88.7,"recall":0.879,"label":1,"qp$":83.7948717948718},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":10.45,"latency":126.8,"recall":0.8208,"label":1,"qp$":53.589743589743584},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":16.18,"latency":87.5,"recall":0.8793,"label":1,"qp$":82.97435897435896},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":9.8,"latency":130.89999999999998,"recall":0.8212,"label":1,"qp$":50.256410256410255},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":36.11,"latency":55.1,"recall":1.0,"label":1,"qp$":185.17948717948718},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":14.84,"latency":92.7,"recall":0.96,"label":1,"qp$":76.1025641025641},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":322.7,"latency":26.4,"recall":0.9478,"label":1,"qp$":276.2842465753425},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":265.5,"latency":26.9,"recall":0.9332,"label":1,"qp$":227.31164383561645},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":303.8,"latency":27.3,"recall":0.9478,"label":1,"qp$":260.10273972602744},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":180.2,"latency":28.2,"recall":0.9335,"label":1,"qp$":154.28082191780823},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":730.7,"latency":24.6,"recall":0.9586,"label":1,"qp$":625.5993150684932},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":104.3,"latency":31.7,"recall":0.9563,"label":1,"qp$":89.29794520547945},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":147.7,"latency":35.3,"recall":0.9707,"label":1,"qp$":189.60205391527597},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":782.5,"latency":25.9,"recall":1.0,"label":1,"qp$":1004.4929396662387},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":18.7634,"latency":153.70000000000002,"recall":0.8737,"label":1,"qp$":192.84069886947586},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":18.3619,"latency":79.8,"recall":0.8741,"label":1,"qp$":188.7142857142857},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":25.2744,"latency":61.199999999999996,"recall":0.9979,"label":1,"qp$":259.7574511819116},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":261.798,"latency":23.099999999999998,"recall":0.9262,"label":1,"qp$":1793.13698630137},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":166.1851,"latency":23.900000000000002,"recall":0.9264,"label":1,"qp$":1138.2541095890413},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":121.7169,"latency":29.0,"recall":0.9693,"label":1,"qp$":833.677397260274},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":46.6189,"latency":43.1,"recall":0.8737,"label":1,"qp$":479.1253854059609},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":42.4856,"latency":44.0,"recall":0.8741,"label":1,"qp$":436.6454265159301},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":138.9479,"latency":26.200000000000003,"recall":0.9979,"label":1,"qp$":1428.0359712230218},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":20.7437,"latency":75.80000000000001,"recall":0.9291,"label":1,"qp$":106.37794871794871},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":20.2993,"latency":76.5,"recall":0.9293,"label":1,"qp$":104.09897435897435},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":26.4719,"latency":67.0,"recall":1.0,"label":1,"qp$":135.75333333333333},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":8.6675,"latency":180.2,"recall":0.8369,"label":1,"qp$":44.44871794871795},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":7.8121,"latency":167.7,"recall":0.8369,"label":1,"qp$":40.06205128205128},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":16.869,"latency":87.8,"recall":0.9814,"label":1,"qp$":86.50769230769231},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":365.0835,"latency":23.599999999999998,"recall":0.945,"label":1,"qp$":312.57148972602744},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":325.5271,"latency":25.1,"recall":0.9452,"label":1,"qp$":278.7047089041096},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":596.7942,"latency":24.2,"recall":0.9693,"label":1,"qp$":510.95393835616443},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":303.2044,"latency":27.400000000000002,"recall":0.9246,"label":1,"qp$":259.5928082191781},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":136.0345,"latency":31.9,"recall":0.9244,"label":1,"qp$":116.46789383561645},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":66.7221,"latency":42.1,"recall":0.963,"label":1,"qp$":57.125085616438355},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":131.2549,"latency":30.200000000000003,"recall":0.9867,"label":1,"qp$":168.49152759948652},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":127.9337,"latency":30.099999999999998,"recall":0.9869,"label":1,"qp$":164.2281129653402},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":595.8462,"latency":23.400000000000002,"recall":1.0,"label":1,"qp$":764.8860077021822},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":330.0144,"latency":9.0,"recall":0.9507,"label":1,"qp$":2075.5622641509435},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":271.6585,"latency":10.1,"recall":0.9678,"label":1,"qp$":1708.5440251572327},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":216.5226,"latency":12.9,"recall":1.0,"label":1,"qp$":1361.777358490566},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":123.9553,"latency":23.0,"recall":0.971,"label":1,"qp$":389.79654088050313},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":59.1479,"latency":44.5,"recall":0.9906,"label":1,"qp$":185.99968553459118},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":40.999,"latency":55.300000000000004,"recall":1.0,"label":1,"qp$":128.92767295597486},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":579.9416,"latency":9.4,"recall":0.9213,"label":1,"qp$":1823.71572327044},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":425.2529,"latency":11.299999999999999,"recall":0.9686,"label":1,"qp$":1337.2732704402515},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":397.0539,"latency":13.799999999999999,"recall":1.0,"label":1,"qp$":1248.5971698113208},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":516.27,"latency":7.0,"recall":0.9463,"label":1,"qp$":3246.9811320754716},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":354.8416,"latency":10.0,"recall":0.9802,"label":1,"qp$":2231.708176100629},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":427.5229,"latency":8.7,"recall":1.0,"label":1,"qp$":2688.8232704402517},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":2884.689,"latency":5.3,"recall":0.8801,"label":1,"qp$":2267.837264150943},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":1689.5799,"latency":6.6,"recall":0.9493,"label":1,"qp$":1328.2860849056603},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1517.6792,"latency":10.0,"recall":1.0,"label":1,"qp$":1193.1440251572328},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":822.5318,"latency":5.6,"recall":0.9294,"label":1,"qp$":646.6444968553459},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":378.9146,"latency":10.3,"recall":0.9758,"label":1,"qp$":297.8888364779874},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":218.6854,"latency":16.2,"recall":1.0,"label":1,"qp$":171.92248427672953},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":297.5,"latency":7.2,"recall":0.974,"label":1,"qp$":1871.0691823899372},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":228.3,"latency":10.6,"recall":0.994,"label":1,"qp$":1435.8490566037735},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":584.0,"latency":4.6,"recall":1.0,"label":1,"qp$":3672.9559748427673},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":1871.0,"latency":7.0,"recall":0.9602,"label":1,"qp$":1470.9119496855346},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":556.7,"latency":6.7,"recall":0.9723,"label":1,"qp$":437.65723270440253},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1583.0,"latency":6.8,"recall":0.9836,"label":1,"qp$":1244.4968553459119},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":294.3,"latency":10.9,"recall":0.9939,"label":1,"qp$":231.3679245283019},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":2345.0,"latency":8.9,"recall":1.0,"label":1,"qp$":1843.553459119497},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":295.6,"latency":12.3,"recall":1.0,"label":1,"qp$":232.38993710691824},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":143.0,"latency":33.5,"recall":0.9818,"label":1,"qp$":899.3710691823899},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":106.0,"latency":20.7,"recall":0.9887,"label":1,"qp$":666.6666666666666},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":189.0,"latency":11.6,"recall":1.0,"label":1,"qp$":1188.6792452830189},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":379.9721,"latency":12.4,"recall":0.982,"label":1,"qp$":1194.8808176100629},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":71.74,"latency":50.8,"recall":0.9883,"label":1,"qp$":225.59748427672955},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":287.0,"latency":14.9,"recall":0.9865,"label":1,"qp$":902.5157232704403},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":34.6654,"latency":64.69999999999999,"recall":0.9961,"label":1,"qp$":109.01069182389936},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":412.0,"latency":10.3,"recall":1.0,"label":1,"qp$":1295.5974842767296},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":42.169,"latency":46.800000000000004,"recall":1.0,"label":1,"qp$":132.6069182389937},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":537.4975,"latency":18.9,"recall":0.8903,"label":1,"qp$":376.9267180925666},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":372.0466,"latency":17.8,"recall":0.8904,"label":1,"qp$":260.9022440392707},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1156.2898,"latency":14.4,"recall":0.9989,"label":1,"qp$":810.8624123422161},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":110.248,"latency":69.0,"recall":0.898,"label":1,"qp$":77.31276297335204},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":87.2601,"latency":27.799999999999997,"recall":0.898,"label":1,"qp$":61.1922159887798},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":125.7846,"latency":23.099999999999998,"recall":0.975,"label":1,"qp$":88.20799438990183},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":240.7209,"latency":17.4,"recall":0.8887,"label":1,"qp$":844.0424263674614},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":189.4399,"latency":17.5,"recall":0.8889,"label":1,"qp$":664.2352734922861},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":313.5116,"latency":16.1,"recall":0.9999,"label":1,"qp$":1099.2692847124824},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":78.7196,"latency":49.4,"recall":0.9203,"label":1,"qp$":474.21445783132526},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":68.3111,"latency":35.5,"recall":0.9202,"label":1,"qp$":411.5126506024096},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":210.2147,"latency":26.8,"recall":0.9996,"label":1,"qp$":1266.3536144578313},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":188.6436,"latency":917.5,"recall":0.9175,"label":1,"qp$":661.4431977559607},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":155.6991,"latency":917.1,"recall":0.9171,"label":1,"qp$":545.929523141655},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":445.3289,"latency":14.1,"recall":0.9999,"label":1,"qp$":1561.461781206171},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":633.6033,"latency":24.6,"recall":0.919,"label":1,"qp$":444.32208976157085},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":95.5682,"latency":58.7,"recall":0.9463,"label":1,"qp$":67.01837307152876},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":434.4062,"latency":17.4,"recall":0.9181,"label":1,"qp$":304.6326788218794},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":80.3192,"latency":26.599999999999998,"recall":0.9462,"label":1,"qp$":56.324824684431974},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":1509.3293,"latency":18.5,"recall":0.9995,"label":1,"qp$":1058.4356942496495},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":166.7252,"latency":19.599999999999998,"recall":0.9988,"label":1,"qp$":116.91809256661992},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":46.8622,"latency":123.0,"recall":0.9957,"label":1,"qp$":4.639821782178218},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1.8388,"latency":1063.5,"recall":0.9957,"label":1,"qp$":0.18205940594059405},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":45.0666,"latency":109.2,"recall":1.0,"label":1,"qp$":4.462039603960396},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":43.5017,"latency":228.7,"recall":0.9957,"label":1,"qp$":1.3344079754601226},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1.5668,"latency":1114.4,"recall":0.9957,"label":1,"qp$":0.04806134969325153},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":41.5443,"latency":159.0,"recall":1.0,"label":1,"qp$":1.2743650306748466},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":67.9121,"latency":179.5,"recall":0.9909,"label":1,"qp$":2.083193251533742},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.7636,"latency":1921.3,"recall":0.9908,"label":1,"qp$":0.02342331288343558},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":32.0,"latency":124.5,"recall":1.0,"label":1,"qp$":0.9815950920245399},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":63.1365,"latency":145.7,"recall":0.991,"label":1,"qp$":6.251138613861386},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.7512,"latency":1983.8,"recall":0.9908,"label":1,"qp$":0.07437623762376237},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":30.1358,"latency":129.8,"recall":1.0,"label":1,"qp$":2.983742574257426},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":37.432,"latency":75.0,"recall":0.9975,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":37.0696,"latency":73.5,"recall":0.9976,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":81.1915,"latency":53.0,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":321.6048,"latency":13.4,"recall":0.989,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":22.1467,"latency":86.8,"recall":0.9972,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":303.2551,"latency":10.9,"recall":0.9876,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":21.5388,"latency":81.69999999999999,"recall":0.997,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":394.5418,"latency":8.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":37.878,"latency":45.0,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":180.2757,"latency":6.0,"recall":0.9942,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":179.0033,"latency":6.4,"recall":0.9943,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":526.8846,"latency":3.6,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":626.5243,"latency":6.2,"recall":0.9954,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":78.4227,"latency":25.3,"recall":0.9982,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":599.4213,"latency":6.6,"recall":0.9955,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":78.5351,"latency":26.3,"recall":0.9982,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":2098.2113,"latency":3.4,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":275.6292,"latency":10.0,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":100.6667,"latency":21.1,"recall":0.9909,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":101.1399,"latency":19.7,"recall":0.9907,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":52.2606,"latency":18.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":61.0661,"latency":49.8,"recall":0.9911,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":58.9326,"latency":44.6,"recall":0.9911,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":42.5977,"latency":54.9,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":536.0726,"latency":8.200000000000001,"recall":0.9728,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":467.179,"latency":7.0,"recall":0.9697,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":431.7512,"latency":8.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":274.5407,"latency":4.8999999999999995,"recall":0.9807,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":236.5672,"latency":10.3,"recall":0.981,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":309.4833,"latency":4.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":178.6585,"latency":13.700000000000001,"recall":0.9843,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":178.3732,"latency":15.0,"recall":0.9844,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":229.3526,"latency":12.5,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":1258.7043,"latency":4.8999999999999995,"recall":0.9799,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":1075.8776,"latency":5.3,"recall":0.98,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1494.8493,"latency":4.7,"recall":1.0,"label":1,"qp$":0.0}] \ No newline at end of file +[{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":0.8836,"latency":2523.0,"recall":0.8528,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":0.8937,"latency":3720.2000000000003,"recall":0.8525,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":1.2145,"latency":3622.3999999999996,"recall":0.7487,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":10.6271,"latency":730.7,"recall":0.8898,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":10.8507,"latency":733.1999999999999,"recall":0.8897,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":75.7055,"latency":121.2,"recall":0.9999,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":11.2945,"latency":3611.2000000000003,"recall":0.996,"label":1,"qp$":23.564573336115167},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":17.3271,"latency":3774.7999999999997,"recall":0.9961,"label":1,"qp$":36.15084498226581},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":26.26,"latency":556.1,"recall":0.9999,"label":1,"qp$":54.78823283955769},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":15.2269,"latency":861.8,"recall":0.9888,"label":1,"qp$":31.76903818068016},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":15.1749,"latency":774.3,"recall":0.989,"label":1,"qp$":31.660546630502814},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":27.6181,"latency":305.5,"recall":0.9999,"label":1,"qp$":57.621740037554765},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":228.4,"latency":22.2,"recall":0.9348,"label":1,"qp$":1564.3835616438357},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":181.5,"latency":26.1,"recall":0.9345,"label":1,"qp$":1243.150684931507},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":205.7,"latency":24.2,"recall":0.9586,"label":1,"qp$":1408.9041095890411},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":67.63,"latency":36.0,"recall":0.8064,"label":1,"qp$":695.0668036998972},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":63.35,"latency":38.4,"recall":0.8065,"label":1,"qp$":651.0791366906475},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":176.7,"latency":27.599999999999998,"recall":1.0,"label":1,"qp$":1816.032887975334},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":15.33,"latency":84.9,"recall":0.8064,"label":1,"qp$":157.55395683453239},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":15.13,"latency":86.7,"recall":0.8065,"label":1,"qp$":155.49845837615624},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":17.41,"latency":74.30000000000001,"recall":1.0,"label":1,"qp$":178.9311408016444},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":16.34,"latency":88.7,"recall":0.879,"label":1,"qp$":83.7948717948718},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":10.45,"latency":126.8,"recall":0.8208,"label":1,"qp$":53.589743589743584},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":16.18,"latency":87.5,"recall":0.8793,"label":1,"qp$":82.97435897435896},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":9.8,"latency":130.89999999999998,"recall":0.8212,"label":1,"qp$":50.256410256410255},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":36.11,"latency":55.1,"recall":1.0,"label":1,"qp$":185.17948717948718},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":14.84,"latency":92.7,"recall":0.96,"label":1,"qp$":76.1025641025641},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":322.7,"latency":26.4,"recall":0.9478,"label":1,"qp$":276.2842465753425},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":265.5,"latency":26.9,"recall":0.9332,"label":1,"qp$":227.31164383561645},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":303.8,"latency":27.3,"recall":0.9478,"label":1,"qp$":260.10273972602744},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":180.2,"latency":28.2,"recall":0.9335,"label":1,"qp$":154.28082191780823},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":730.7,"latency":24.6,"recall":0.9586,"label":1,"qp$":625.5993150684932},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":104.3,"latency":31.7,"recall":0.9563,"label":1,"qp$":89.29794520547945},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":147.7,"latency":35.3,"recall":0.9707,"label":1,"qp$":189.60205391527597},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":782.5,"latency":25.9,"recall":1.0,"label":1,"qp$":1004.4929396662387},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":18.7634,"latency":153.70000000000002,"recall":0.8737,"label":1,"qp$":192.84069886947586},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":18.3619,"latency":79.8,"recall":0.8741,"label":1,"qp$":188.7142857142857},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":25.2744,"latency":61.199999999999996,"recall":0.9979,"label":1,"qp$":259.7574511819116},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":261.798,"latency":23.099999999999998,"recall":0.9262,"label":1,"qp$":1793.13698630137},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":166.1851,"latency":23.900000000000002,"recall":0.9264,"label":1,"qp$":1138.2541095890413},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":121.7169,"latency":29.0,"recall":0.9693,"label":1,"qp$":833.677397260274},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":46.6189,"latency":43.1,"recall":0.8737,"label":1,"qp$":479.1253854059609},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":42.4856,"latency":44.0,"recall":0.8741,"label":1,"qp$":436.6454265159301},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":138.9479,"latency":26.200000000000003,"recall":0.9979,"label":1,"qp$":1428.0359712230218},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":20.7437,"latency":75.80000000000001,"recall":0.9291,"label":1,"qp$":106.37794871794871},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":20.2993,"latency":76.5,"recall":0.9293,"label":1,"qp$":104.09897435897435},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":26.4719,"latency":67.0,"recall":1.0,"label":1,"qp$":135.75333333333333},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":8.6675,"latency":180.2,"recall":0.8369,"label":1,"qp$":44.44871794871795},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":7.8121,"latency":167.7,"recall":0.8369,"label":1,"qp$":40.06205128205128},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":16.869,"latency":87.8,"recall":0.9814,"label":1,"qp$":86.50769230769231},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":365.0835,"latency":23.599999999999998,"recall":0.945,"label":1,"qp$":312.57148972602744},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":325.5271,"latency":25.1,"recall":0.9452,"label":1,"qp$":278.7047089041096},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":596.7942,"latency":24.2,"recall":0.9693,"label":1,"qp$":510.95393835616443},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":303.2044,"latency":27.400000000000002,"recall":0.9246,"label":1,"qp$":259.5928082191781},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":136.0345,"latency":31.9,"recall":0.9244,"label":1,"qp$":116.46789383561645},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":66.7221,"latency":42.1,"recall":0.963,"label":1,"qp$":57.125085616438355},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":131.2549,"latency":30.200000000000003,"recall":0.9867,"label":1,"qp$":168.49152759948652},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":127.9337,"latency":30.099999999999998,"recall":0.9869,"label":1,"qp$":164.2281129653402},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":595.8462,"latency":23.400000000000002,"recall":1.0,"label":1,"qp$":764.8860077021822},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":330.0144,"latency":9.0,"recall":0.9507,"label":1,"qp$":2075.5622641509435},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":271.6585,"latency":10.1,"recall":0.9678,"label":1,"qp$":1708.5440251572327},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":216.5226,"latency":12.9,"recall":1.0,"label":1,"qp$":1361.777358490566},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":123.9553,"latency":23.0,"recall":0.971,"label":1,"qp$":389.79654088050313},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":59.1479,"latency":44.5,"recall":0.9906,"label":1,"qp$":185.99968553459118},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":40.999,"latency":55.300000000000004,"recall":1.0,"label":1,"qp$":128.92767295597486},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":579.9416,"latency":9.4,"recall":0.9213,"label":1,"qp$":1823.71572327044},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":425.2529,"latency":11.299999999999999,"recall":0.9686,"label":1,"qp$":1337.2732704402515},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":397.0539,"latency":13.799999999999999,"recall":1.0,"label":1,"qp$":1248.5971698113208},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":516.27,"latency":7.0,"recall":0.9463,"label":1,"qp$":3246.9811320754716},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":354.8416,"latency":10.0,"recall":0.9802,"label":1,"qp$":2231.708176100629},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":427.5229,"latency":8.7,"recall":1.0,"label":1,"qp$":2688.8232704402517},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":2884.689,"latency":5.3,"recall":0.8801,"label":1,"qp$":2267.837264150943},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":1689.5799,"latency":6.6,"recall":0.9493,"label":1,"qp$":1328.2860849056603},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1517.6792,"latency":10.0,"recall":1.0,"label":1,"qp$":1193.1440251572328},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":822.5318,"latency":5.6,"recall":0.9294,"label":1,"qp$":646.6444968553459},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":378.9146,"latency":10.3,"recall":0.9758,"label":1,"qp$":297.8888364779874},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":218.6854,"latency":16.2,"recall":1.0,"label":1,"qp$":171.92248427672953},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":297.5,"latency":7.2,"recall":0.974,"label":1,"qp$":1871.0691823899372},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":228.3,"latency":10.6,"recall":0.994,"label":1,"qp$":1435.8490566037735},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":584.0,"latency":4.6,"recall":1.0,"label":1,"qp$":3672.9559748427673},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":1871.0,"latency":7.0,"recall":0.9602,"label":1,"qp$":1470.9119496855346},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":556.7,"latency":6.7,"recall":0.9723,"label":1,"qp$":437.65723270440253},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1583.0,"latency":6.8,"recall":0.9836,"label":1,"qp$":1244.4968553459119},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":294.3,"latency":10.9,"recall":0.9939,"label":1,"qp$":231.3679245283019},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":2345.0,"latency":8.9,"recall":1.0,"label":1,"qp$":1843.553459119497},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":295.6,"latency":12.3,"recall":1.0,"label":1,"qp$":232.38993710691824},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":143.0,"latency":33.5,"recall":0.9818,"label":1,"qp$":899.3710691823899},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":106.0,"latency":20.7,"recall":0.9887,"label":1,"qp$":666.6666666666666},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":189.0,"latency":11.6,"recall":1.0,"label":1,"qp$":1188.6792452830189},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":379.9721,"latency":12.4,"recall":0.982,"label":1,"qp$":1194.8808176100629},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":71.74,"latency":50.8,"recall":0.9883,"label":1,"qp$":225.59748427672955},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":287.0,"latency":14.9,"recall":0.9865,"label":1,"qp$":902.5157232704403},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":34.6654,"latency":64.69999999999999,"recall":0.9961,"label":1,"qp$":109.01069182389936},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":412.0,"latency":10.3,"recall":1.0,"label":1,"qp$":1295.5974842767296},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":42.169,"latency":46.800000000000004,"recall":1.0,"label":1,"qp$":132.6069182389937},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":537.4975,"latency":18.9,"recall":0.8903,"label":1,"qp$":376.9267180925666},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":372.0466,"latency":17.8,"recall":0.8904,"label":1,"qp$":260.9022440392707},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1156.2898,"latency":14.4,"recall":0.9989,"label":1,"qp$":810.8624123422161},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":110.248,"latency":69.0,"recall":0.898,"label":1,"qp$":77.31276297335204},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":87.2601,"latency":27.799999999999997,"recall":0.898,"label":1,"qp$":61.1922159887798},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":125.7846,"latency":23.099999999999998,"recall":0.975,"label":1,"qp$":88.20799438990183},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":240.7209,"latency":17.4,"recall":0.8887,"label":1,"qp$":844.0424263674614},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":189.4399,"latency":17.5,"recall":0.8889,"label":1,"qp$":664.2352734922861},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":313.5116,"latency":16.1,"recall":0.9999,"label":1,"qp$":1099.2692847124824},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":78.7196,"latency":49.4,"recall":0.9203,"label":1,"qp$":474.21445783132526},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":68.3111,"latency":35.5,"recall":0.9202,"label":1,"qp$":411.5126506024096},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":210.2147,"latency":26.8,"recall":0.9996,"label":1,"qp$":1266.3536144578313},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":188.6436,"latency":917.5,"recall":0.9175,"label":1,"qp$":661.4431977559607},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":155.6991,"latency":917.1,"recall":0.9171,"label":1,"qp$":545.929523141655},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":445.3289,"latency":14.1,"recall":0.9999,"label":1,"qp$":1561.461781206171},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":633.6033,"latency":24.6,"recall":0.919,"label":1,"qp$":444.32208976157085},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":95.5682,"latency":58.7,"recall":0.9463,"label":1,"qp$":67.01837307152876},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":434.4062,"latency":17.4,"recall":0.9181,"label":1,"qp$":304.6326788218794},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":80.3192,"latency":26.599999999999998,"recall":0.9462,"label":1,"qp$":56.324824684431974},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":1509.3293,"latency":18.5,"recall":0.9995,"label":1,"qp$":1058.4356942496495},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":166.7252,"latency":19.599999999999998,"recall":0.9988,"label":1,"qp$":116.91809256661992},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":46.8622,"latency":123.0,"recall":0.9957,"label":1,"qp$":4.639821782178218},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1.8388,"latency":1063.5,"recall":0.9957,"label":1,"qp$":0.18205940594059405},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":45.0666,"latency":109.2,"recall":1.0,"label":1,"qp$":4.462039603960396},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":43.5017,"latency":228.7,"recall":0.9957,"label":1,"qp$":1.3344079754601226},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1.5668,"latency":1114.4,"recall":0.9957,"label":1,"qp$":0.04806134969325153},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":41.5443,"latency":159.0,"recall":1.0,"label":1,"qp$":1.2743650306748466},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":67.9121,"latency":179.5,"recall":0.9909,"label":1,"qp$":2.083193251533742},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.7636,"latency":1921.3,"recall":0.9908,"label":1,"qp$":0.02342331288343558},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":32.0,"latency":124.5,"recall":1.0,"label":1,"qp$":0.9815950920245399},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":63.1365,"latency":145.7,"recall":0.991,"label":1,"qp$":6.251138613861386},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.7512,"latency":1983.8,"recall":0.9908,"label":1,"qp$":0.07437623762376237},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":30.1358,"latency":129.8,"recall":1.0,"label":1,"qp$":2.983742574257426},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":37.432,"latency":75.0,"recall":0.9975,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":37.0696,"latency":73.5,"recall":0.9976,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":81.1915,"latency":53.0,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":321.6048,"latency":13.4,"recall":0.989,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":22.1467,"latency":86.8,"recall":0.9972,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":303.2551,"latency":10.9,"recall":0.9876,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":21.5388,"latency":81.69999999999999,"recall":0.997,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":394.5418,"latency":8.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":37.878,"latency":45.0,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":180.2757,"latency":6.0,"recall":0.9942,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":179.0033,"latency":6.4,"recall":0.9943,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":526.8846,"latency":3.6,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":626.5243,"latency":6.2,"recall":0.9954,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":78.4227,"latency":25.3,"recall":0.9982,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":599.4213,"latency":6.6,"recall":0.9955,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":78.5351,"latency":26.3,"recall":0.9982,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":2098.2113,"latency":3.4,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":275.6292,"latency":10.0,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":100.6667,"latency":21.1,"recall":0.9909,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":101.1399,"latency":19.7,"recall":0.9907,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":52.2606,"latency":18.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":61.0661,"latency":49.8,"recall":0.9911,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":58.9326,"latency":44.6,"recall":0.9911,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":42.5977,"latency":54.9,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":536.0726,"latency":8.200000000000001,"recall":0.9728,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":467.179,"latency":7.0,"recall":0.9697,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":431.7512,"latency":8.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":274.5407,"latency":4.8999999999999995,"recall":0.9807,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":236.5672,"latency":10.3,"recall":0.981,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":309.4833,"latency":4.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":178.6585,"latency":13.700000000000001,"recall":0.9843,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":178.3732,"latency":15.0,"recall":0.9844,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":229.3526,"latency":12.5,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":1258.7043,"latency":4.8999999999999995,"recall":0.9799,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":1075.8776,"latency":5.3,"recall":0.98,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1494.8493,"latency":4.7,"recall":1.0,"label":1,"qp$":0.0}] \ No newline at end of file From 1a82468ad6bd32208316932ada968e8794e90874 Mon Sep 17 00:00:00 2001 From: yangxuan Date: Fri, 18 Aug 2023 18:39:02 +0800 Subject: [PATCH 018/327] Wait for index done Signed-off-by: yangxuan --- .../backend/clients/milvus/milvus.py | 24 +++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/vectordb_bench/backend/clients/milvus/milvus.py b/vectordb_bench/backend/clients/milvus/milvus.py index ae494f885..e5531eb7c 100644 --- a/vectordb_bench/backend/clients/milvus/milvus.py +++ b/vectordb_bench/backend/clients/milvus/milvus.py @@ -1,6 +1,7 @@ """Wrapper around the Milvus vector database over VectorDB""" import logging +import time from contextlib import contextmanager from typing import Iterable, Type @@ -54,12 +55,17 @@ def __init__( log.info(f"{self.name} create collection: {self.collection_name}") # Create the collection - Collection( + col = Collection( name=self.collection_name, schema=CollectionSchema(fields), consistency_level="Session", ) + col.create_index( + self._vector_field, + self.case_config.index_param(), + index_name=self._index_name, + ) # self._pre_load(coll) connections.disconnect("default") @@ -118,16 +124,26 @@ def _post_insert(self): log.info(f"{self.name} post insert before optimize") try: self.col.flush() - self.col.compact() - self.col.wait_for_compaction_completed() - # wait for index done and load refresh self.col.create_index( self._vector_field, self.case_config.index_param(), index_name=self._index_name, ) + utility.wait_for_index_building_complete(self.collection_name) + def wait_index(): + while True: + progress = utility.index_building_progress(self.collection_name) + if progress.get("pending_index_rows", -1) == 0: + break + time.sleep(5) + + wait_index() + self.col.compact() + self.col.wait_for_compaction_completed() + wait_index() + except Exception as e: log.warning(f"{self.name} optimize error: {e}") raise e from None From ed1866ab1a9d6d3333db5f9f99fb73902b4d96fc Mon Sep 17 00:00:00 2001 From: yangxuan Date: Tue, 10 Oct 2023 15:07:00 +0800 Subject: [PATCH 019/327] Import database client while in use As we support more and more databases, vectordb-bench depends on more and more database clients, which causes some dependency problems. Especially when we only cares for one or two databases. This PR removes the dependency of all database clients. We are still able to browse the standard results with no database clients installed. And we can install the database clients when we need to do benchmarks on the sepcific database. 1. The defatul vectordb-bench now only contains PyMilvus pip install vectordb-bench # pymilvus only 2. We can still install all database clients pip install vectordb-bench[all] 3. Or some other database client pip install vectordb-bench[pinecone] Signed-off-by: yangxuan --- README.md | 64 +++++++-- pyproject.toml | 40 ++++-- tests/test_chroma.py | 4 +- tests/test_elasticsearch_cloud.py | 4 +- vectordb_bench/backend/assembler.py | 4 + vectordb_bench/backend/clients/__init__.py | 129 ++++++++++++++---- vectordb_bench/backend/clients/api.py | 16 +-- .../backend/clients/chroma/chroma.py | 16 +-- .../clients/elastic_cloud/elastic_cloud.py | 17 +-- .../backend/clients/milvus/milvus.py | 13 +- .../backend/clients/pgvector/pgvector.py | 18 +-- .../backend/clients/pinecone/config.py | 2 +- .../backend/clients/qdrant_cloud/config.py | 13 +- .../clients/qdrant_cloud/qdrant_cloud.py | 12 +- vectordb_bench/backend/clients/redis/redis.py | 9 +- .../backend/clients/weaviate_cloud/config.py | 3 +- .../clients/weaviate_cloud/weaviate_cloud.py | 16 +-- .../clients/zilliz_cloud/zilliz_cloud.py | 15 +- vectordb_bench/backend/task_runner.py | 9 +- .../components/run_test/dbConfigSetting.py | 5 +- .../components/run_test/generateTasks.py | 2 +- vectordb_bench/interface.py | 13 +- vectordb_bench/models.py | 5 +- 23 files changed, 244 insertions(+), 185 deletions(-) diff --git a/README.md b/README.md index cfaa86215..1fc4f0934 100644 --- a/README.md +++ b/README.md @@ -10,9 +10,35 @@ python >= 3.11 ``` ### Install -``` shell +**Install vectordb-bench with only PyMilvus** +```shell pip install vectordb-bench ``` + +**Install all database clients** + +``` shell +pip install vectordb-bench[all] +``` +**Install the specific database client** + +```shell +pip install vectordb-bench[pinecone] +``` +All the database client supported + +|Optional database client|install command| +|---------------|---------------| +|pymilvus(*default*)|`pip install vectordb-bench`| +|all|`pip install vectordb-bench[all]`| +|qdrant|`pip install vectordb-bench[qdrant]`| +|pinecone|`pip install vectordb-bench[pinecone]`| +|weaviate|`pip install vectordb-bench[weaviate]`| +|elastic|`pip install vectordb-bench[elastic]`| +|pgvector|`pip install vectordb-bench[pgvector]`| +|redis|`pip install vectordb-bench[redis]`| +|chromadb|`pip install vectordb-bench[chromadb]`| + ### Run ``` shell @@ -50,6 +76,8 @@ To facilitate the presentation of test results and provide a comprehensive perfo ### Install requirements ``` shell pip install -e '.[test]' + +pip install -e '.[pinecone]' ``` ### Run test server ``` @@ -185,27 +213,41 @@ In this final step, you will import your DB client into clients/__init__.py and 2. Add your NewClient to the DB enum. 3. Update the db2client dictionary by adding an entry for your NewClient. Example implementation in clients/__init__.py: + ```python #clients/__init__.py -from .new_client.new_client import NewClient - -#Add NewClient to the DB enum +# Add NewClient to the DB enum class DB(Enum): ... DB.NewClient = "NewClient" -#Add NewClient to the db2client dictionary -db2client = { - DB.Milvus: Milvus, - ... - DB.NewClient: NewClient -} + @property + def init_cls(self) -> Type[VectorDB]: + ... + if self == DB.NewClient: + from .new_client.new_client import NewClient + return NewClient + ... + + @property + def config_cls(self) -> Type[DBConfig]: + ... + if self == DB.NewClient: + from .new_client.config import NewClientConfig + return NewClientConfig + ... + + def case_config_cls(self, ...) + if self == DB.NewClient: + from .new_client.config import NewClientCaseConfig + return NewClientCaseConfig + ``` That's it! You have successfully added a new DB client to the vectordb_bench project. ## Rules -### Installation +### Installation The system under test can be installed in any form to achieve optimal performance. This includes but is not limited to binary deployment, Docker, and cloud services. ### Fine-Tuning For the system under test, we use the default server-side configuration to maintain the authenticity and representativeness of our results. diff --git a/pyproject.toml b/pyproject.toml index 7ac8855a0..28200f3de 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,32 +24,44 @@ dependencies = [ "streamlit-autorefresh", "streamlit>=1.23.0", "streamlit_extras", + "tqdm", + "s3fs", + "psutil", + "polars", + "plotly", + "environs", + "pydantic db2runner[db] = [] db2runner[db].append(r) + # check dbclient installed + for k in db2runner.keys(): + _ = k.init_cls + # sort by dataset size for k in db2runner.keys(): db2runner[k].sort(key=lambda x:x.ca.dataset.data.size) diff --git a/vectordb_bench/backend/clients/__init__.py b/vectordb_bench/backend/clients/__init__.py index 77cf063ea..63cc607e8 100644 --- a/vectordb_bench/backend/clients/__init__.py +++ b/vectordb_bench/backend/clients/__init__.py @@ -9,15 +9,6 @@ MetricType, ) -from .milvus.milvus import Milvus -from .elastic_cloud.elastic_cloud import ElasticCloud -from .pinecone.pinecone import Pinecone -from .weaviate_cloud.weaviate_cloud import WeaviateCloud -from .qdrant_cloud.qdrant_cloud import QdrantCloud -from .zilliz_cloud.zilliz_cloud import ZillizCloud -from .pgvector.pgvector import PgVector -from .redis.redis import Redis -from .chroma.chroma import ChromaClient class DB(Enum): """Database types @@ -44,23 +35,109 @@ class DB(Enum): @property def init_cls(self) -> Type[VectorDB]: - return db2client.get(self) - - -db2client = { - DB.Milvus: Milvus, - DB.ZillizCloud: ZillizCloud, - DB.WeaviateCloud: WeaviateCloud, - DB.ElasticCloud: ElasticCloud, - DB.QdrantCloud: QdrantCloud, - DB.Pinecone: Pinecone, - DB.PgVector: PgVector, - DB.Redis: Redis, - DB.Chroma: ChromaClient -} - -for db in DB: - assert issubclass(db.init_cls, VectorDB) + """Import while in use""" + if self == DB.Milvus: + from .milvus.milvus import Milvus + return Milvus + + if self == DB.ZillizCloud: + from .zilliz_cloud.zilliz_cloud import ZillizCloud + return ZillizCloud + + if self == DB.Pinecone: + from .pinecone.pinecone import Pinecone + return Pinecone + + if self == DB.ElasticCloud: + from .elastic_cloud.elastic_cloud import ElasticCloud + return ElasticCloud + + if self == DB.QdrantCloud: + from .qdrant_cloud.qdrant_cloud import QdrantClient + return QdrantClient + + if self == DB.WeaviateCloud: + from .weaviate_cloud.weaviate_cloud import WeaviateCloud + return WeaviateCloud + + if self == DB.PgVector: + from .pgvector.pgvector import PgVector + return PgVector + + if self == DB.Redis: + from .redis.redis import Redis + return Redis + + if self == DB.Chroma: + from .chroma.chroma import ChromaClient + return ChromaClient + + @property + def config_cls(self) -> Type[DBConfig]: + """Import while in use""" + if self == DB.Milvus: + from .milvus.config import MilvusConfig + return MilvusConfig + + if self == DB.ZillizCloud: + from .zilliz_cloud.config import ZillizCloudConfig + return ZillizCloudConfig + + if self == DB.Pinecone: + from .pinecone.config import PineconeConfig + return PineconeConfig + + if self == DB.ElasticCloud: + from .elastic_cloud.config import ElasticCloudConfig + return ElasticCloudConfig + + if self == DB.QdrantCloud: + from .qdrant_cloud.config import QdrantConfig + return QdrantConfig + + if self == DB.WeaviateCloud: + from .weaviate_cloud.config import WeaviateConfig + return WeaviateConfig + + if self == DB.PgVector: + from .pgvector.config import PgVectorConfig + return PgVectorConfig + + if self == DB.Redis: + from .redis.config import RedisConfig + return RedisConfig + + if self == DB.Chroma: + from .chroma.config import ChromaConfig + return ChromaConfig + + def case_config_cls(self, index_type: IndexType | None = None) -> Type[DBCaseConfig]: + if self == DB.Milvus: + from .milvus.config import _milvus_case_config + return _milvus_case_config.get(index_type) + + if self == DB.ZillizCloud: + from .zilliz_cloud.config import AutoIndexConfig + return AutoIndexConfig + + if self == DB.ElasticCloud: + from .elastic_cloud.config import ElasticCloudIndexConfig + return ElasticCloudIndexConfig + + if self == DB.QdrantCloud: + from .qdrant_cloud.config import QdrantIndexConfig + return QdrantIndexConfig + + if self == DB.WeaviateCloud: + from .weaviate_cloud.config import WeaviateIndexConfig + return WeaviateIndexConfig + + if self == DB.PgVector: + from .pgvector.config import PgVectorIndexConfig + return PgVectorIndexConfig + + # DB.Pinecone, DB.Chroma, DB.Redis + return EmptyDBCaseConfig __all__ = [ diff --git a/vectordb_bench/backend/clients/api.py b/vectordb_bench/backend/clients/api.py index 4d28d55e9..7cc311722 100644 --- a/vectordb_bench/backend/clients/api.py +++ b/vectordb_bench/backend/clients/api.py @@ -105,18 +105,6 @@ def __init__( """ raise NotImplementedError - @classmethod - @abstractmethod - def config_cls(self) -> Type[DBConfig]: - raise NotImplementedError - - - @classmethod - @abstractmethod - def case_config_cls(self, index_type: IndexType | None = None) -> Type[DBCaseConfig]: - raise NotImplementedError - - @abstractmethod @contextmanager def init(self) -> None: @@ -128,6 +116,10 @@ def init(self) -> None: """ raise NotImplementedError + def need_normalize_cosine(self) -> bool: + """Wheather this database need to normalize dataset to support COSINE""" + return False + @abstractmethod def insert_embeddings( self, diff --git a/vectordb_bench/backend/clients/chroma/chroma.py b/vectordb_bench/backend/clients/chroma/chroma.py index a87ea5718..8325909c0 100644 --- a/vectordb_bench/backend/clients/chroma/chroma.py +++ b/vectordb_bench/backend/clients/chroma/chroma.py @@ -1,11 +1,8 @@ import chromadb import logging -import numpy as np from contextlib import contextmanager -from typing import Any, Type -from ..api import VectorDB, DBConfig, DBCaseConfig, EmptyDBCaseConfig, IndexType -from .config import ChromaConfig -from chromadb.config import Settings +from typing import Any +from ..api import VectorDB, DBCaseConfig log = logging.getLogger(__name__) class ChromaClient(VectorDB): @@ -40,13 +37,6 @@ def __init__( drop_old = False log.info(f"Chroma client drop_old collection: {self.collection_name}") - - def config_cls() -> Type[DBConfig]: - return ChromaConfig - - def case_config_cls(index_type: IndexType | None = None) -> Type[DBCaseConfig]: - return EmptyDBCaseConfig - @contextmanager def init(self) -> None: """ create and destory connections to database. @@ -135,4 +125,4 @@ def search_embedding( return [int(i) for i in results.get('ids')[0]] results = self.collection.query(query_embeddings=query, n_results=k) return [int(i) for i in results.get('ids')[0]] - \ No newline at end of file + diff --git a/vectordb_bench/backend/clients/elastic_cloud/elastic_cloud.py b/vectordb_bench/backend/clients/elastic_cloud/elastic_cloud.py index 8fd33de40..64f27e490 100644 --- a/vectordb_bench/backend/clients/elastic_cloud/elastic_cloud.py +++ b/vectordb_bench/backend/clients/elastic_cloud/elastic_cloud.py @@ -1,9 +1,9 @@ import logging import time from contextlib import contextmanager -from typing import Iterable, Type -from ..api import VectorDB, DBCaseConfig, DBConfig, IndexType -from .config import ElasticCloudIndexConfig, ElasticCloudConfig +from typing import Iterable +from ..api import VectorDB +from .config import ElasticCloudIndexConfig from elasticsearch.helpers import bulk @@ -42,17 +42,6 @@ def __init__( client.indices.delete(index=self.indice) self._create_indice(client) - - @classmethod - def config_cls(cls) -> Type[DBConfig]: - return ElasticCloudConfig - - - @classmethod - def case_config_cls(cls, index_type: IndexType | None = None) -> Type[DBCaseConfig]: - return ElasticCloudIndexConfig - - @contextmanager def init(self) -> None: """connect to elasticsearch""" diff --git a/vectordb_bench/backend/clients/milvus/milvus.py b/vectordb_bench/backend/clients/milvus/milvus.py index e5531eb7c..356935467 100644 --- a/vectordb_bench/backend/clients/milvus/milvus.py +++ b/vectordb_bench/backend/clients/milvus/milvus.py @@ -70,15 +70,6 @@ def __init__( connections.disconnect("default") - @classmethod - def config_cls(cls) -> Type[DBConfig]: - return MilvusConfig - - @classmethod - def case_config_cls(cls, index_type: IndexType | None = None) -> Type[DBCaseConfig]: - return _milvus_case_config.get(index_type) - - @contextmanager def init(self) -> None: """ @@ -157,6 +148,10 @@ def optimize(self): assert self.col, "Please call self.init() before" self._optimize() + def need_normalize_cosine(self) -> bool: + """Wheather this database need to normalize dataset to support COSINE""" + return True + def insert_embeddings( self, embeddings: Iterable[list[float]], diff --git a/vectordb_bench/backend/clients/pgvector/pgvector.py b/vectordb_bench/backend/clients/pgvector/pgvector.py index 53f1000ab..7e634a8f5 100644 --- a/vectordb_bench/backend/clients/pgvector/pgvector.py +++ b/vectordb_bench/backend/clients/pgvector/pgvector.py @@ -1,14 +1,11 @@ """Wrapper around the Pgvector vector database over VectorDB""" import logging -import time from contextlib import contextmanager -from typing import Any, Type -from functools import wraps +from typing import Any -from ..api import VectorDB, DBConfig, DBCaseConfig, IndexType +from ..api import VectorDB, DBCaseConfig from pgvector.sqlalchemy import Vector -from .config import PgVectorConfig, PgVectorIndexConfig from sqlalchemy import ( MetaData, create_engine, @@ -67,15 +64,6 @@ def __init__( pq_metadata.drop_all(pg_engine) self._create_table(dim, pg_engine) - - @classmethod - def config_cls(cls) -> Type[DBConfig]: - return PgVectorConfig - - @classmethod - def case_config_cls(cls, index_type: IndexType | None = None) -> Type[DBCaseConfig]: - return PgVectorIndexConfig - @contextmanager def init(self) -> None: """ @@ -168,4 +156,4 @@ def search_embedding( else: res = self.pg_session.scalars(select(self.pg_table).order_by(op_fun(query)).limit(k)) return list(res) - \ No newline at end of file + diff --git a/vectordb_bench/backend/clients/pinecone/config.py b/vectordb_bench/backend/clients/pinecone/config.py index a2ac06da9..dc1596379 100644 --- a/vectordb_bench/backend/clients/pinecone/config.py +++ b/vectordb_bench/backend/clients/pinecone/config.py @@ -1,4 +1,4 @@ -from pydantic import BaseModel, SecretStr +from pydantic import SecretStr from ..api import DBConfig diff --git a/vectordb_bench/backend/clients/qdrant_cloud/config.py b/vectordb_bench/backend/clients/qdrant_cloud/config.py index ed3f556e1..a2c8d1a86 100644 --- a/vectordb_bench/backend/clients/qdrant_cloud/config.py +++ b/vectordb_bench/backend/clients/qdrant_cloud/config.py @@ -1,7 +1,6 @@ from pydantic import BaseModel, SecretStr from ..api import DBConfig, DBCaseConfig, MetricType -from qdrant_client.models import Distance class QdrantConfig(DBConfig): @@ -20,14 +19,16 @@ class QdrantIndexConfig(BaseModel, DBCaseConfig): def parse_metric(self) -> str: if self.metric_type == MetricType.L2: - return Distance.EUCLID - elif self.metric_type == MetricType.IP: - return Distance.DOT - return Distance.COSINE + return "Euclid" + + if self.metric_type == MetricType.IP: + return "Dot" + + return "Cosine" def index_param(self) -> dict: params = {"distance": self.parse_metric()} return params def search_param(self) -> dict: - return {} \ No newline at end of file + return {} diff --git a/vectordb_bench/backend/clients/qdrant_cloud/qdrant_cloud.py b/vectordb_bench/backend/clients/qdrant_cloud/qdrant_cloud.py index e3649b3b0..c4619e657 100644 --- a/vectordb_bench/backend/clients/qdrant_cloud/qdrant_cloud.py +++ b/vectordb_bench/backend/clients/qdrant_cloud/qdrant_cloud.py @@ -3,10 +3,8 @@ import logging import time from contextlib import contextmanager -from typing import Type -from ..api import VectorDB, DBConfig, DBCaseConfig, IndexType -from .config import QdrantConfig, QdrantIndexConfig +from ..api import VectorDB, DBCaseConfig from qdrant_client.http.models import ( CollectionStatus, VectorParams, @@ -49,14 +47,6 @@ def __init__( self._create_collection(dim, tmp_client) tmp_client = None - @classmethod - def config_cls(cls) -> Type[DBConfig]: - return QdrantConfig - - @classmethod - def case_config_cls(cls, index_type: IndexType | None = None) -> Type[DBCaseConfig]: - return QdrantIndexConfig - @contextmanager def init(self) -> None: """ diff --git a/vectordb_bench/backend/clients/redis/redis.py b/vectordb_bench/backend/clients/redis/redis.py index a4eb33848..e3cf8890c 100644 --- a/vectordb_bench/backend/clients/redis/redis.py +++ b/vectordb_bench/backend/clients/redis/redis.py @@ -67,13 +67,6 @@ def make_index(self, vector_dimensions: int, conn: redis.Redis): rs = conn.ft(INDEX_NAME) rs.create_index(schema, definition=definition) - - def config_cls() -> Type[DBConfig]: - return RedisConfig - - def case_config_cls(index_type: IndexType | None = None) -> Type[DBCaseConfig]: - return EmptyDBCaseConfig - @contextmanager def init(self) -> None: """ create and destory connections to database. @@ -156,4 +149,4 @@ def search_embedding( return [int(doc["id"]) for doc in res.docs] - \ No newline at end of file + diff --git a/vectordb_bench/backend/clients/weaviate_cloud/config.py b/vectordb_bench/backend/clients/weaviate_cloud/config.py index ada113643..8b2f3ab81 100644 --- a/vectordb_bench/backend/clients/weaviate_cloud/config.py +++ b/vectordb_bench/backend/clients/weaviate_cloud/config.py @@ -1,5 +1,4 @@ from pydantic import BaseModel, SecretStr -import weaviate from ..api import DBConfig, DBCaseConfig, MetricType @@ -11,7 +10,7 @@ class WeaviateConfig(DBConfig): def to_dict(self) -> dict: return { "url": self.url.get_secret_value(), - "auth_client_secret": weaviate.AuthApiKey(api_key=self.api_key.get_secret_value()), + "auth_client_secret": self.api_key.get_secret_value(), } diff --git a/vectordb_bench/backend/clients/weaviate_cloud/weaviate_cloud.py b/vectordb_bench/backend/clients/weaviate_cloud/weaviate_cloud.py index a79df6e65..4c8bd12da 100644 --- a/vectordb_bench/backend/clients/weaviate_cloud/weaviate_cloud.py +++ b/vectordb_bench/backend/clients/weaviate_cloud/weaviate_cloud.py @@ -1,14 +1,13 @@ """Wrapper around the Weaviate vector database over VectorDB""" import logging -from typing import Iterable, Type +from typing import Iterable from contextlib import contextmanager +import weaviate from weaviate.exceptions import WeaviateBaseError -from ..api import VectorDB, DBConfig, DBCaseConfig, IndexType -from .config import WeaviateConfig, WeaviateIndexConfig - +from ..api import VectorDB, DBCaseConfig log = logging.getLogger(__name__) @@ -24,6 +23,7 @@ def __init__( **kwargs, ): """Initialize wrapper around the weaviate vector database.""" + db_config.update("auth_client_secret", weaviate.AuthApiKey(api_key=db_config.get("auth_client_secret"))) self.db_config = db_config self.case_config = db_case_config self.collection_name = collection_name @@ -45,14 +45,6 @@ def __init__( self._create_collection(client) client = None - @classmethod - def config_cls(cls) -> Type[DBConfig]: - return WeaviateConfig - - @classmethod - def case_config_cls(cls, index_type: IndexType | None = None) -> Type[DBCaseConfig]: - return WeaviateIndexConfig - @contextmanager def init(self) -> None: """ diff --git a/vectordb_bench/backend/clients/zilliz_cloud/zilliz_cloud.py b/vectordb_bench/backend/clients/zilliz_cloud/zilliz_cloud.py index 297138ff8..36f7fb204 100644 --- a/vectordb_bench/backend/clients/zilliz_cloud/zilliz_cloud.py +++ b/vectordb_bench/backend/clients/zilliz_cloud/zilliz_cloud.py @@ -1,9 +1,7 @@ """Wrapper around the ZillizCloud vector database over VectorDB""" -from typing import Type from ..milvus.milvus import Milvus -from ..api import DBConfig, DBCaseConfig, IndexType -from .config import ZillizCloudConfig, AutoIndexConfig +from ..api import DBCaseConfig class ZillizCloud(Milvus): @@ -15,7 +13,7 @@ def __init__( collection_name: str = "ZillizCloudVectorDBBench", drop_old: bool = False, name: str = "ZillizCloud", - **kwargs, + **kwargs, ): super().__init__( dim=dim, @@ -26,12 +24,3 @@ def __init__( name=name, **kwargs, ) - - @classmethod - def config_cls(cls) -> Type[DBConfig]: - return ZillizCloudConfig - - - @classmethod - def case_config_cls(cls, index_type: IndexType | None = None) -> Type[DBCaseConfig]: - return AutoIndexConfig diff --git a/vectordb_bench/backend/task_runner.py b/vectordb_bench/backend/task_runner.py index 21ca9d28f..46265297a 100644 --- a/vectordb_bench/backend/task_runner.py +++ b/vectordb_bench/backend/task_runner.py @@ -12,8 +12,6 @@ from .clients import ( api, - ZillizCloud, - Milvus, MetricType ) from ..metric import Metric @@ -68,7 +66,7 @@ def display(self) -> dict: @property def normalize(self) -> bool: assert self.db - return isinstance(self.db, (Milvus, ZillizCloud)) and \ + return self.db.need_normalize_cosine() and \ self.ca.dataset.data.metric_type == MetricType.COSINE def init_db(self, drop_old: bool = True) -> None: @@ -83,8 +81,11 @@ def init_db(self, drop_old: bool = True) -> None: def _pre_run(self, drop_old: bool = True): try: - self.ca.dataset.prepare() self.init_db(drop_old) + self.ca.dataset.prepare() + except ModuleNotFoundError as e: + log.warning(f"pre run case error: please install client for db: {self.config.db}, error={e}") + raise e from None except Exception as e: log.warning(f"pre run case error: {e}") raise e from None diff --git a/vectordb_bench/frontend/components/run_test/dbConfigSetting.py b/vectordb_bench/frontend/components/run_test/dbConfigSetting.py index c5bc9d1fd..ffd52721f 100644 --- a/vectordb_bench/frontend/components/run_test/dbConfigSetting.py +++ b/vectordb_bench/frontend/components/run_test/dbConfigSetting.py @@ -16,7 +16,7 @@ def dbConfigSettings(st, activedDbList): dbConfigSettingItemContainer = expander.container() dbConfig = dbConfigSettingItem(dbConfigSettingItemContainer, activeDb) try: - dbConfigs[activeDb] = activeDb.init_cls.config_cls()(**dbConfig) + dbConfigs[activeDb] = activeDb.config_cls(**dbConfig) except ValidationError as e: isAllValid = False errTexts = [] @@ -38,8 +38,7 @@ def dbConfigSettingItem(st, activeDb): ) columns = st.columns(DB_CONFIG_SETTING_COLUMNS) - activeDbCls = activeDb.init_cls - dbConfigClass = activeDbCls.config_cls() + dbConfigClass = activeDb.config_cls properties = dbConfigClass.schema().get("properties") propertiesItems = list(properties.items()) moveDBLabelToLast(propertiesItems) diff --git a/vectordb_bench/frontend/components/run_test/generateTasks.py b/vectordb_bench/frontend/components/run_test/generateTasks.py index 0c847d2cc..55f3c8399 100644 --- a/vectordb_bench/frontend/components/run_test/generateTasks.py +++ b/vectordb_bench/frontend/components/run_test/generateTasks.py @@ -12,7 +12,7 @@ def generate_tasks(activedDbList, dbConfigs, activedCaseList, allCaseConfigs): case_id=case.value, custom_case={}, ), - db_case_config=db.init_cls.case_config_cls( + db_case_config=db.case_config_cls( allCaseConfigs[db][case].get(CaseConfigParamType.IndexType, None) )(**{key.value: value for key, value in allCaseConfigs[db][case].items()}), ) diff --git a/vectordb_bench/interface.py b/vectordb_bench/interface.py index 737912d15..360fbc64f 100644 --- a/vectordb_bench/interface.py +++ b/vectordb_bench/interface.py @@ -41,6 +41,7 @@ def __init__(self): def run(self, tasks: list[TaskConfig], task_label: str | None = None) -> bool: """run all the tasks in the configs, write one result into the path""" + self.latest_error = "" if self.running_task is not None: log.warning("There're still tasks running in the background") return False @@ -58,8 +59,15 @@ def run(self, tasks: list[TaskConfig], task_label: str | None = None) -> bool: self.receive_conn, send_conn = mp.Pipe() self.latest_error = "" - self.running_task = Assembler.assemble_all(run_id, task_label, tasks) - self.running_task.display() + + try: + self.running_task = Assembler.assemble_all(run_id, task_label, tasks) + self.running_task.display() + except ModuleNotFoundError as e: + msg = f"Please install client for database, error={e}" + log.warning(msg) + self.latest_error = msg + return True return self._run_async(send_conn) @@ -101,7 +109,6 @@ def get_tasks_count(self) -> int: return self.running_task.num_cases() return 0 - def get_current_task_id(self) -> int: """ the index of current running task return -1 if not running diff --git a/vectordb_bench/models.py b/vectordb_bench/models.py index 28b34916c..153842524 100644 --- a/vectordb_bench/models.py +++ b/vectordb_bench/models.py @@ -144,12 +144,11 @@ def read_file(cls, full_path: pathlib.Path, trans_unit: bool = False) -> Self: for case_result in test_result["results"]: task_config = case_result.get("task_config") db = DB(task_config.get("db")) - dbcls = db.init_cls - task_config["db_config"] = dbcls.config_cls()( + task_config["db_config"] = db.config_cls( **task_config["db_config"] ) - task_config["db_case_config"] = dbcls.case_config_cls( + task_config["db_case_config"] = db.case_config_cls( index_type=task_config["db_case_config"].get("index", None), )(**task_config["db_case_config"]) From 9fa36920e3a5ba0f0564d4ec89b55f13ba2f9997 Mon Sep 17 00:00:00 2001 From: "min.tian" Date: Fri, 20 Oct 2023 16:56:17 +0800 Subject: [PATCH 020/327] fix_spelling Signed-off-by: min.tian --- vectordb_bench/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vectordb_bench/models.py b/vectordb_bench/models.py index 153842524..4e32181b4 100644 --- a/vectordb_bench/models.py +++ b/vectordb_bench/models.py @@ -207,8 +207,8 @@ def append_return(x, y): TITLE = DATA_FORMAT % ( "DB", "db_label", "case", "label", "load_dur", "qps", "latency(p99)", "recall", "max_load_count", "label") SPLIT = DATA_FORMAT%tuple(map(lambda x:"-"*x, LENGTH)) - SUMMERY_FORMAT = ("Task summery: run_id=%s, task_label=%s") % (self.run_id[:5], self.task_label) - fmt = [SUMMERY_FORMAT, TITLE, SPLIT] + SUMMARY_FORMAT = ("Task summary: run_id=%s, task_label=%s") % (self.run_id[:5], self.task_label) + fmt = [SUMMARY_FORMAT, TITLE, SPLIT] for f in filtered_results: From 4a09d38ccfe77113bf555bc1dac0acd17ecec24e Mon Sep 17 00:00:00 2001 From: "min.tian" Date: Mon, 23 Oct 2023 17:17:24 +0800 Subject: [PATCH 021/327] fix qp_dollar calculation Signed-off-by: min.tian --- vectordb_bench/results/getLeaderboardData.py | 2 +- vectordb_bench/results/leaderboard.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/vectordb_bench/results/getLeaderboardData.py b/vectordb_bench/results/getLeaderboardData.py index 645e75593..9eac371a7 100644 --- a/vectordb_bench/results/getLeaderboardData.py +++ b/vectordb_bench/results/getLeaderboardData.py @@ -43,7 +43,7 @@ def main(): db_label = d["db_label"] qps = d["qps"] price = DB_DBLABEL_TO_PRICE.get(db, {}).get(db_label, 0) - d["qp$"] = qps / price if price > 0 else 0.0 + d["qp$"] = (qps / price * 3600) if price > 0 else 0.0 with open(pathlib.Path(config.RESULTS_LOCAL_DIR, "leaderboard.json"), "w") as f: ujson.dump(data, f) diff --git a/vectordb_bench/results/leaderboard.json b/vectordb_bench/results/leaderboard.json index a09e4ba1b..1fa4a495a 100644 --- a/vectordb_bench/results/leaderboard.json +++ b/vectordb_bench/results/leaderboard.json @@ -1 +1 @@ -[{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":0.8836,"latency":2523.0,"recall":0.8528,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":0.8937,"latency":3720.2000000000003,"recall":0.8525,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":1.2145,"latency":3622.3999999999996,"recall":0.7487,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":10.6271,"latency":730.7,"recall":0.8898,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":10.8507,"latency":733.1999999999999,"recall":0.8897,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":75.7055,"latency":121.2,"recall":0.9999,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":11.2945,"latency":3611.2000000000003,"recall":0.996,"label":1,"qp$":23.564573336115167},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":17.3271,"latency":3774.7999999999997,"recall":0.9961,"label":1,"qp$":36.15084498226581},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":26.26,"latency":556.1,"recall":0.9999,"label":1,"qp$":54.78823283955769},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":15.2269,"latency":861.8,"recall":0.9888,"label":1,"qp$":31.76903818068016},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":15.1749,"latency":774.3,"recall":0.989,"label":1,"qp$":31.660546630502814},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":27.6181,"latency":305.5,"recall":0.9999,"label":1,"qp$":57.621740037554765},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":228.4,"latency":22.2,"recall":0.9348,"label":1,"qp$":1564.3835616438357},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":181.5,"latency":26.1,"recall":0.9345,"label":1,"qp$":1243.150684931507},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":205.7,"latency":24.2,"recall":0.9586,"label":1,"qp$":1408.9041095890411},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":67.63,"latency":36.0,"recall":0.8064,"label":1,"qp$":695.0668036998972},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":63.35,"latency":38.4,"recall":0.8065,"label":1,"qp$":651.0791366906475},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":176.7,"latency":27.599999999999998,"recall":1.0,"label":1,"qp$":1816.032887975334},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":15.33,"latency":84.9,"recall":0.8064,"label":1,"qp$":157.55395683453239},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":15.13,"latency":86.7,"recall":0.8065,"label":1,"qp$":155.49845837615624},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":17.41,"latency":74.30000000000001,"recall":1.0,"label":1,"qp$":178.9311408016444},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":16.34,"latency":88.7,"recall":0.879,"label":1,"qp$":83.7948717948718},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":10.45,"latency":126.8,"recall":0.8208,"label":1,"qp$":53.589743589743584},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":16.18,"latency":87.5,"recall":0.8793,"label":1,"qp$":82.97435897435896},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":9.8,"latency":130.89999999999998,"recall":0.8212,"label":1,"qp$":50.256410256410255},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":36.11,"latency":55.1,"recall":1.0,"label":1,"qp$":185.17948717948718},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":14.84,"latency":92.7,"recall":0.96,"label":1,"qp$":76.1025641025641},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":322.7,"latency":26.4,"recall":0.9478,"label":1,"qp$":276.2842465753425},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":265.5,"latency":26.9,"recall":0.9332,"label":1,"qp$":227.31164383561645},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":303.8,"latency":27.3,"recall":0.9478,"label":1,"qp$":260.10273972602744},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":180.2,"latency":28.2,"recall":0.9335,"label":1,"qp$":154.28082191780823},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":730.7,"latency":24.6,"recall":0.9586,"label":1,"qp$":625.5993150684932},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":104.3,"latency":31.7,"recall":0.9563,"label":1,"qp$":89.29794520547945},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":147.7,"latency":35.3,"recall":0.9707,"label":1,"qp$":189.60205391527597},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":782.5,"latency":25.9,"recall":1.0,"label":1,"qp$":1004.4929396662387},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":18.7634,"latency":153.70000000000002,"recall":0.8737,"label":1,"qp$":192.84069886947586},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":18.3619,"latency":79.8,"recall":0.8741,"label":1,"qp$":188.7142857142857},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":25.2744,"latency":61.199999999999996,"recall":0.9979,"label":1,"qp$":259.7574511819116},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":261.798,"latency":23.099999999999998,"recall":0.9262,"label":1,"qp$":1793.13698630137},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":166.1851,"latency":23.900000000000002,"recall":0.9264,"label":1,"qp$":1138.2541095890413},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":121.7169,"latency":29.0,"recall":0.9693,"label":1,"qp$":833.677397260274},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":46.6189,"latency":43.1,"recall":0.8737,"label":1,"qp$":479.1253854059609},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":42.4856,"latency":44.0,"recall":0.8741,"label":1,"qp$":436.6454265159301},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":138.9479,"latency":26.200000000000003,"recall":0.9979,"label":1,"qp$":1428.0359712230218},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":20.7437,"latency":75.80000000000001,"recall":0.9291,"label":1,"qp$":106.37794871794871},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":20.2993,"latency":76.5,"recall":0.9293,"label":1,"qp$":104.09897435897435},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":26.4719,"latency":67.0,"recall":1.0,"label":1,"qp$":135.75333333333333},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":8.6675,"latency":180.2,"recall":0.8369,"label":1,"qp$":44.44871794871795},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":7.8121,"latency":167.7,"recall":0.8369,"label":1,"qp$":40.06205128205128},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":16.869,"latency":87.8,"recall":0.9814,"label":1,"qp$":86.50769230769231},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":365.0835,"latency":23.599999999999998,"recall":0.945,"label":1,"qp$":312.57148972602744},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":325.5271,"latency":25.1,"recall":0.9452,"label":1,"qp$":278.7047089041096},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":596.7942,"latency":24.2,"recall":0.9693,"label":1,"qp$":510.95393835616443},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":303.2044,"latency":27.400000000000002,"recall":0.9246,"label":1,"qp$":259.5928082191781},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":136.0345,"latency":31.9,"recall":0.9244,"label":1,"qp$":116.46789383561645},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":66.7221,"latency":42.1,"recall":0.963,"label":1,"qp$":57.125085616438355},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":131.2549,"latency":30.200000000000003,"recall":0.9867,"label":1,"qp$":168.49152759948652},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":127.9337,"latency":30.099999999999998,"recall":0.9869,"label":1,"qp$":164.2281129653402},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":595.8462,"latency":23.400000000000002,"recall":1.0,"label":1,"qp$":764.8860077021822},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":330.0144,"latency":9.0,"recall":0.9507,"label":1,"qp$":2075.5622641509435},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":271.6585,"latency":10.1,"recall":0.9678,"label":1,"qp$":1708.5440251572327},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":216.5226,"latency":12.9,"recall":1.0,"label":1,"qp$":1361.777358490566},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":123.9553,"latency":23.0,"recall":0.971,"label":1,"qp$":389.79654088050313},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":59.1479,"latency":44.5,"recall":0.9906,"label":1,"qp$":185.99968553459118},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":40.999,"latency":55.300000000000004,"recall":1.0,"label":1,"qp$":128.92767295597486},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":579.9416,"latency":9.4,"recall":0.9213,"label":1,"qp$":1823.71572327044},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":425.2529,"latency":11.299999999999999,"recall":0.9686,"label":1,"qp$":1337.2732704402515},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":397.0539,"latency":13.799999999999999,"recall":1.0,"label":1,"qp$":1248.5971698113208},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":516.27,"latency":7.0,"recall":0.9463,"label":1,"qp$":3246.9811320754716},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":354.8416,"latency":10.0,"recall":0.9802,"label":1,"qp$":2231.708176100629},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":427.5229,"latency":8.7,"recall":1.0,"label":1,"qp$":2688.8232704402517},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":2884.689,"latency":5.3,"recall":0.8801,"label":1,"qp$":2267.837264150943},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":1689.5799,"latency":6.6,"recall":0.9493,"label":1,"qp$":1328.2860849056603},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1517.6792,"latency":10.0,"recall":1.0,"label":1,"qp$":1193.1440251572328},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":822.5318,"latency":5.6,"recall":0.9294,"label":1,"qp$":646.6444968553459},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":378.9146,"latency":10.3,"recall":0.9758,"label":1,"qp$":297.8888364779874},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":218.6854,"latency":16.2,"recall":1.0,"label":1,"qp$":171.92248427672953},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":297.5,"latency":7.2,"recall":0.974,"label":1,"qp$":1871.0691823899372},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":228.3,"latency":10.6,"recall":0.994,"label":1,"qp$":1435.8490566037735},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":584.0,"latency":4.6,"recall":1.0,"label":1,"qp$":3672.9559748427673},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":1871.0,"latency":7.0,"recall":0.9602,"label":1,"qp$":1470.9119496855346},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":556.7,"latency":6.7,"recall":0.9723,"label":1,"qp$":437.65723270440253},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1583.0,"latency":6.8,"recall":0.9836,"label":1,"qp$":1244.4968553459119},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":294.3,"latency":10.9,"recall":0.9939,"label":1,"qp$":231.3679245283019},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":2345.0,"latency":8.9,"recall":1.0,"label":1,"qp$":1843.553459119497},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":295.6,"latency":12.3,"recall":1.0,"label":1,"qp$":232.38993710691824},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":143.0,"latency":33.5,"recall":0.9818,"label":1,"qp$":899.3710691823899},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":106.0,"latency":20.7,"recall":0.9887,"label":1,"qp$":666.6666666666666},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":189.0,"latency":11.6,"recall":1.0,"label":1,"qp$":1188.6792452830189},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":379.9721,"latency":12.4,"recall":0.982,"label":1,"qp$":1194.8808176100629},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":71.74,"latency":50.8,"recall":0.9883,"label":1,"qp$":225.59748427672955},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":287.0,"latency":14.9,"recall":0.9865,"label":1,"qp$":902.5157232704403},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":34.6654,"latency":64.69999999999999,"recall":0.9961,"label":1,"qp$":109.01069182389936},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":412.0,"latency":10.3,"recall":1.0,"label":1,"qp$":1295.5974842767296},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":42.169,"latency":46.800000000000004,"recall":1.0,"label":1,"qp$":132.6069182389937},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":537.4975,"latency":18.9,"recall":0.8903,"label":1,"qp$":376.9267180925666},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":372.0466,"latency":17.8,"recall":0.8904,"label":1,"qp$":260.9022440392707},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1156.2898,"latency":14.4,"recall":0.9989,"label":1,"qp$":810.8624123422161},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":110.248,"latency":69.0,"recall":0.898,"label":1,"qp$":77.31276297335204},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":87.2601,"latency":27.799999999999997,"recall":0.898,"label":1,"qp$":61.1922159887798},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":125.7846,"latency":23.099999999999998,"recall":0.975,"label":1,"qp$":88.20799438990183},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":240.7209,"latency":17.4,"recall":0.8887,"label":1,"qp$":844.0424263674614},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":189.4399,"latency":17.5,"recall":0.8889,"label":1,"qp$":664.2352734922861},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":313.5116,"latency":16.1,"recall":0.9999,"label":1,"qp$":1099.2692847124824},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":78.7196,"latency":49.4,"recall":0.9203,"label":1,"qp$":474.21445783132526},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":68.3111,"latency":35.5,"recall":0.9202,"label":1,"qp$":411.5126506024096},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":210.2147,"latency":26.8,"recall":0.9996,"label":1,"qp$":1266.3536144578313},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":188.6436,"latency":917.5,"recall":0.9175,"label":1,"qp$":661.4431977559607},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":155.6991,"latency":917.1,"recall":0.9171,"label":1,"qp$":545.929523141655},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":445.3289,"latency":14.1,"recall":0.9999,"label":1,"qp$":1561.461781206171},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":633.6033,"latency":24.6,"recall":0.919,"label":1,"qp$":444.32208976157085},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":95.5682,"latency":58.7,"recall":0.9463,"label":1,"qp$":67.01837307152876},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":434.4062,"latency":17.4,"recall":0.9181,"label":1,"qp$":304.6326788218794},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":80.3192,"latency":26.599999999999998,"recall":0.9462,"label":1,"qp$":56.324824684431974},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":1509.3293,"latency":18.5,"recall":0.9995,"label":1,"qp$":1058.4356942496495},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":166.7252,"latency":19.599999999999998,"recall":0.9988,"label":1,"qp$":116.91809256661992},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":46.8622,"latency":123.0,"recall":0.9957,"label":1,"qp$":4.639821782178218},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1.8388,"latency":1063.5,"recall":0.9957,"label":1,"qp$":0.18205940594059405},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":45.0666,"latency":109.2,"recall":1.0,"label":1,"qp$":4.462039603960396},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":43.5017,"latency":228.7,"recall":0.9957,"label":1,"qp$":1.3344079754601226},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1.5668,"latency":1114.4,"recall":0.9957,"label":1,"qp$":0.04806134969325153},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":41.5443,"latency":159.0,"recall":1.0,"label":1,"qp$":1.2743650306748466},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":67.9121,"latency":179.5,"recall":0.9909,"label":1,"qp$":2.083193251533742},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.7636,"latency":1921.3,"recall":0.9908,"label":1,"qp$":0.02342331288343558},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":32.0,"latency":124.5,"recall":1.0,"label":1,"qp$":0.9815950920245399},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":63.1365,"latency":145.7,"recall":0.991,"label":1,"qp$":6.251138613861386},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.7512,"latency":1983.8,"recall":0.9908,"label":1,"qp$":0.07437623762376237},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":30.1358,"latency":129.8,"recall":1.0,"label":1,"qp$":2.983742574257426},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":37.432,"latency":75.0,"recall":0.9975,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":37.0696,"latency":73.5,"recall":0.9976,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":81.1915,"latency":53.0,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":321.6048,"latency":13.4,"recall":0.989,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":22.1467,"latency":86.8,"recall":0.9972,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":303.2551,"latency":10.9,"recall":0.9876,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":21.5388,"latency":81.69999999999999,"recall":0.997,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":394.5418,"latency":8.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":37.878,"latency":45.0,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":180.2757,"latency":6.0,"recall":0.9942,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":179.0033,"latency":6.4,"recall":0.9943,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":526.8846,"latency":3.6,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":626.5243,"latency":6.2,"recall":0.9954,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":78.4227,"latency":25.3,"recall":0.9982,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":599.4213,"latency":6.6,"recall":0.9955,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":78.5351,"latency":26.3,"recall":0.9982,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":2098.2113,"latency":3.4,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":275.6292,"latency":10.0,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":100.6667,"latency":21.1,"recall":0.9909,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":101.1399,"latency":19.7,"recall":0.9907,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":52.2606,"latency":18.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":61.0661,"latency":49.8,"recall":0.9911,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":58.9326,"latency":44.6,"recall":0.9911,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":42.5977,"latency":54.9,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":536.0726,"latency":8.200000000000001,"recall":0.9728,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":467.179,"latency":7.0,"recall":0.9697,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":431.7512,"latency":8.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":274.5407,"latency":4.8999999999999995,"recall":0.9807,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":236.5672,"latency":10.3,"recall":0.981,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":309.4833,"latency":4.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":178.6585,"latency":13.700000000000001,"recall":0.9843,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":178.3732,"latency":15.0,"recall":0.9844,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":229.3526,"latency":12.5,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":1258.7043,"latency":4.8999999999999995,"recall":0.9799,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":1075.8776,"latency":5.3,"recall":0.98,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1494.8493,"latency":4.7,"recall":1.0,"label":1,"qp$":0.0}] \ No newline at end of file +[{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":0.8836,"latency":2523.0,"recall":0.8528,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":0.8937,"latency":3720.2000000000003,"recall":0.8525,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":1.2145,"latency":3622.3999999999996,"recall":0.7487,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":10.6271,"latency":730.7,"recall":0.8898,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":10.8507,"latency":733.1999999999999,"recall":0.8897,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":75.7055,"latency":121.2,"recall":0.9999,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":11.2945,"latency":3611.2000000000003,"recall":0.996,"label":1,"qp$":84832.4640100146},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":17.3271,"latency":3774.7999999999997,"recall":0.9961,"label":1,"qp$":130143.04193615691},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":26.26,"latency":556.1,"recall":0.9999,"label":1,"qp$":197237.6382224077},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":15.2269,"latency":861.8,"recall":0.9888,"label":1,"qp$":114368.53745044857},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":15.1749,"latency":774.3,"recall":0.989,"label":1,"qp$":113977.96786981013},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":27.6181,"latency":305.5,"recall":0.9999,"label":1,"qp$":207438.26413519715},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":228.4,"latency":22.2,"recall":0.9348,"label":1,"qp$":5631780.821917809},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":181.5,"latency":26.1,"recall":0.9345,"label":1,"qp$":4475342.465753425},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":205.7,"latency":24.2,"recall":0.9586,"label":1,"qp$":5072054.794520548},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":67.63,"latency":36.0,"recall":0.8064,"label":1,"qp$":2502240.4933196297},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":63.35,"latency":38.4,"recall":0.8065,"label":1,"qp$":2343884.892086331},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":176.7,"latency":27.599999999999998,"recall":1.0,"label":1,"qp$":6537718.396711202},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":15.33,"latency":84.9,"recall":0.8064,"label":1,"qp$":567194.2446043165},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":15.13,"latency":86.7,"recall":0.8065,"label":1,"qp$":559794.4501541625},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":17.41,"latency":74.30000000000001,"recall":1.0,"label":1,"qp$":644152.1068859198},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":16.34,"latency":88.7,"recall":0.879,"label":1,"qp$":301661.53846153844},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":10.45,"latency":126.8,"recall":0.8208,"label":1,"qp$":192923.0769230769},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":16.18,"latency":87.5,"recall":0.8793,"label":1,"qp$":298707.69230769225},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":9.8,"latency":130.89999999999998,"recall":0.8212,"label":1,"qp$":180923.0769230769},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":36.11,"latency":55.1,"recall":1.0,"label":1,"qp$":666646.1538461539},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":14.84,"latency":92.7,"recall":0.96,"label":1,"qp$":273969.23076923075},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":322.7,"latency":26.4,"recall":0.9478,"label":1,"qp$":994623.287671233},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":265.5,"latency":26.9,"recall":0.9332,"label":1,"qp$":818321.9178082192},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":303.8,"latency":27.3,"recall":0.9478,"label":1,"qp$":936369.8630136987},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":180.2,"latency":28.2,"recall":0.9335,"label":1,"qp$":555410.9589041097},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":730.7,"latency":24.6,"recall":0.9586,"label":1,"qp$":2252157.5342465756},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":104.3,"latency":31.7,"recall":0.9563,"label":1,"qp$":321472.602739726},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":147.7,"latency":35.3,"recall":0.9707,"label":1,"qp$":682567.3940949935},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":782.5,"latency":25.9,"recall":1.0,"label":1,"qp$":3616174.5827984596},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":18.7634,"latency":153.70000000000002,"recall":0.8737,"label":1,"qp$":694226.515930113},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":18.3619,"latency":79.8,"recall":0.8741,"label":1,"qp$":679371.4285714285},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":25.2744,"latency":61.199999999999996,"recall":0.9979,"label":1,"qp$":935126.8242548818},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":261.798,"latency":23.099999999999998,"recall":0.9262,"label":1,"qp$":6455293.150684931},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":166.1851,"latency":23.900000000000002,"recall":0.9264,"label":1,"qp$":4097714.7945205485},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":121.7169,"latency":29.0,"recall":0.9693,"label":1,"qp$":3001238.6301369863},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":46.6189,"latency":43.1,"recall":0.8737,"label":1,"qp$":1724851.3874614593},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":42.4856,"latency":44.0,"recall":0.8741,"label":1,"qp$":1571923.5354573485},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":138.9479,"latency":26.200000000000003,"recall":0.9979,"label":1,"qp$":5140929.496402878},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":20.7437,"latency":75.80000000000001,"recall":0.9291,"label":1,"qp$":382960.6153846154},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":20.2993,"latency":76.5,"recall":0.9293,"label":1,"qp$":374756.30769230763},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":26.4719,"latency":67.0,"recall":1.0,"label":1,"qp$":488712.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":8.6675,"latency":180.2,"recall":0.8369,"label":1,"qp$":160015.38461538462},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":7.8121,"latency":167.7,"recall":0.8369,"label":1,"qp$":144223.3846153846},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":16.869,"latency":87.8,"recall":0.9814,"label":1,"qp$":311427.6923076923},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":365.0835,"latency":23.599999999999998,"recall":0.945,"label":1,"qp$":1125257.3630136987},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":325.5271,"latency":25.1,"recall":0.9452,"label":1,"qp$":1003336.9520547946},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":596.7942,"latency":24.2,"recall":0.9693,"label":1,"qp$":1839434.1780821919},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":303.2044,"latency":27.400000000000002,"recall":0.9246,"label":1,"qp$":934534.1095890411},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":136.0345,"latency":31.9,"recall":0.9244,"label":1,"qp$":419284.4178082192},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":66.7221,"latency":42.1,"recall":0.963,"label":1,"qp$":205650.30821917808},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":131.2549,"latency":30.200000000000003,"recall":0.9867,"label":1,"qp$":606569.4993581515},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":127.9337,"latency":30.099999999999998,"recall":0.9869,"label":1,"qp$":591221.2066752247},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":595.8462,"latency":23.400000000000002,"recall":1.0,"label":1,"qp$":2753589.627727856},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":330.0144,"latency":9.0,"recall":0.9507,"label":1,"qp$":7472024.150943397},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":271.6585,"latency":10.1,"recall":0.9678,"label":1,"qp$":6150758.490566038},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":216.5226,"latency":12.9,"recall":1.0,"label":1,"qp$":4902398.490566038},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":123.9553,"latency":23.0,"recall":0.971,"label":1,"qp$":1403267.5471698113},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":59.1479,"latency":44.5,"recall":0.9906,"label":1,"qp$":669598.8679245282},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":40.999,"latency":55.300000000000004,"recall":1.0,"label":1,"qp$":464139.62264150946},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":579.9416,"latency":9.4,"recall":0.9213,"label":1,"qp$":6565376.603773585},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":425.2529,"latency":11.299999999999999,"recall":0.9686,"label":1,"qp$":4814183.773584905},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":397.0539,"latency":13.799999999999999,"recall":1.0,"label":1,"qp$":4494949.811320755},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":516.27,"latency":7.0,"recall":0.9463,"label":1,"qp$":11689132.075471697},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":354.8416,"latency":10.0,"recall":0.9802,"label":1,"qp$":8034149.433962264},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":427.5229,"latency":8.7,"recall":1.0,"label":1,"qp$":9679763.773584906},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":2884.689,"latency":5.3,"recall":0.8801,"label":1,"qp$":8164214.150943396},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":1689.5799,"latency":6.6,"recall":0.9493,"label":1,"qp$":4781829.905660377},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1517.6792,"latency":10.0,"recall":1.0,"label":1,"qp$":4295318.4905660385},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":822.5318,"latency":5.6,"recall":0.9294,"label":1,"qp$":2327920.1886792453},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":378.9146,"latency":10.3,"recall":0.9758,"label":1,"qp$":1072399.8113207547},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":218.6854,"latency":16.2,"recall":1.0,"label":1,"qp$":618920.9433962263},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":297.5,"latency":7.2,"recall":0.974,"label":1,"qp$":6735849.0566037735},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":228.3,"latency":10.6,"recall":0.994,"label":1,"qp$":5169056.603773585},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":584.0,"latency":4.6,"recall":1.0,"label":1,"qp$":13222641.509433962},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":1871.0,"latency":7.0,"recall":0.9602,"label":1,"qp$":5295283.018867925},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":556.7,"latency":6.7,"recall":0.9723,"label":1,"qp$":1575566.0377358492},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1583.0,"latency":6.8,"recall":0.9836,"label":1,"qp$":4480188.679245283},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":294.3,"latency":10.9,"recall":0.9939,"label":1,"qp$":832924.5283018869},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":2345.0,"latency":8.9,"recall":1.0,"label":1,"qp$":6636792.452830189},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":295.6,"latency":12.3,"recall":1.0,"label":1,"qp$":836603.7735849057},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":143.0,"latency":33.5,"recall":0.9818,"label":1,"qp$":3237735.8490566034},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":106.0,"latency":20.7,"recall":0.9887,"label":1,"qp$":2400000.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":189.0,"latency":11.6,"recall":1.0,"label":1,"qp$":4279245.2830188675},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":379.9721,"latency":12.4,"recall":0.982,"label":1,"qp$":4301570.9433962265},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":71.74,"latency":50.8,"recall":0.9883,"label":1,"qp$":812150.9433962264},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":287.0,"latency":14.9,"recall":0.9865,"label":1,"qp$":3249056.603773585},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":34.6654,"latency":64.69999999999999,"recall":0.9961,"label":1,"qp$":392438.4905660377},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":412.0,"latency":10.3,"recall":1.0,"label":1,"qp$":4664150.9433962265},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":42.169,"latency":46.800000000000004,"recall":1.0,"label":1,"qp$":477384.9056603773},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":537.4975,"latency":18.9,"recall":0.8903,"label":1,"qp$":1356936.1851332397},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":372.0466,"latency":17.8,"recall":0.8904,"label":1,"qp$":939248.0785413746},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1156.2898,"latency":14.4,"recall":0.9989,"label":1,"qp$":2919104.684431978},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":110.248,"latency":69.0,"recall":0.898,"label":1,"qp$":278325.94670406735},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":87.2601,"latency":27.799999999999997,"recall":0.898,"label":1,"qp$":220291.97755960727},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":125.7846,"latency":23.099999999999998,"recall":0.975,"label":1,"qp$":317548.77980364655},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":240.7209,"latency":17.4,"recall":0.8887,"label":1,"qp$":3038552.734922861},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":189.4399,"latency":17.5,"recall":0.8889,"label":1,"qp$":2391246.98457223},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":313.5116,"latency":16.1,"recall":0.9999,"label":1,"qp$":3957369.424964937},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":78.7196,"latency":49.4,"recall":0.9203,"label":1,"qp$":1707172.048192771},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":68.3111,"latency":35.5,"recall":0.9202,"label":1,"qp$":1481445.5421686745},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":210.2147,"latency":26.8,"recall":0.9996,"label":1,"qp$":4558873.012048192},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":188.6436,"latency":917.5,"recall":0.9175,"label":1,"qp$":2381195.5119214584},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":155.6991,"latency":917.1,"recall":0.9171,"label":1,"qp$":1965346.283309958},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":445.3289,"latency":14.1,"recall":0.9999,"label":1,"qp$":5621262.412342216},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":633.6033,"latency":24.6,"recall":0.919,"label":1,"qp$":1599559.5231416551},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":95.5682,"latency":58.7,"recall":0.9463,"label":1,"qp$":241266.14305750353},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":434.4062,"latency":17.4,"recall":0.9181,"label":1,"qp$":1096677.643758766},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":80.3192,"latency":26.599999999999998,"recall":0.9462,"label":1,"qp$":202769.3688639551},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":1509.3293,"latency":18.5,"recall":0.9995,"label":1,"qp$":3810368.4992987383},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":166.7252,"latency":19.599999999999998,"recall":0.9988,"label":1,"qp$":420905.1332398317},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":46.8622,"latency":123.0,"recall":0.9957,"label":1,"qp$":16703.358415841583},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1.8388,"latency":1063.5,"recall":0.9957,"label":1,"qp$":655.4138613861386},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":45.0666,"latency":109.2,"recall":1.0,"label":1,"qp$":16063.342574257427},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":43.5017,"latency":228.7,"recall":0.9957,"label":1,"qp$":4803.8687116564415},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1.5668,"latency":1114.4,"recall":0.9957,"label":1,"qp$":173.02085889570552},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":41.5443,"latency":159.0,"recall":1.0,"label":1,"qp$":4587.714110429448},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":67.9121,"latency":179.5,"recall":0.9909,"label":1,"qp$":7499.495705521471},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.7636,"latency":1921.3,"recall":0.9908,"label":1,"qp$":84.32392638036809},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":32.0,"latency":124.5,"recall":1.0,"label":1,"qp$":3533.7423312883434},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":63.1365,"latency":145.7,"recall":0.991,"label":1,"qp$":22504.09900990099},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.7512,"latency":1983.8,"recall":0.9908,"label":1,"qp$":267.7544554455445},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":30.1358,"latency":129.8,"recall":1.0,"label":1,"qp$":10741.473267326734},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":37.432,"latency":75.0,"recall":0.9975,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":37.0696,"latency":73.5,"recall":0.9976,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":81.1915,"latency":53.0,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":321.6048,"latency":13.4,"recall":0.989,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":22.1467,"latency":86.8,"recall":0.9972,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":303.2551,"latency":10.9,"recall":0.9876,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":21.5388,"latency":81.69999999999999,"recall":0.997,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":394.5418,"latency":8.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":37.878,"latency":45.0,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":180.2757,"latency":6.0,"recall":0.9942,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":179.0033,"latency":6.4,"recall":0.9943,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":526.8846,"latency":3.6,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":626.5243,"latency":6.2,"recall":0.9954,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":78.4227,"latency":25.3,"recall":0.9982,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":599.4213,"latency":6.6,"recall":0.9955,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":78.5351,"latency":26.3,"recall":0.9982,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":2098.2113,"latency":3.4,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":275.6292,"latency":10.0,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":100.6667,"latency":21.1,"recall":0.9909,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":101.1399,"latency":19.7,"recall":0.9907,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":52.2606,"latency":18.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":61.0661,"latency":49.8,"recall":0.9911,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":58.9326,"latency":44.6,"recall":0.9911,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":42.5977,"latency":54.9,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":536.0726,"latency":8.200000000000001,"recall":0.9728,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":467.179,"latency":7.0,"recall":0.9697,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":431.7512,"latency":8.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":274.5407,"latency":4.8999999999999995,"recall":0.9807,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":236.5672,"latency":10.3,"recall":0.981,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":309.4833,"latency":4.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":178.6585,"latency":13.700000000000001,"recall":0.9843,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":178.3732,"latency":15.0,"recall":0.9844,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":229.3526,"latency":12.5,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":1258.7043,"latency":4.8999999999999995,"recall":0.9799,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":1075.8776,"latency":5.3,"recall":0.98,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1494.8493,"latency":4.7,"recall":1.0,"label":1,"qp$":0.0}] \ No newline at end of file From 3b50316c5fc53311424f3835d92cdee4cccb041d Mon Sep 17 00:00:00 2001 From: yangxuan Date: Fri, 10 Nov 2023 11:48:00 +0800 Subject: [PATCH 022/327] Fix milvus capacity test unable to load Fixes: #238 Signed-off-by: yangxuan --- .../backend/clients/milvus/milvus.py | 33 ++++++++++--------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/vectordb_bench/backend/clients/milvus/milvus.py b/vectordb_bench/backend/clients/milvus/milvus.py index 356935467..b3ed56047 100644 --- a/vectordb_bench/backend/clients/milvus/milvus.py +++ b/vectordb_bench/backend/clients/milvus/milvus.py @@ -88,21 +88,6 @@ def init(self) -> None: yield connections.disconnect("default") - def _pre_load(self, coll: Collection): - if not coll.has_index(index_name=self._index_name): - log.info(f"{self.name} create index and load") - try: - coll.create_index( - self._vector_field, - self.case_config.index_param(), - index_name=self._index_name, - ) - - coll.load() - except Exception as e: - log.warning(f"{self.name} pre load error: {e}") - raise e from None - def _optimize(self): log.info(f"{self.name} optimizing before search") try: @@ -142,7 +127,23 @@ def wait_index(): def ready_to_load(self): assert self.col, "Please call self.init() before" self._pre_load(self.col) - pass + + def _pre_load(self, coll: Collection): + try: + if not coll.has_index(index_name=self._index_name): + log.info(f"{self.name} create index") + coll.create_index( + self._vector_field, + self.case_config.index_param(), + index_name=self._index_name, + ) + + coll.load() + log.info(f"{self.name} load") + except Exception as e: + log.warning(f"{self.name} pre load error: {e}") + raise e from None + def optimize(self): assert self.col, "Please call self.init() before" From e0ef44043f1950457523780b1b348a50b466933f Mon Sep 17 00:00:00 2001 From: cutecutecat Date: Tue, 14 Nov 2023 10:13:20 +0800 Subject: [PATCH 023/327] feat: add new db pgvecto.rs Signed-off-by: cutecutecat --- README.md | 1 + pyproject.toml | 2 + vectordb_bench/backend/clients/__init__.py | 13 ++ .../backend/clients/pgvecto_rs/config.py | 115 +++++++++++ .../backend/clients/pgvecto_rs/pgvecto_rs.py | 192 ++++++++++++++++++ .../frontend/const/dbCaseConfigs.py | 69 ++++++- vectordb_bench/frontend/const/styles.py | 3 +- vectordb_bench/models.py | 5 +- 8 files changed, 395 insertions(+), 5 deletions(-) create mode 100644 vectordb_bench/backend/clients/pgvecto_rs/config.py create mode 100644 vectordb_bench/backend/clients/pgvecto_rs/pgvecto_rs.py diff --git a/README.md b/README.md index 1fc4f0934..d656e8dc4 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,7 @@ All the database client supported |weaviate|`pip install vectordb-bench[weaviate]`| |elastic|`pip install vectordb-bench[elastic]`| |pgvector|`pip install vectordb-bench[pgvector]`| +|pgvecto.rs|`pip install vectordb-bench[pgvecto_rs]`| |redis|`pip install vectordb-bench[redis]`| |chromadb|`pip install vectordb-bench[chromadb]`| diff --git a/pyproject.toml b/pyproject.toml index 28200f3de..f73bc2940 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,6 +53,7 @@ all = [ "sqlalchemy", "redis", "chromadb", + "psycopg2", ] qdrant = [ "qdrant-client" ] @@ -60,6 +61,7 @@ pinecone = [ "pinecone-client" ] weaviate = [ "weaviate-client" ] elastic = [ "elasticsearch" ] pgvector = [ "pgvector", "sqlalchemy" ] +pgvecto_rs = [ "psycopg2" ] redis = [ "redis" ] chromadb = [ "chromadb" ] diff --git a/vectordb_bench/backend/clients/__init__.py b/vectordb_bench/backend/clients/__init__.py index 63cc607e8..3df11610b 100644 --- a/vectordb_bench/backend/clients/__init__.py +++ b/vectordb_bench/backend/clients/__init__.py @@ -29,6 +29,7 @@ class DB(Enum): QdrantCloud = "QdrantCloud" WeaviateCloud = "WeaviateCloud" PgVector = "PgVector" + PgVectoRS = "PgVectoRS" Redis = "Redis" Chroma = "Chroma" @@ -64,6 +65,10 @@ def init_cls(self) -> Type[VectorDB]: from .pgvector.pgvector import PgVector return PgVector + if self == DB.PgVectoRS: + from .pgvecto_rs.pgvecto_rs import PgVectoRS + return PgVectoRS + if self == DB.Redis: from .redis.redis import Redis return Redis @@ -103,6 +108,10 @@ def config_cls(self) -> Type[DBConfig]: from .pgvector.config import PgVectorConfig return PgVectorConfig + if self == DB.PgVectoRS: + from .pgvecto_rs.config import PgVectoRSConfig + return PgVectoRSConfig + if self == DB.Redis: from .redis.config import RedisConfig return RedisConfig @@ -136,6 +145,10 @@ def case_config_cls(self, index_type: IndexType | None = None) -> Type[DBCaseCon from .pgvector.config import PgVectorIndexConfig return PgVectorIndexConfig + if self == DB.PgVectoRS: + from .pgvecto_rs.config import _pgvecto_rs_case_config + return _pgvecto_rs_case_config.get(index_type) + # DB.Pinecone, DB.Chroma, DB.Redis return EmptyDBCaseConfig diff --git a/vectordb_bench/backend/clients/pgvecto_rs/config.py b/vectordb_bench/backend/clients/pgvecto_rs/config.py new file mode 100644 index 000000000..7ad5983f1 --- /dev/null +++ b/vectordb_bench/backend/clients/pgvecto_rs/config.py @@ -0,0 +1,115 @@ +from typing import Literal +from pydantic import BaseModel, SecretStr +from ..api import DBConfig, DBCaseConfig, MetricType, IndexType + +POSTGRE_URL_PLACEHOLDER = "postgresql://%s:%s@%s/%s" + + +class PgVectoRSConfig(DBConfig): + user_name: SecretStr = "postgres" + password: SecretStr + url: SecretStr + db_name: str + + def to_dict(self) -> dict: + user_str = self.user_name.get_secret_value() + pwd_str = self.password.get_secret_value() + url_str = self.url.get_secret_value() + host, port = url_str.split(":") + return { + "host": host, + "port": port, + "dbname": self.db_name, + "user": user_str, + "password": pwd_str, + } + + +class PgVectoRSIndexConfig(BaseModel, DBCaseConfig): + metric_type: MetricType | None = None + quantizationType: Literal["trivial", "scalar", "product"] + quantizationRatio: None | Literal["x4", "x8", "x16", "x32", "x64"] + + def parse_quantization(self) -> str: + if self.quantizationType == "trivial": + return "quantization = { trivial = { } }" + elif self.quantizationType == "scalar": + return "quantization = { scalar = { } }" + else: + return f'quantization = {{ product = {{ ratio = "{self.quantizationRatio}" }} }}' + + def parse_metric(self) -> str: + if self.metric_type == MetricType.L2: + return "l2_ops" + elif self.metric_type == MetricType.IP: + return "dot_ops" + return "cosine_ops" + + def parse_metric_fun_op(self) -> str: + if self.metric_type == MetricType.L2: + return "<->" + elif self.metric_type == MetricType.IP: + return "<#>" + return "<=>" + + +class HNSWConfig(PgVectoRSIndexConfig): + M: int + efConstruction: int + index: IndexType = IndexType.HNSW + + def index_param(self) -> dict: + options = f""" +capacity = 1048576 +[algorithm.hnsw] +m = {self.M} +ef_construction = {self.efConstruction} +{self.parse_quantization()} +""" + return {"options": options, "metric": self.parse_metric()} + + def search_param(self) -> dict: + return {"metrics_op": self.parse_metric_fun_op()} + + +class IVFFlatConfig(PgVectoRSIndexConfig): + nlist: int + nprobe: int | None = None + index: IndexType = IndexType.IVFFlat + + def index_param(self) -> dict: + options = f""" +capacity = 1048576 +[algorithm.ivf] +nlist = {self.nlist} +nprob = {self.nprobe if self.nprobe else 10} +{self.parse_quantization()} +""" + return {"options": options, "metric": self.parse_metric()} + + def search_param(self) -> dict: + return {"metrics_op": self.parse_metric_fun_op()} + + +class FLATConfig(PgVectoRSIndexConfig): + index: IndexType = IndexType.Flat + + def index_param(self) -> dict: + options = f""" +capacity = 1048576 +[algorithm.flat] +{self.parse_quantization()} +""" + return {"options": options, "metric": self.parse_metric()} + + def search_param(self) -> dict: + return {"metrics_op": self.parse_metric_fun_op()} + + +_pgvecto_rs_case_config = { + IndexType.AUTOINDEX: HNSWConfig, + IndexType.HNSW: HNSWConfig, + IndexType.DISKANN: HNSWConfig, + IndexType.IVFFlat: IVFFlatConfig, + IndexType.Flat: FLATConfig, +} diff --git a/vectordb_bench/backend/clients/pgvecto_rs/pgvecto_rs.py b/vectordb_bench/backend/clients/pgvecto_rs/pgvecto_rs.py new file mode 100644 index 000000000..22476522c --- /dev/null +++ b/vectordb_bench/backend/clients/pgvecto_rs/pgvecto_rs.py @@ -0,0 +1,192 @@ +"""Wrapper around the Pgvector vector database over VectorDB""" + +import io +import logging +from contextlib import contextmanager +from typing import Any +import pandas as pd + +import psycopg2 + +from ..api import VectorDB, DBCaseConfig + +log = logging.getLogger(__name__) + + +class PgVectoRS(VectorDB): + """Use SQLAlchemy instructions""" + + def __init__( + self, + dim: int, + db_config: dict, + db_case_config: DBCaseConfig, + collection_name: str = "PgVectorCollection", + drop_old: bool = False, + **kwargs, + ): + self.db_config = db_config + self.case_config = db_case_config + self.table_name = collection_name + self.dim = dim + + self._index_name = "pqvector_index" + self._primary_field = "id" + self._vector_field = "embedding" + + # construct basic units + self.conn = psycopg2.connect(**self.db_config) + self.conn.autocommit = False + self.cursor = self.conn.cursor() + + # create vector extension + self.cursor.execute("CREATE EXTENSION IF NOT EXISTS vectors") + self.conn.commit() + + if drop_old: + log.info(f"Pgvecto.rs client drop table : {self.table_name}") + self._drop_index() + self._drop_table() + self._create_table(dim) + self._create_index() + + self.cursor.close() + self.conn.close() + self.cursor = None + self.conn = None + + @contextmanager + def init(self) -> None: + """ + Examples: + >>> with self.init(): + >>> self.insert_embeddings() + >>> self.search_embedding() + """ + self.conn = psycopg2.connect(**self.db_config) + self.conn.autocommit = False + self.cursor = self.conn.cursor() + + try: + yield + finally: + self.cursor.close() + self.conn.close() + self.cursor = None + self.conn = None + + def _drop_table(self): + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + self.cursor.execute(f'DROP TABLE IF EXISTS public."{self.table_name}"') + self.conn.commit() + + def ready_to_load(self): + pass + + def optimize(self): + pass + + def ready_to_search(self): + pass + + def _drop_index(self): + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + self.cursor.execute(f'DROP INDEX IF EXISTS "{self._index_name}"') + self.conn.commit() + + def _create_index(self): + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + index_param = self.case_config.index_param() + + try: + # create table + self.cursor.execute( + f'CREATE INDEX IF NOT EXISTS {self._index_name} ON public."{self.table_name}" \ + USING vectors (embedding {index_param["metric"]}) WITH (options = $${index_param["options"]}$$);' + ) + self.conn.commit() + except Exception as e: + log.warning( + f"Failed to create pgvector table: {self.table_name} error: {e}" + ) + raise e from None + + def _create_table(self, dim: int): + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + try: + # create table + self.cursor.execute( + f'CREATE TABLE IF NOT EXISTS public."{self.table_name}" \ + (id Integer PRIMARY KEY, embedding vector({dim}));' + ) + self.cursor.execute( + f'ALTER TABLE public."{self.table_name}" ALTER COLUMN embedding SET STORAGE PLAIN;' + ) + self.conn.commit() + except Exception as e: + log.warning( + f"Failed to create pgvector table: {self.table_name} error: {e}" + ) + raise e from None + + def insert_embeddings( + self, + embeddings: list[list[float]], + metadata: list[int], + **kwargs: Any, + ) -> (int, Exception): + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + try: + items = {"id": metadata, "embedding": embeddings} + df = pd.DataFrame(items) + csv_buffer = io.StringIO() + df.to_csv(csv_buffer, index=False, header=False) + csv_buffer.seek(0) + self.cursor.copy_expert( + f'COPY public."{self.table_name}" FROM STDIN WITH (FORMAT CSV)', + csv_buffer, + ) + self.conn.commit() + return len(metadata), None + except Exception as e: + log.warning( + f"Failed to insert data into pgvector table ({self.table_name}), error: {e}" + ) + + def search_embedding( + self, + query: list[float], + k: int = 100, + filters: dict | None = None, + timeout: int | None = None, + ) -> list[int]: + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + search_param = self.case_config.search_param() + + if filters: + gt = filters.get("id") + self.cursor.execute( + f"SELECT id FROM (SELECT * FROM public.\"{self.table_name}\" ORDER BY embedding \ + {search_param['metrics_op']} '{query}' LIMIT {k}) AS X WHERE id > {gt} ;" + ) + else: + self.cursor.execute( + f"SELECT id FROM public.\"{self.table_name}\" ORDER BY embedding \ + {search_param['metrics_op']} '{query}' LIMIT {k};" + ) + self.conn.commit() + result = self.cursor.fetchall() + + return [int(i[0]) for i in result] diff --git a/vectordb_bench/frontend/const/dbCaseConfigs.py b/vectordb_bench/frontend/const/dbCaseConfigs.py index bff623bb0..1298983ff 100644 --- a/vectordb_bench/frontend/const/dbCaseConfigs.py +++ b/vectordb_bench/frontend/const/dbCaseConfigs.py @@ -24,7 +24,7 @@ CaseType.Performance768D1M1P, DIVIDER, CaseType.Performance1536D5M1P, - CaseType.Performance1536D500K1P, + CaseType.Performance1536D500K1P, DIVIDER, CaseType.Performance768D10M99P, CaseType.Performance768D1M99P, @@ -111,6 +111,18 @@ class CaseConfigInput(BaseModel): }, ) +CaseConfigParamInput_EFConstruction_PgVectoRS = CaseConfigInput( + label=CaseConfigParamType.EFConstruction, + inputType=InputType.Number, + inputConfig={ + "min": 8, + "max": 512, + "value": 360, + }, + isDisplayed=lambda config: config[CaseConfigParamType.IndexType] + == IndexType.HNSW.value, +) + CaseConfigParamInput_M_ES = CaseConfigInput( label=CaseConfigParamType.M, inputType=InputType.Number, @@ -215,6 +227,23 @@ class CaseConfigInput(BaseModel): }, ) +CaseConfigParamInput_QuantizationType_PgVectoRS = CaseConfigInput( + label=CaseConfigParamType.quantizationType, + inputType=InputType.Option, + inputConfig={ + "options": ["trivial", "scalar", "product"], + }, +) + +CaseConfigParamInput_QuantizationRatio_PgVectoRS = CaseConfigInput( + label=CaseConfigParamType.quantizationRatio, + inputType=InputType.Option, + inputConfig={ + "options": ["x4", "x8", "x16", "x32", "x64"], + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.quantizationType, None) + == "product", +) MilvusLoadConfig = [ CaseConfigParamInput_IndexType, @@ -252,6 +281,25 @@ class CaseConfigInput(BaseModel): PgVectorLoadingConfig = [CaseConfigParamInput_Lists] PgVectorPerformanceConfig = [CaseConfigParamInput_Lists, CaseConfigParamInput_Probes] +PgVectoRSLoadingConfig = [ + CaseConfigParamInput_IndexType, + CaseConfigParamInput_M, + CaseConfigParamInput_EFConstruction_PgVectoRS, + CaseConfigParamInput_Nlist, + CaseConfigParamInput_QuantizationType_PgVectoRS, + CaseConfigParamInput_QuantizationRatio_PgVectoRS, +] + +PgVectoRSPerformanceConfig = [ + CaseConfigParamInput_IndexType, + CaseConfigParamInput_M, + CaseConfigParamInput_EFConstruction_PgVectoRS, + CaseConfigParamInput_Nlist, + CaseConfigParamInput_Nprobe, + CaseConfigParamInput_QuantizationType_PgVectoRS, + CaseConfigParamInput_QuantizationRatio_PgVectoRS, +] + CASE_CONFIG_MAP = { DB.Milvus: { CaseType.CapacityDim960: MilvusLoadConfig, @@ -268,7 +316,7 @@ class CaseConfigInput(BaseModel): CaseType.Performance1536D5M1P: MilvusPerformanceConfig, CaseType.Performance1536D500K1P: MilvusPerformanceConfig, CaseType.Performance1536D5M99P: MilvusPerformanceConfig, - CaseType.Performance1536D500K99P: MilvusPerformanceConfig, + CaseType.Performance1536D500K99P: MilvusPerformanceConfig, }, DB.WeaviateCloud: { CaseType.CapacityDim960: WeaviateLoadConfig, @@ -321,4 +369,21 @@ class CaseConfigInput(BaseModel): CaseType.Performance1536D5M99P: PgVectorPerformanceConfig, CaseType.Performance1536D500K99P: PgVectorPerformanceConfig, }, + DB.PgVectoRS: { + CaseType.CapacityDim960: PgVectoRSLoadingConfig, + CaseType.CapacityDim128: PgVectoRSLoadingConfig, + CaseType.Performance768D100M: PgVectoRSPerformanceConfig, + CaseType.Performance768D10M: PgVectoRSPerformanceConfig, + CaseType.Performance768D1M: PgVectoRSPerformanceConfig, + CaseType.Performance768D10M1P: PgVectoRSPerformanceConfig, + CaseType.Performance768D1M1P: PgVectoRSPerformanceConfig, + CaseType.Performance768D10M99P: PgVectoRSPerformanceConfig, + CaseType.Performance768D1M99P: PgVectoRSPerformanceConfig, + CaseType.Performance1536D5M: PgVectoRSPerformanceConfig, + CaseType.Performance1536D500K: PgVectoRSPerformanceConfig, + CaseType.Performance1536D5M1P: PgVectoRSPerformanceConfig, + CaseType.Performance1536D500K1P: PgVectoRSPerformanceConfig, + CaseType.Performance1536D5M99P: PgVectorPerformanceConfig, + CaseType.Performance1536D500K99P: PgVectoRSPerformanceConfig, + }, } diff --git a/vectordb_bench/frontend/const/styles.py b/vectordb_bench/frontend/const/styles.py index dd69897e8..52d1017a9 100644 --- a/vectordb_bench/frontend/const/styles.py +++ b/vectordb_bench/frontend/const/styles.py @@ -43,8 +43,9 @@ def getPatternShape(i): DB.QdrantCloud: "https://assets.zilliz.com/qdrant_b691674fcd.png", DB.WeaviateCloud: "https://assets.zilliz.com/weaviate_4f6f171ebe.png", DB.PgVector: "https://assets.zilliz.com/PG_Vector_d464f2ef5f.png", + DB.PgVectoRS: "https://assets.zilliz.com/PG_Vector_d464f2ef5f.png", DB.Redis: "https://assets.zilliz.com/Redis_Cloud_74b8bfef39.png", - DB.Chroma: "https://assets.zilliz.com/chroma_ceb3f06ed7.png", + DB.Chroma: "https://assets.zilliz.com/chroma_ceb3f06ed7.png", } # RedisCloud color: #0D6EFD diff --git a/vectordb_bench/models.py b/vectordb_bench/models.py index 4e32181b4..2f9575db3 100644 --- a/vectordb_bench/models.py +++ b/vectordb_bench/models.py @@ -24,11 +24,11 @@ class LoadTimeoutError(TimeoutError): pass + class PerformanceTimeoutError(TimeoutError): pass - class CaseConfigParamType(Enum): """ Value will be the key of CaseConfig.params and displayed in UI @@ -45,6 +45,8 @@ class CaseConfigParamType(Enum): numCandidates = "num_candidates" lists = "lists" probes = "probes" + quantizationType = "quantizationType" + quantizationRatio = "quantizationRatio" class CustomizedCase(BaseModel): @@ -104,7 +106,6 @@ def flush(self): db=db.value.lower(), ) - def get_db_results(self) -> dict[DB, CaseResult]: db2case = {} for res in self.results: From 8fb33af59ddd3b04dc7628e9e9db46a90b3fcd35 Mon Sep 17 00:00:00 2001 From: "min.tian" Date: Tue, 28 Nov 2023 11:31:18 +0800 Subject: [PATCH 024/327] fix bugs: milvus _preload Signed-off-by: min.tian --- vectordb_bench/backend/clients/milvus/milvus.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/vectordb_bench/backend/clients/milvus/milvus.py b/vectordb_bench/backend/clients/milvus/milvus.py index b3ed56047..12b46dfb1 100644 --- a/vectordb_bench/backend/clients/milvus/milvus.py +++ b/vectordb_bench/backend/clients/milvus/milvus.py @@ -132,11 +132,11 @@ def _pre_load(self, coll: Collection): try: if not coll.has_index(index_name=self._index_name): log.info(f"{self.name} create index") - coll.create_index( - self._vector_field, - self.case_config.index_param(), - index_name=self._index_name, - ) + coll.create_index( + self._vector_field, + self.case_config.index_param(), + index_name=self._index_name, + ) coll.load() log.info(f"{self.name} load") From 6fc38fc46ec96b93dcf3582c85b8a4a310c32604 Mon Sep 17 00:00:00 2001 From: Eric Preston Date: Mon, 18 Dec 2023 20:34:36 -0600 Subject: [PATCH 025/327] Send password, batch pipeline (#247) * Send password, batch pipeline * set default batch to 1000 and use i for count --- vectordb_bench/backend/clients/redis/redis.py | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/vectordb_bench/backend/clients/redis/redis.py b/vectordb_bench/backend/clients/redis/redis.py index e3cf8890c..8acf669d2 100644 --- a/vectordb_bench/backend/clients/redis/redis.py +++ b/vectordb_bench/backend/clients/redis/redis.py @@ -29,8 +29,8 @@ def __init__( self.collection_name = INDEX_NAME # Create a redis connection, if db has password configured, add it to the connection here and in init(): - # password=self.db_config["password"] - conn = redis.Redis(host=self.db_config["host"], port=self.db_config["port"], db=0) + password=self.db_config["password"] + conn = redis.Redis(host=self.db_config["host"], port=self.db_config["port"], password=password, db=0) if drop_old: @@ -54,7 +54,7 @@ def make_index(self, vector_dimensions: int, conn: redis.Redis): TagField("id"), NumericField("metadata"), VectorField("vector", # Vector Field Name - "FLAT", { # Vector Index Type: FLAT or HNSW + "HNSW", { # Vector Index Type: FLAT or HNSW "TYPE": "FLOAT32", # FLOAT32 or FLOAT64 "DIM": vector_dimensions, # Number of Vector Dimensions "DISTANCE_METRIC": "COSINE", # Vector Search Distance Metric @@ -75,7 +75,7 @@ def init(self) -> None: >>> with self.init(): >>> self.insert_embeddings() """ - self.conn = redis.Redis(host=self.db_config["host"], port=self.db_config["port"], db=0) + self.conn = redis.Redis(host=self.db_config["host"], port=self.db_config["port"], password=self.db_config["password"], db=0) yield self.conn.close() self.conn = None @@ -94,7 +94,6 @@ def optimize(self) -> None: def insert_embeddings( - self, embeddings: list[list[float]], metadata: list[int], @@ -103,8 +102,10 @@ def insert_embeddings( """Insert embeddings into the database. Should call self.init() first. """ + + batch_size = 1000 # Adjust this as needed, but don't make too big try: - with self.conn.pipeline() as pipe: + with self.conn.pipeline(transaction=False) as pipe: for i, embedding in enumerate(embeddings): embedding = np.array(embedding).astype(np.float32) pipe.hset(metadata[i], mapping = { @@ -112,11 +113,16 @@ def insert_embeddings( "metadata": metadata[i], "vector": embedding.tobytes(), }) + # Execute the pipe so we don't keep too much in memory at once + if i % batch_size == 0: + res = pipe.execute() + res = pipe.execute() + result_len = i + 1 except Exception as e: return 0, e - return len(res), None + return result_len, None def search_embedding( self, From d69c6789bcdaebe2055ac2c448ff60d910f8c368 Mon Sep 17 00:00:00 2001 From: devanshsa5 Date: Wed, 20 Dec 2023 13:37:00 +0530 Subject: [PATCH 026/327] Correcting SIFT datasize for capacity case --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d656e8dc4..418391d1b 100644 --- a/README.md +++ b/README.md @@ -126,7 +126,7 @@ Our client module is designed with flexibility and extensibility in mind, aiming We've developed an array of 15 comprehensive benchmark cases to test vector databases' various capabilities, each designed to give you a different piece of the puzzle. These cases are categorized into three main types: #### Capacity Case - **Large Dim:** Tests the database's loading capacity by inserting large-dimension vectors (GIST 100K vectors, 960 dimensions) until fully loaded. The final number of inserted vectors is reported. -- **Small Dim:** Similar to the Large Dim case but uses small-dimension vectors (SIFT 100K vectors, 128 dimensions). +- **Small Dim:** Similar to the Large Dim case but uses small-dimension vectors (SIFT 500K vectors, 128 dimensions). #### Search Performance Case - **XLarge Dataset:** Measures search performance with a massive dataset (LAION 100M vectors, 768 dimensions) at varying parallel levels. The results include index building time, recall, latency, and maximum QPS. - **Large Dataset:** Similar to the XLarge Dataset case, but uses a slightly smaller dataset (10M-768dim, 5M-1536dim). @@ -140,7 +140,7 @@ For a quick reference, here is a table summarizing the key aspects of each case: Case No. | Case Type | Dataset Size | Filtering Rate | Results | |----------|-----------|--------------|----------------|---------| -1 | Capacity Case | SIFT 100K vectors, 128 dimensions | N/A | Number of inserted vectors | +1 | Capacity Case | SIFT 500K vectors, 128 dimensions | N/A | Number of inserted vectors | 2 | Capacity Case | GIST 100K vectors, 960 dimensions | N/A | Number of inserted vectors | 3 | Search Performance Case | LAION 100M vectors, 768 dimensions | N/A | Index building time, recall, latency, maximum QPS | 4 | Search Performance Case | Cohere 10M vectors, 768 dimensions | N/A | Index building time, recall, latency, maximum QPS | From 2bfb1122791643523511386abc97de80ddd83dfc Mon Sep 17 00:00:00 2001 From: "min.tian" Date: Wed, 10 Jan 2024 14:59:33 +0800 Subject: [PATCH 027/327] support gpu_index_type for milvus Signed-off-by: min.tian --- vectordb_bench/__init__.py | 1 + vectordb_bench/backend/clients/api.py | 8 +- .../backend/clients/milvus/config.py | 83 ++++++++++- .../backend/clients/milvus/milvus.py | 2 +- vectordb_bench/backend/runner/mp_runner.py | 2 +- .../frontend/const/dbCaseConfigs.py | 140 +++++++++++++++++- vectordb_bench/models.py | 121 ++++++++++----- 7 files changed, 312 insertions(+), 45 deletions(-) diff --git a/vectordb_bench/__init__.py b/vectordb_bench/__init__.py index 4bc5488a5..efbec71be 100644 --- a/vectordb_bench/__init__.py +++ b/vectordb_bench/__init__.py @@ -16,6 +16,7 @@ class config: DROP_OLD = env.bool("DROP_OLD", True) USE_SHUFFLED_DATA = env.bool("USE_SHUFFLED_DATA", True) + NUM_CONCURRENCY = [1, 5, 10, 15, 20, 25, 30, 35] RESULTS_LOCAL_DIR = pathlib.Path(__file__).parent.joinpath("results") diff --git a/vectordb_bench/backend/clients/api.py b/vectordb_bench/backend/clients/api.py index 7cc311722..1e3455fd7 100644 --- a/vectordb_bench/backend/clients/api.py +++ b/vectordb_bench/backend/clients/api.py @@ -19,6 +19,9 @@ class IndexType(str, Enum): Flat = "FLAT" AUTOINDEX = "AUTOINDEX" ES_HNSW = "hnsw" + GPU_IVF_FLAT = "GPU_IVF_FLAT" + GPU_IVF_PQ = "GPU_IVF_PQ" + GPU_CAGRA = "GPU_CAGRA" class DBConfig(ABC, BaseModel): @@ -49,6 +52,7 @@ def not_empty_field(cls, v, field): class DBCaseConfig(ABC): """Case specific vector database configs, usually uesed for index params like HNSW""" + @abstractmethod def index_param(self) -> dict: raise NotImplementedError @@ -60,7 +64,9 @@ def search_param(self) -> dict: class EmptyDBCaseConfig(BaseModel, DBCaseConfig): """EmptyDBCaseConfig will be used if the vector database has no case specific configs""" + null: str | None = None + def index_param(self) -> dict: return {} @@ -108,7 +114,7 @@ def __init__( @abstractmethod @contextmanager def init(self) -> None: - """ create and destory connections to database. + """create and destory connections to database. Examples: >>> with self.init(): diff --git a/vectordb_bench/backend/clients/milvus/config.py b/vectordb_bench/backend/clients/milvus/config.py index 86a73850c..1a4b16a7c 100644 --- a/vectordb_bench/backend/clients/milvus/config.py +++ b/vectordb_bench/backend/clients/milvus/config.py @@ -19,8 +19,8 @@ def parse_metric(self) -> str: if not self.metric_type: return "" - if self.metric_type == MetricType.COSINE: - return MetricType.L2.value + # if self.metric_type == MetricType.COSINE: + # return MetricType.L2.value return self.metric_type.value @@ -39,6 +39,7 @@ def search_param(self) -> dict: "metric_type": self.parse_metric(), } + class HNSWConfig(MilvusIndexConfig, DBCaseConfig): M: int efConstruction: int @@ -112,11 +113,87 @@ def search_param(self) -> dict: "params": {}, } + +class GPUIVFFlatConfig(MilvusIndexConfig, DBCaseConfig): + nlist: int = 1024 + nprobe: int = 64 + index: IndexType = IndexType.GPU_IVF_FLAT + + def index_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "index_type": self.index.value, + "params": {"nlist": self.nlist}, + } + + def search_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "params": {"nprobe": self.nprobe}, + } + + +class GPUIVFPQConfig(MilvusIndexConfig, DBCaseConfig): + nlist: int = 1024 + m: int = 0 + nbits: int = 8 + nprobe: int = 32 + index: IndexType = IndexType.GPU_IVF_PQ + + def index_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "index_type": self.index.value, + "params": {"nlist": self.nlist, "m": self.m, "nbits": self.nbits}, + } + + def search_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "params": {"nprobe": self.nprobe}, + } + + +class GPUCAGRAConfig(MilvusIndexConfig, DBCaseConfig): + intermediate_graph_degree: int = 64 + graph_degree: int = 32 + itopk_size: int = 128 + team_size: int = 0 + search_width: int = 4 + min_iterations: int = 0 + max_iterations: int = 0 + index: IndexType = IndexType.GPU_CAGRA + + def index_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "index_type": self.index.value, + "params": { + "intermediate_graph_degree": self.intermediate_graph_degree, + "graph_degree": self.graph_degree, + }, + } + + def search_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "params": { + "team_size": self.team_size, + "search_width": self.search_width, + "itopk_size": self.itopk_size, + "min_iterations": self.min_iterations, + "max_iterations": self.max_iterations, + }, + } + + _milvus_case_config = { IndexType.AUTOINDEX: AutoIndexConfig, IndexType.HNSW: HNSWConfig, IndexType.DISKANN: DISKANNConfig, IndexType.IVFFlat: IVFFlatConfig, IndexType.Flat: FLATConfig, + IndexType.GPU_IVF_FLAT: GPUIVFFlatConfig, + IndexType.GPU_IVF_PQ: GPUIVFPQConfig, + IndexType.GPU_CAGRA: GPUCAGRAConfig, } - diff --git a/vectordb_bench/backend/clients/milvus/milvus.py b/vectordb_bench/backend/clients/milvus/milvus.py index 12b46dfb1..b92e1c59b 100644 --- a/vectordb_bench/backend/clients/milvus/milvus.py +++ b/vectordb_bench/backend/clients/milvus/milvus.py @@ -151,7 +151,7 @@ def optimize(self): def need_normalize_cosine(self) -> bool: """Wheather this database need to normalize dataset to support COSINE""" - return True + return False def insert_embeddings( self, diff --git a/vectordb_bench/backend/runner/mp_runner.py b/vectordb_bench/backend/runner/mp_runner.py index 86e245a1a..25e383bf7 100644 --- a/vectordb_bench/backend/runner/mp_runner.py +++ b/vectordb_bench/backend/runner/mp_runner.py @@ -26,7 +26,7 @@ def __init__( test_data: list[list[float]], k: int = 100, filters: dict | None = None, - concurrencies: Iterable[int] = (1, 5, 10, 15, 20, 25, 30, 35), + concurrencies: Iterable[int] = config.NUM_CONCURRENCY, duration: int = 30, ): self.db = db diff --git a/vectordb_bench/frontend/const/dbCaseConfigs.py b/vectordb_bench/frontend/const/dbCaseConfigs.py index 1298983ff..61cf29d58 100644 --- a/vectordb_bench/frontend/const/dbCaseConfigs.py +++ b/vectordb_bench/frontend/const/dbCaseConfigs.py @@ -63,6 +63,9 @@ class CaseConfigInput(BaseModel): IndexType.DISKANN.value, IndexType.Flat.value, IndexType.AUTOINDEX.value, + IndexType.GPU_IVF_FLAT.value, + IndexType.GPU_IVF_PQ.value, + IndexType.GPU_CAGRA.value, ], }, ) @@ -189,10 +192,14 @@ class CaseConfigInput(BaseModel): inputConfig={ "min": 1, "max": 65536, - "value": 1000, + "value": 1024, }, isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) - == IndexType.IVFFlat.value, + in [ + IndexType.IVFFlat.value, + IndexType.GPU_IVF_FLAT.value, + IndexType.GPU_IVF_PQ.value, + ], ) CaseConfigParamInput_Nprobe = CaseConfigInput( @@ -201,10 +208,122 @@ class CaseConfigInput(BaseModel): inputConfig={ "min": 1, "max": 65536, - "value": 10, + "value": 64, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) + in [ + IndexType.IVFFlat.value, + IndexType.GPU_IVF_FLAT.value, + IndexType.GPU_IVF_PQ.value, + ], +) + +CaseConfigParamInput_M_PQ = CaseConfigInput( + label=CaseConfigParamType.m, + inputType=InputType.Number, + inputConfig={ + "min": 0, + "max": 65536, + "value": 0, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) + in [IndexType.GPU_IVF_PQ.value], +) + +CaseConfigParamInput_Nbits_PQ = CaseConfigInput( + label=CaseConfigParamType.nbits, + inputType=InputType.Number, + inputConfig={ + "min": 1, + "max": 65536, + "value": 8, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) + in [IndexType.GPU_IVF_PQ.value], +) + +CaseConfigParamInput_intermediate_graph_degree = CaseConfigInput( + label=CaseConfigParamType.intermediate_graph_degree, + inputType=InputType.Number, + inputConfig={ + "min": 1, + "max": 65536, + "value": 64, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) + in [IndexType.GPU_CAGRA.value], +) + +CaseConfigParamInput_graph_degree = CaseConfigInput( + label=CaseConfigParamType.graph_degree, + inputType=InputType.Number, + inputConfig={ + "min": 1, + "max": 65536, + "value": 32, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) + in [IndexType.GPU_CAGRA.value], +) + +CaseConfigParamInput_itopk_size = CaseConfigInput( + label=CaseConfigParamType.itopk_size, + inputType=InputType.Number, + inputConfig={ + "min": 1, + "max": 65536, + "value": 128, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) + in [IndexType.GPU_CAGRA.value], +) + +CaseConfigParamInput_team_size = CaseConfigInput( + label=CaseConfigParamType.team_size, + inputType=InputType.Number, + inputConfig={ + "min": 0, + "max": 65536, + "value": 0, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) + in [IndexType.GPU_CAGRA.value], +) + +CaseConfigParamInput_search_width = CaseConfigInput( + label=CaseConfigParamType.search_width, + inputType=InputType.Number, + inputConfig={ + "min": 0, + "max": 65536, + "value": 4, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) + in [IndexType.GPU_CAGRA.value], +) + +CaseConfigParamInput_min_iterations = CaseConfigInput( + label=CaseConfigParamType.min_iterations, + inputType=InputType.Number, + inputConfig={ + "min": 0, + "max": 65536, + "value": 0, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) + in [IndexType.GPU_CAGRA.value], +) + +CaseConfigParamInput_max_iterations = CaseConfigInput( + label=CaseConfigParamType.max_iterations, + inputType=InputType.Number, + inputConfig={ + "min": 0, + "max": 65536, + "value": 0, }, isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) - == IndexType.IVFFlat.value, + in [IndexType.GPU_CAGRA.value], ) CaseConfigParamInput_Lists = CaseConfigInput( @@ -250,6 +369,10 @@ class CaseConfigInput(BaseModel): CaseConfigParamInput_M, CaseConfigParamInput_EFConstruction_Milvus, CaseConfigParamInput_Nlist, + CaseConfigParamInput_M_PQ, + CaseConfigParamInput_Nbits_PQ, + CaseConfigParamInput_intermediate_graph_degree, + CaseConfigParamInput_graph_degree, ] MilvusPerformanceConfig = [ CaseConfigParamInput_IndexType, @@ -259,6 +382,15 @@ class CaseConfigInput(BaseModel): CaseConfigParamInput_SearchList, CaseConfigParamInput_Nlist, CaseConfigParamInput_Nprobe, + CaseConfigParamInput_M_PQ, + CaseConfigParamInput_Nbits_PQ, + CaseConfigParamInput_intermediate_graph_degree, + CaseConfigParamInput_graph_degree, + CaseConfigParamInput_itopk_size, + CaseConfigParamInput_team_size, + CaseConfigParamInput_search_width, + CaseConfigParamInput_min_iterations, + CaseConfigParamInput_max_iterations, ] WeaviateLoadConfig = [ diff --git a/vectordb_bench/models.py b/vectordb_bench/models.py index 2f9575db3..7d63139c1 100644 --- a/vectordb_bench/models.py +++ b/vectordb_bench/models.py @@ -47,6 +47,15 @@ class CaseConfigParamType(Enum): probes = "probes" quantizationType = "quantizationType" quantizationRatio = "quantizationRatio" + m = "m" + nbits = "nbits" + intermediate_graph_degree = "intermediate_graph_degree" + graph_degree = "graph_degree" + itopk_size = "itopk_size" + team_size = "team_size" + search_width = "search_width" + min_iterations = "min_iterations" + max_iterations = "max_iterations" class CustomizedCase(BaseModel): @@ -90,7 +99,7 @@ class TestResult(BaseModel): task_label: str results: list[CaseResult] - file_fmt: str = "result_{}_{}_{}.json" # result_20230718_statndard_milvus.json + file_fmt: str = "result_{}_{}_{}.json" # result_20230718_statndard_milvus.json def flush(self): db2case = self.get_db_results() @@ -100,9 +109,8 @@ def flush(self): self.write_db_file( result_dir=result_root.joinpath(db.value), partial=TestResult( - run_id=self.run_id, - task_label=self.task_label, - results=result), + run_id=self.run_id, task_label=self.task_label, results=result + ), db=db.value.lower(), ) @@ -120,7 +128,9 @@ def write_db_file(self, result_dir: pathlib.Path, partial: Self, db: str): log.info(f"local result directory not exist, creating it: {result_dir}") result_dir.mkdir(parents=True) - file_name = self.file_fmt.format(date.today().strftime("%Y%m%d"), partial.task_label, db) + file_name = self.file_fmt.format( + date.today().strftime("%Y%m%d"), partial.task_label, db + ) result_file = result_dir.joinpath(file_name) if result_file.exists(): log.warning( @@ -146,9 +156,7 @@ def read_file(cls, full_path: pathlib.Path, trans_unit: bool = False) -> Self: task_config = case_result.get("task_config") db = DB(task_config.get("db")) - task_config["db_config"] = db.config_cls( - **task_config["db_config"] - ) + task_config["db_config"] = db.config_cls(**task_config["db_config"]) task_config["db_case_config"] = db.case_config_cls( index_type=task_config["db_case_config"].get("index", None), )(**task_config["db_case_config"]) @@ -173,31 +181,59 @@ def read_file(cls, full_path: pathlib.Path, trans_unit: bool = False) -> Self: def display(self, dbs: list[DB] | None = None): filter_list = dbs if dbs and isinstance(dbs, list) else None - sorted_results = sorted(self.results, key=lambda x: ( - x.task_config.db.name, - x.task_config.db_config.db_label, - x.task_config.case_config.case_id.name, - ), reverse=True) + sorted_results = sorted( + self.results, + key=lambda x: ( + x.task_config.db.name, + x.task_config.db_config.db_label, + x.task_config.case_config.case_id.name, + ), + reverse=True, + ) - filtered_results = [r for r in sorted_results if not filter_list or r.task_config.db not in filter_list] + filtered_results = [ + r + for r in sorted_results + if not filter_list or r.task_config.db not in filter_list + ] def append_return(x, y): x.append(y) return x max_db = max(map(len, [f.task_config.db.name for f in filtered_results])) - max_db_labels = max(map(len, [f.task_config.db_config.db_label for f in filtered_results])) + 3 - max_case = max(map(len, [f.task_config.case_config.case_id.name for f in filtered_results])) - max_load_dur = max(map(len, [str(f.metrics.load_duration) for f in filtered_results])) + 3 + max_db_labels = ( + max(map(len, [f.task_config.db_config.db_label for f in filtered_results])) + + 3 + ) + max_case = max( + map(len, [f.task_config.case_config.case_id.name for f in filtered_results]) + ) + max_load_dur = ( + max(map(len, [str(f.metrics.load_duration) for f in filtered_results])) + 3 + ) max_qps = max(map(len, [str(f.metrics.qps) for f in filtered_results])) + 3 - max_recall = max(map(len, [str(f.metrics.recall) for f in filtered_results])) + 3 + max_recall = ( + max(map(len, [str(f.metrics.recall) for f in filtered_results])) + 3 + ) max_db_labels = 8 if max_db_labels < 8 else max_db_labels max_load_dur = 11 if max_load_dur < 11 else max_load_dur max_qps = 10 if max_qps < 10 else max_qps max_recall = 13 if max_recall < 13 else max_recall - LENGTH = (max_db, max_db_labels, max_case, len(self.task_label), max_load_dur, max_qps, 15, max_recall, 14, 5) + LENGTH = ( + max_db, + max_db_labels, + max_case, + len(self.task_label), + max_load_dur, + max_qps, + 15, + max_recall, + 14, + 5, + ) DATA_FORMAT = ( f"%-{max_db}s | %-{max_db_labels}s %-{max_case}s %-{len(self.task_label)}s" @@ -206,25 +242,40 @@ def append_return(x, y): ) TITLE = DATA_FORMAT % ( - "DB", "db_label", "case", "label", "load_dur", "qps", "latency(p99)", "recall", "max_load_count", "label") - SPLIT = DATA_FORMAT%tuple(map(lambda x:"-"*x, LENGTH)) - SUMMARY_FORMAT = ("Task summary: run_id=%s, task_label=%s") % (self.run_id[:5], self.task_label) + "DB", + "db_label", + "case", + "label", + "load_dur", + "qps", + "latency(p99)", + "recall", + "max_load_count", + "label", + ) + SPLIT = DATA_FORMAT % tuple(map(lambda x: "-" * x, LENGTH)) + SUMMARY_FORMAT = ("Task summary: run_id=%s, task_label=%s") % ( + self.run_id[:5], + self.task_label, + ) fmt = [SUMMARY_FORMAT, TITLE, SPLIT] - for f in filtered_results: - fmt.append(DATA_FORMAT%( - f.task_config.db.name, - f.task_config.db_config.db_label, - f.task_config.case_config.case_id.name, - self.task_label, - f.metrics.load_duration, - f.metrics.qps, - f.metrics.serial_latency_p99, - f.metrics.recall, - f.metrics.max_load_count, - f.label.value, - )) + fmt.append( + DATA_FORMAT + % ( + f.task_config.db.name, + f.task_config.db_config.db_label, + f.task_config.case_config.case_id.name, + self.task_label, + f.metrics.load_duration, + f.metrics.qps, + f.metrics.serial_latency_p99, + f.metrics.recall, + f.metrics.max_load_count, + f.label.value, + ) + ) tmp_logger = logging.getLogger("no_color") for f in fmt: From 2b37f9e2be4efd1de433a9fdefcd9b53c49947f1 Mon Sep 17 00:00:00 2001 From: "min.tian" Date: Wed, 17 Jan 2024 14:58:47 +0800 Subject: [PATCH 028/327] ui support setting drop_old and dataset_address Signed-off-by: min.tian --- vectordb_bench/__init__.py | 1 + .../components/run_test/submitTask.py | 26 +++++++++++++++++-- vectordb_bench/interface.py | 15 ++++++++++- 3 files changed, 39 insertions(+), 3 deletions(-) diff --git a/vectordb_bench/__init__.py b/vectordb_bench/__init__.py index efbec71be..b31c75358 100644 --- a/vectordb_bench/__init__.py +++ b/vectordb_bench/__init__.py @@ -11,6 +11,7 @@ class config: LOG_LEVEL = env.str("LOG_LEVEL", "INFO") DEFAULT_DATASET_URL = env.str("DEFAULT_DATASET_URL", "assets.zilliz.com/benchmark/") + DEFAULT_DATASET_URL_ALIYUN = env.str("DEFAULT_DATASET_URL", "assets.zilliz.com.cn/benchmark/") DATASET_LOCAL_DIR = env.path("DATASET_LOCAL_DIR", "/tmp/vectordb_bench/dataset") NUM_PER_BATCH = env.int("NUM_PER_BATCH", 5000) diff --git a/vectordb_bench/frontend/components/run_test/submitTask.py b/vectordb_bench/frontend/components/run_test/submitTask.py index 150f636c2..22d34b4f5 100644 --- a/vectordb_bench/frontend/components/run_test/submitTask.py +++ b/vectordb_bench/frontend/components/run_test/submitTask.py @@ -34,10 +34,32 @@ def taskLabelInput(st): return taskLabel +def advancedSettings(st): + container = st.columns([1, 2]) + index_already_exists = container[0].checkbox("Index already exists", value=False) + container[1].caption("if actived, inserting and building will be skipped.") + + container = st.columns([1, 2]) + use_aliyun = container[0].checkbox("Dataset from Aliyun (Shanghai)", value=False) + container[1].caption( + "if actived, the dataset will be downloaded from Aliyun OSS shanghai, default AWS S3 aws-us-west." + ) + + return index_already_exists, use_aliyun + + def controlPanel(st, tasks, taskLabel, isAllValid): + index_already_exists, use_aliyun = advancedSettings(st) + + def runHandler(): + benchMarkRunner.set_drop_old(not index_already_exists) + benchMarkRunner.set_download_address(use_aliyun) + benchMarkRunner.run(tasks, taskLabel) + + def stopHandler(): + benchMarkRunner.stop_running() + isRunning = benchMarkRunner.has_running() - runHandler = lambda: benchMarkRunner.run(tasks, taskLabel) - stopHandler = lambda: benchMarkRunner.stop_running() if isRunning: currentTaskId = benchMarkRunner.get_current_task_id() diff --git a/vectordb_bench/interface.py b/vectordb_bench/interface.py index 360fbc64f..96d6265a8 100644 --- a/vectordb_bench/interface.py +++ b/vectordb_bench/interface.py @@ -38,6 +38,14 @@ class BenchMarkRunner: def __init__(self): self.running_task: TaskRunner | None = None self.latest_error: str | None = None + self.drop_old: bool = True + + def set_drop_old(self, drop_old: bool): + self.drop_old = drop_old + + def set_download_address(self, use_aliyun: bool): + # todo + pass def run(self, tasks: list[TaskConfig], task_label: str | None = None) -> bool: """run all the tasks in the configs, write one result into the path""" @@ -145,7 +153,12 @@ def _async_task_v2(self, running_task: TaskRunner, send_conn: Connection) -> Non task_config=runner.config, ) - drop_old = False if latest_runner and runner == latest_runner else config.DROP_OLD + # drop_old = False if latest_runner and runner == latest_runner else config.DROP_OLD + drop_old = config.DROP_OLD + if latest_runner and runner == latest_runner: + drop_old = False + elif not self.drop_old: + drop_old = False try: log.info(f"[{idx+1}/{running_task.num_cases()}] start case: {runner.display()}, drop_old={drop_old}") case_res.metrics = runner.run(drop_old) From dfd4ddbf876b387439d61d4955308a8d0109438a Mon Sep 17 00:00:00 2001 From: "min.tian" Date: Thu, 7 Dec 2023 10:31:54 +0800 Subject: [PATCH 029/327] update the scoring rules of latency, from 0 to 100, the lower the latency, the higher the score, the better the performance. Signed-off-by: min.tian --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 418391d1b..2cdeeb460 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ To facilitate the presentation of test results and provide a comprehensive perfo 1. For each case, select a base value and score each system based on relative values. - For QPS and QP$, we use the highest value as the reference, denoted as `base_QPS` or `base_QP$`, and the score of each system is `(QPS/base_QPS) * 100` or `(QP$/base_QP$) * 100`. - - For Latency, we use the lowest value as the reference, that is, `base_Latency`, and the score of each system is `(Latency + 10ms)/(base_Latency + 10ms)`. + - For Latency, we use the lowest value as the reference, that is, `base_Latency`, and the score of each system is `(base_Latency + 10ms)/(Latency + 10ms) * 100`. We want to give equal weight to different cases, and not let a case with high absolute result values become the sole reason for the overall scoring. Therefore, when scoring different systems in each case, we need to use relative values. From f0f6ee64c388c93ed043f00ebe969ebafdc2e2f2 Mon Sep 17 00:00:00 2001 From: "min.tian" Date: Thu, 18 Jan 2024 14:43:45 +0800 Subject: [PATCH 030/327] skip compaction for milvus gpu index. Signed-off-by: min.tian --- vectordb_bench/backend/clients/milvus/milvus.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/vectordb_bench/backend/clients/milvus/milvus.py b/vectordb_bench/backend/clients/milvus/milvus.py index b92e1c59b..218e20397 100644 --- a/vectordb_bench/backend/clients/milvus/milvus.py +++ b/vectordb_bench/backend/clients/milvus/milvus.py @@ -3,13 +3,13 @@ import logging import time from contextlib import contextmanager -from typing import Iterable, Type +from typing import Iterable from pymilvus import Collection, utility from pymilvus import CollectionSchema, DataType, FieldSchema, MilvusException -from ..api import VectorDB, DBCaseConfig, DBConfig, IndexType -from .config import MilvusConfig, _milvus_case_config +from ..api import VectorDB, IndexType +from .config import MilvusIndexConfig log = logging.getLogger(__name__) @@ -21,7 +21,7 @@ def __init__( self, dim: int, db_config: dict, - db_case_config: DBCaseConfig, + db_case_config: MilvusIndexConfig, collection_name: str = "VectorDBBenchCollection", drop_old: bool = False, name: str = "Milvus", @@ -98,6 +98,11 @@ def _optimize(self): def _post_insert(self): log.info(f"{self.name} post insert before optimize") + + if self.case_config.index in [IndexType.GPU_CAGRA, IndexType.GPU_IVF_FLAT, IndexType.GPU_IVF_PQ]: + log.debug("skip compaction for gpu index type.") + return + try: self.col.flush() # wait for index done and load refresh From 77ca3578a1c79cbc797a3fd6cd0848286b1c6285 Mon Sep 17 00:00:00 2001 From: "yusheng.ma" Date: Thu, 18 Jan 2024 14:37:12 +0800 Subject: [PATCH 031/327] adition param for raft Signed-off-by: yusheng.ma --- .../backend/clients/milvus/config.py | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/vectordb_bench/backend/clients/milvus/config.py b/vectordb_bench/backend/clients/milvus/config.py index 1a4b16a7c..03ab1b42f 100644 --- a/vectordb_bench/backend/clients/milvus/config.py +++ b/vectordb_bench/backend/clients/milvus/config.py @@ -117,19 +117,24 @@ def search_param(self) -> dict: class GPUIVFFlatConfig(MilvusIndexConfig, DBCaseConfig): nlist: int = 1024 nprobe: int = 64 + cache_dataset_on_device: str + refine_ratio: float | None = None index: IndexType = IndexType.GPU_IVF_FLAT def index_param(self) -> dict: return { "metric_type": self.parse_metric(), "index_type": self.index.value, - "params": {"nlist": self.nlist}, + "params": { + "nlist": self.nlist, + "cache_dataset_on_device": self.cache_dataset_on_device, + }, } def search_param(self) -> dict: return { "metric_type": self.parse_metric(), - "params": {"nprobe": self.nprobe}, + "params": {"nprobe": self.nprobe, "refine_ratio": self.refine_ratio}, } @@ -138,19 +143,26 @@ class GPUIVFPQConfig(MilvusIndexConfig, DBCaseConfig): m: int = 0 nbits: int = 8 nprobe: int = 32 + refine_ratio: float | None = None + cache_dataset_on_device: str index: IndexType = IndexType.GPU_IVF_PQ def index_param(self) -> dict: return { "metric_type": self.parse_metric(), "index_type": self.index.value, - "params": {"nlist": self.nlist, "m": self.m, "nbits": self.nbits}, + "params": { + "nlist": self.nlist, + "m": self.m, + "nbits": self.nbits, + "cache_dataset_on_device": self.cache_dataset_on_device, + }, } def search_param(self) -> dict: return { "metric_type": self.parse_metric(), - "params": {"nprobe": self.nprobe}, + "params": {"nprobe": self.nprobe, "refine_ratio": self.refine_ratio}, } @@ -162,6 +174,9 @@ class GPUCAGRAConfig(MilvusIndexConfig, DBCaseConfig): search_width: int = 4 min_iterations: int = 0 max_iterations: int = 0 + build_algo: str = "IVF_PQ" # IVF_PQ; NN_DESCENT; + cache_dataset_on_device: str + refine_ratio: float | None = None index: IndexType = IndexType.GPU_CAGRA def index_param(self) -> dict: @@ -171,6 +186,8 @@ def index_param(self) -> dict: "params": { "intermediate_graph_degree": self.intermediate_graph_degree, "graph_degree": self.graph_degree, + "build_algo": self.build_algo, + "cache_dataset_on_device": self.cache_dataset_on_device, }, } @@ -183,6 +200,7 @@ def search_param(self) -> dict: "itopk_size": self.itopk_size, "min_iterations": self.min_iterations, "max_iterations": self.max_iterations, + "refine_ratio": self.refine_ratio, }, } From c0f0286d137cf1099bf2e86fab7a98f99fa0953e Mon Sep 17 00:00:00 2001 From: "min.tian" Date: Thu, 18 Jan 2024 15:12:24 +0800 Subject: [PATCH 032/327] more params setting for milvus gpu index Signed-off-by: min.tian --- .../components/run_test/caseSelector.py | 4 +- .../frontend/const/dbCaseConfigs.py | 47 +++++++++++++++++++ vectordb_bench/models.py | 3 ++ 3 files changed, 52 insertions(+), 2 deletions(-) diff --git a/vectordb_bench/frontend/components/run_test/caseSelector.py b/vectordb_bench/frontend/components/run_test/caseSelector.py index 5dfc672bb..56521c831 100644 --- a/vectordb_bench/frontend/components/run_test/caseSelector.py +++ b/vectordb_bench/frontend/components/run_test/caseSelector.py @@ -78,8 +78,8 @@ def caseConfigSetting(st, allCaseConfigs, case, activedDbList): elif config.inputType == InputType.Number: caseConfig[config.label] = column.number_input( config.label.value, - format="%d", - step=1, + # format="%d", + step=config.inputConfig.get("step", 1), min_value=config.inputConfig["min"], max_value=config.inputConfig["max"], key=key, diff --git a/vectordb_bench/frontend/const/dbCaseConfigs.py b/vectordb_bench/frontend/const/dbCaseConfigs.py index 61cf29d58..f351ad9b2 100644 --- a/vectordb_bench/frontend/const/dbCaseConfigs.py +++ b/vectordb_bench/frontend/const/dbCaseConfigs.py @@ -326,6 +326,48 @@ class CaseConfigInput(BaseModel): in [IndexType.GPU_CAGRA.value], ) +CaseConfigParamInput_build_algo = CaseConfigInput( + label=CaseConfigParamType.build_algo, + inputType=InputType.Option, + inputConfig={ + "options": ["IVF_PQ", "NN_DESCENT"], + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) + in [IndexType.GPU_CAGRA.value], +) + + +CaseConfigParamInput_cache_dataset_on_device = CaseConfigInput( + label=CaseConfigParamType.cache_dataset_on_device, + inputType=InputType.Option, + inputConfig={ + "options": ["false", "true"], + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) + in [ + IndexType.GPU_CAGRA.value, + IndexType.GPU_IVF_PQ.value, + IndexType.GPU_IVF_FLAT.value, + ], +) + +CaseConfigParamInput_refine_ratio = CaseConfigInput( + label=CaseConfigParamType.refine_ratio, + inputType=InputType.Number, + inputConfig={ + "min": 1.0, + "max": 2.0, + "value": 1.0, + "step": 0.01, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) + in [ + IndexType.GPU_CAGRA.value, + IndexType.GPU_IVF_PQ.value, + IndexType.GPU_IVF_FLAT.value, + ], +) + CaseConfigParamInput_Lists = CaseConfigInput( label=CaseConfigParamType.lists, inputType=InputType.Number, @@ -373,6 +415,8 @@ class CaseConfigInput(BaseModel): CaseConfigParamInput_Nbits_PQ, CaseConfigParamInput_intermediate_graph_degree, CaseConfigParamInput_graph_degree, + CaseConfigParamInput_build_algo, + CaseConfigParamInput_cache_dataset_on_device, ] MilvusPerformanceConfig = [ CaseConfigParamInput_IndexType, @@ -391,6 +435,9 @@ class CaseConfigInput(BaseModel): CaseConfigParamInput_search_width, CaseConfigParamInput_min_iterations, CaseConfigParamInput_max_iterations, + CaseConfigParamInput_build_algo, + CaseConfigParamInput_cache_dataset_on_device, + CaseConfigParamInput_refine_ratio, ] WeaviateLoadConfig = [ diff --git a/vectordb_bench/models.py b/vectordb_bench/models.py index 7d63139c1..fc53d78db 100644 --- a/vectordb_bench/models.py +++ b/vectordb_bench/models.py @@ -56,6 +56,9 @@ class CaseConfigParamType(Enum): search_width = "search_width" min_iterations = "min_iterations" max_iterations = "max_iterations" + build_algo = "build_algo" + cache_dataset_on_device = "cache_dataset_on_device" + refine_ratio = "refine_ratio" class CustomizedCase(BaseModel): From 949caed25586a4f237620af945c5c4590a53f4c5 Mon Sep 17 00:00:00 2001 From: "min.tian" Date: Thu, 18 Jan 2024 15:46:03 +0800 Subject: [PATCH 033/327] add test results of zilliz_cloud_beta_2024_01 Signed-off-by: min.tian --- vectordb_bench/frontend/vdb_benchmark.py | 1 + ...sult_20240105_beta_202401_zillizcloud.json | 979 ++++++++++++++++++ 2 files changed, 980 insertions(+) create mode 100644 vectordb_bench/results/ZillizCloud/result_20240105_beta_202401_zillizcloud.json diff --git a/vectordb_bench/frontend/vdb_benchmark.py b/vectordb_bench/frontend/vdb_benchmark.py index d9269b6b5..0be43470e 100644 --- a/vectordb_bench/frontend/vdb_benchmark.py +++ b/vectordb_bench/frontend/vdb_benchmark.py @@ -20,6 +20,7 @@ def main(): allResults = benchMarkRunner.get_results() st.title("Vector Database Benchmark") + st.caption("Note that all testing was completed in July 2023, except for the times already noted.") # results selector and filter resultSelectorContainer = st.sidebar.container() diff --git a/vectordb_bench/results/ZillizCloud/result_20240105_beta_202401_zillizcloud.json b/vectordb_bench/results/ZillizCloud/result_20240105_beta_202401_zillizcloud.json new file mode 100644 index 000000000..7c1f3667c --- /dev/null +++ b/vectordb_bench/results/ZillizCloud/result_20240105_beta_202401_zillizcloud.json @@ -0,0 +1,979 @@ +{ + "run_id": "0ae10e14e34c4c3c9a7116e7a9591d01", + "task_label": "zilliz-beta-202401", + "results": [ + { + "metrics": { + "max_load_count": 0, + "load_duration": 861.8295, + "qps": 5115.5303, + "serial_latency_p99": 0.0087, + "recall": 0.9469 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf-beta-202401", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 10, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 861.8295, + "qps": 3685.0767, + "serial_latency_p99": 0.0091, + "recall": 0.9736 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf-beta-202401", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 12, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 861.8295, + "qps": 4742.1617, + "serial_latency_p99": 0.0091, + "recall": 0.9936 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf-beta-202401", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 14, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 888.3719, + "qps": 6054.4428, + "serial_latency_p99": 0.0081, + "recall": 0.9155 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf-beta-202401", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 5, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 888.3719, + "qps": 4104.2598, + "serial_latency_p99": 0.0106, + "recall": 0.9506 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf-beta-202401", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 7, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 888.3719, + "qps": 4252.1267, + "serial_latency_p99": 0.0092, + "recall": 0.9964 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf-beta-202401", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 9, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 7920.7947, + "qps": 1685.3091, + "serial_latency_p99": 0.0133, + "recall": 0.9718 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf-beta-202401", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 11, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 7920.7947, + "qps": 769.8991, + "serial_latency_p99": 0.0107, + "recall": 0.9884 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf-beta-202401", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 13, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 7920.7947, + "qps": 945.4061, + "serial_latency_p99": 0.0109, + "recall": 0.9941 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf-beta-202401", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 15, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 9149.0776, + "qps": 2214.9028, + "serial_latency_p99": 0.0084, + "recall": 0.9249 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf-beta-202401", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 4, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 9149.0776, + "qps": 827.975, + "serial_latency_p99": 0.012, + "recall": 0.9692 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf-beta-202401", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 6, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 9149.0776, + "qps": 776.9454, + "serial_latency_p99": 0.0114, + "recall": 0.9966 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf-beta-202401", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 8, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 2689.1534, + "qps": 269.5464, + "serial_latency_p99": 0.0098, + "recall": 0.9776 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-cap-beta-202401", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 10, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 2689.1534, + "qps": 240.0363, + "serial_latency_p99": 0.0106, + "recall": 0.9822 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-cap-beta-202401", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 12, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 2689.1534, + "qps": 218.0627, + "serial_latency_p99": 0.0114, + "recall": 0.9936 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-cap-beta-202401", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 14, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 2431.0279, + "qps": 392.8825, + "serial_latency_p99": 0.0069, + "recall": 0.9581 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-cap-beta-202401", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 5, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 2431.0279, + "qps": 343.8204, + "serial_latency_p99": 0.0084, + "recall": 0.968 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-cap-beta-202401", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 7, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 2431.0279, + "qps": 216.6773, + "serial_latency_p99": 0.0101, + "recall": 0.9968 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-cap-beta-202401", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 9, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1730.3904, + "qps": 503.2284, + "serial_latency_p99": 0.009, + "recall": 0.9677 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "2cu-cap-beta-202401", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 10, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1730.3904, + "qps": 413.3232, + "serial_latency_p99": 0.0096, + "recall": 0.981 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "2cu-cap-beta-202401", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 12, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1730.3904, + "qps": 425.5492, + "serial_latency_p99": 0.0102, + "recall": 0.9936 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "2cu-cap-beta-202401", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 14, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1779.991, + "qps": 789.1229, + "serial_latency_p99": 0.0056, + "recall": 0.9396 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "2cu-cap-beta-202401", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 5, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1779.991, + "qps": 571.4257, + "serial_latency_p99": 0.0077, + "recall": 0.9668 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "2cu-cap-beta-202401", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 7, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 1779.991, + "qps": 411.7653, + "serial_latency_p99": 0.0091, + "recall": 0.9968 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "2cu-cap-beta-202401", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 9, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 12608.007, + "qps": 98.0448, + "serial_latency_p99": 0.0161, + "recall": 0.9803 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "2cu-cap-beta-202401", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 11, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 12608.007, + "qps": 58.3152, + "serial_latency_p99": 0.028, + "recall": 0.9891 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "2cu-cap-beta-202401", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 13, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 12608.007, + "qps": 46.8304, + "serial_latency_p99": 0.0286, + "recall": 0.9941 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "2cu-cap-beta-202401", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 15, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 14895.0657, + "qps": 170.5693, + "serial_latency_p99": 0.0089, + "recall": 0.9605 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "2cu-cap-beta-202401", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 4, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 14895.0657, + "qps": 94.7766, + "serial_latency_p99": 0.0196, + "recall": 0.9843 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "2cu-cap-beta-202401", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 6, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 14895.0657, + "qps": 44.8695, + "serial_latency_p99": 0.0309, + "recall": 0.9966 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "2cu-cap-beta-202401", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 8, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 949.3081, + "qps": 722.0315, + "serial_latency_p99": 0.0077, + "recall": 0.976 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-perf-beta-202401", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 10, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 949.3081, + "qps": 467.5795, + "serial_latency_p99": 0.0088, + "recall": 0.9898 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-perf-beta-202401", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 12, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 949.3081, + "qps": 975.2503, + "serial_latency_p99": 0.0082, + "recall": 0.9936 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-perf-beta-202401", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 14, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 960.7296, + "qps": 871.5513, + "serial_latency_p99": 0.0213, + "recall": 0.9471 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-perf-beta-202401", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 5, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 960.7296, + "qps": 544.6203, + "serial_latency_p99": 0.0084, + "recall": 0.977 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-perf-beta-202401", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 7, + "custom_case": {} + } + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 960.7296, + "qps": 930.9164, + "serial_latency_p99": 0.0096, + "recall": 0.9968 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-perf-beta-202401", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 9, + "custom_case": {} + } + }, + "label": ":)" + } + ], + "file_fmt": "result_{}_{}_{}.json" +} \ No newline at end of file From 178fd2f1fc0fda35aad25d30d9b565b685aa16ef Mon Sep 17 00:00:00 2001 From: "min.tian" Date: Thu, 18 Jan 2024 18:52:50 +0800 Subject: [PATCH 034/327] [Milvus client] Skip compaction if use GPU indexType Signed-off-by: min.tian --- vectordb_bench/backend/clients/milvus/milvus.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/vectordb_bench/backend/clients/milvus/milvus.py b/vectordb_bench/backend/clients/milvus/milvus.py index 218e20397..4e8a35275 100644 --- a/vectordb_bench/backend/clients/milvus/milvus.py +++ b/vectordb_bench/backend/clients/milvus/milvus.py @@ -98,11 +98,6 @@ def _optimize(self): def _post_insert(self): log.info(f"{self.name} post insert before optimize") - - if self.case_config.index in [IndexType.GPU_CAGRA, IndexType.GPU_IVF_FLAT, IndexType.GPU_IVF_PQ]: - log.debug("skip compaction for gpu index type.") - return - try: self.col.flush() # wait for index done and load refresh @@ -121,9 +116,14 @@ def wait_index(): time.sleep(5) wait_index() - self.col.compact() - self.col.wait_for_compaction_completed() - wait_index() + + # Skip compaction if use GPU indexType + if self.case_config.index in [IndexType.GPU_CAGRA, IndexType.GPU_IVF_FLAT, IndexType.GPU_IVF_PQ]: + log.debug("skip compaction for gpu index type.") + else : + self.col.compact() + self.col.wait_for_compaction_completed() + wait_index() except Exception as e: log.warning(f"{self.name} optimize error: {e}") From c53082c7dbfc51a53def1ac30df70ace3cfafdb4 Mon Sep 17 00:00:00 2001 From: yangxuan Date: Thu, 18 Jan 2024 11:16:15 +0800 Subject: [PATCH 035/327] Enable aliyun OSS Add data_source.py, vdb bench now can download dataset from Aliyun OSS. Signed-off-by: yangxuan --- pyproject.toml | 1 + tests/test_data_source.py | 78 ++++++++++ tests/test_dataset.py | 33 ++++- vectordb_bench/__init__.py | 6 +- vectordb_bench/backend/assembler.py | 14 +- vectordb_bench/backend/data_source.py | 204 ++++++++++++++++++++++++++ vectordb_bench/backend/dataset.py | 195 ++++++++++-------------- vectordb_bench/backend/task_runner.py | 6 +- vectordb_bench/interface.py | 16 +- vectordb_bench/log_util.py | 1 - 10 files changed, 418 insertions(+), 136 deletions(-) create mode 100644 tests/test_data_source.py create mode 100644 vectordb_bench/backend/data_source.py diff --git a/pyproject.toml b/pyproject.toml index f73bc2940..075ec92be 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,7 @@ dependencies = [ "streamlit_extras", "tqdm", "s3fs", + "oss2", "psutil", "polars", "plotly", diff --git a/tests/test_data_source.py b/tests/test_data_source.py new file mode 100644 index 000000000..a2b1d0ed8 --- /dev/null +++ b/tests/test_data_source.py @@ -0,0 +1,78 @@ +import logging +import pathlib +import pytest +from vectordb_bench.backend.data_source import AliyunOSSReader, AwsS3Reader +from vectordb_bench.backend.dataset import Dataset, DatasetManager + +log = logging.getLogger(__name__) + +class TestReader: + @pytest.mark.parametrize("size", [ + 100_000, + 1_000_000, + 10_000_000, + ]) + def test_cohere(self, size): + cohere = Dataset.COHERE.manager(size) + self.per_dataset_test(cohere) + + @pytest.mark.parametrize("size", [ + 100_000, + 1_000_000, + ]) + def test_gist(self, size): + gist = Dataset.GIST.manager(size) + self.per_dataset_test(gist) + + @pytest.mark.parametrize("size", [ + 1_000_000, + ]) + def test_glove(self, size): + glove = Dataset.GLOVE.manager(size) + self.per_dataset_test(glove) + + @pytest.mark.parametrize("size", [ + 500_000, + 5_000_000, + # 50_000_000, + ]) + def test_sift(self, size): + sift = Dataset.SIFT.manager(size) + self.per_dataset_test(sift) + + @pytest.mark.parametrize("size", [ + 50_000, + 500_000, + 5_000_000, + ]) + def test_openai(self, size): + openai = Dataset.OPENAI.manager(size) + self.per_dataset_test(openai) + + + def per_dataset_test(self, dataset: DatasetManager): + s3_reader = AwsS3Reader() + all_files = s3_reader.ls_all(dataset.data.dir_name) + + + remote_f_names = [] + for file in all_files: + remote_f = pathlib.Path(file).name + if dataset.data.use_shuffled and remote_f.startswith("train"): + continue + + elif (not dataset.data.use_shuffled) and remote_f.startswith("shuffle"): + continue + + remote_f_names.append(remote_f) + + + assert set(dataset.data.files) == set(remote_f_names) + + aliyun_reader = AliyunOSSReader() + for fname in dataset.data.files: + p = pathlib.Path("benchmark", dataset.data.dir_name, fname) + assert aliyun_reader.bucket.object_exists(p.as_posix()) + + log.info(f"downloading to {dataset.data_dir}") + aliyun_reader.read(dataset.data.dir_name.lower(), dataset.data.files, dataset.data_dir) diff --git a/tests/test_dataset.py b/tests/test_dataset.py index f0578c218..56c3715e5 100644 --- a/tests/test_dataset.py +++ b/tests/test_dataset.py @@ -1,4 +1,4 @@ -from vectordb_bench.backend.dataset import Dataset +from vectordb_bench.backend.dataset import Dataset, get_files import logging import pytest from pydantic import ValidationError @@ -34,3 +34,34 @@ def test_iter_cohere(self): for i in cohere_10m: log.debug(i.head(1)) + +class TestGetFiles: + @pytest.mark.parametrize("train_count", [ + 1, + 10, + 50, + 100, + ]) + @pytest.mark.parametrize("with_gt", [True, False]) + def test_train_count(self, train_count, with_gt): + files = get_files(train_count, True, with_gt) + log.info(files) + + if with_gt: + assert len(files) - 4 == train_count + else: + assert len(files) - 1 == train_count + + @pytest.mark.parametrize("use_shuffled", [True, False]) + def test_use_shuffled(self, use_shuffled): + files = get_files(1, use_shuffled, True) + log.info(files) + + trains = [f for f in files if "train" in f] + if use_shuffled: + for t in trains: + assert "shuffle_train" in t + else: + for t in trains: + assert "shuffle" not in t + assert "train" in t diff --git a/vectordb_bench/__init__.py b/vectordb_bench/__init__.py index b31c75358..eca190832 100644 --- a/vectordb_bench/__init__.py +++ b/vectordb_bench/__init__.py @@ -8,10 +8,12 @@ env.read_env(".env") class config: + ALIYUN_OSS_URL = "assets.zilliz.com.cn/benchmark/" + AWS_S3_URL = "assets.zilliz.com/benchmark/" + LOG_LEVEL = env.str("LOG_LEVEL", "INFO") - DEFAULT_DATASET_URL = env.str("DEFAULT_DATASET_URL", "assets.zilliz.com/benchmark/") - DEFAULT_DATASET_URL_ALIYUN = env.str("DEFAULT_DATASET_URL", "assets.zilliz.com.cn/benchmark/") + DEFAULT_DATASET_URL = env.str("DEFAULT_DATASET_URL", AWS_S3_URL) DATASET_LOCAL_DIR = env.path("DATASET_LOCAL_DIR", "/tmp/vectordb_bench/dataset") NUM_PER_BATCH = env.int("NUM_PER_BATCH", 5000) diff --git a/vectordb_bench/backend/assembler.py b/vectordb_bench/backend/assembler.py index 6aaec0b63..6b0e3c81d 100644 --- a/vectordb_bench/backend/assembler.py +++ b/vectordb_bench/backend/assembler.py @@ -2,6 +2,7 @@ from .task_runner import CaseRunner, RunningStatus, TaskRunner from ..models import TaskConfig from ..backend.clients import EmptyDBCaseConfig +from ..backend.data_source import DatasetSource import logging @@ -10,7 +11,7 @@ class Assembler: @classmethod - def assemble(cls, run_id , task: TaskConfig) -> CaseRunner: + def assemble(cls, run_id , task: TaskConfig, source: DatasetSource) -> CaseRunner: c_cls = task.case_config.case_id.case_cls c = c_cls() @@ -22,14 +23,21 @@ def assemble(cls, run_id , task: TaskConfig) -> CaseRunner: config=task, ca=c, status=RunningStatus.PENDING, + dataset_source=source, ) return runner @classmethod - def assemble_all(cls, run_id: str, task_label: str, tasks: list[TaskConfig]) -> TaskRunner: + def assemble_all( + cls, + run_id: str, + task_label: str, + tasks: list[TaskConfig], + source: DatasetSource, + ) -> TaskRunner: """group by case type, db, and case dataset""" - runners = [cls.assemble(run_id, task) for task in tasks] + runners = [cls.assemble(run_id, task, source) for task in tasks] load_runners = [r for r in runners if r.ca.label == CaseLabel.Load] perf_runners = [r for r in runners if r.ca.label == CaseLabel.Performance] diff --git a/vectordb_bench/backend/data_source.py b/vectordb_bench/backend/data_source.py new file mode 100644 index 000000000..0398ec653 --- /dev/null +++ b/vectordb_bench/backend/data_source.py @@ -0,0 +1,204 @@ +import logging +import pathlib +import typing +from enum import Enum +from tqdm import tqdm +from hashlib import md5 +import os +from abc import ABC, abstractmethod + +from .. import config + +logging.getLogger("s3fs").setLevel(logging.CRITICAL) + +log = logging.getLogger(__name__) + +DatasetReader = typing.TypeVar("DatasetReader") + +class DatasetSource(Enum): + S3 = "S3" + AliyunOSS = "AliyunOSS" + + def reader(self) -> DatasetReader: + if self == DatasetSource.S3: + return AwsS3Reader() + + if self == DatasetSource.AliyunOSS: + return AliyunOSSReader() + + +class DatasetReader(ABC): + source: DatasetSource + remote_root: str + + @abstractmethod + def read(self, dataset: str, files: list[str], local_ds_root: pathlib.Path, check_etag: bool = True): + """read dataset files from remote_root to local_ds_root, + + Args: + dataset(str): for instance "sift_small_500k" + files(list[str]): all filenames of the dataset + local_ds_root(pathlib.Path): whether to write the remote data. + check_etag(bool): whether to check the etag + """ + pass + + @abstractmethod + def validate_file(self, remote: pathlib.Path, local: pathlib.Path) -> bool: + pass + + +class AliyunOSSReader(DatasetReader): + source: DatasetSource = DatasetSource.AliyunOSS + remote_root: str = config.ALIYUN_OSS_URL + + def __init__(self): + import oss2 + self.bucket = oss2.Bucket(oss2.AnonymousAuth(), self.remote_root, "benchmark", True) + + def validate_file(self, remote: pathlib.Path, local: pathlib.Path, check_etag: bool) -> bool: + info = self.bucket.get_object_meta(remote.as_posix()) + + # check size equal + remote_size, local_size = info.content_length, os.path.getsize(local) + if remote_size != local_size: + log.info(f"local file: {local} size[{local_size}] not match with remote size[{remote_size}]") + return False + + # check etag equal + if check_etag: + return match_etag(info.etag.strip('"').lower(), local) + + + return True + + def read(self, dataset: str, files: list[str], local_ds_root: pathlib.Path, check_etag: bool = False): + downloads = [] + if not local_ds_root.exists(): + log.info(f"local dataset root path not exist, creating it: {local_ds_root}") + local_ds_root.mkdir(parents=True) + downloads = [(pathlib.Path("benchmark", dataset, f), local_ds_root.joinpath(f)) for f in files] + + else: + for file in files: + remote_file = pathlib.Path("benchmark", dataset, file) + local_file = local_ds_root.joinpath(file) + + if (not local_file.exists()) or (not self.validate_file(remote_file, local_file, check_etag)): + log.info(f"local file: {local_file} not match with remote: {remote_file}; add to downloading list") + downloads.append((remote_file, local_file)) + + if len(downloads) == 0: + return + + log.info(f"Start to downloading files, total count: {len(downloads)}") + for remote_file, local_file in tqdm(downloads): + log.debug(f"downloading file {remote_file} to {local_ds_root}") + self.bucket.get_object_to_file(remote_file.as_posix(), local_file.as_posix()) + + log.info(f"Succeed to download all files, downloaded file count = {len(downloads)}") + + + +class AwsS3Reader(DatasetReader): + source: DatasetSource = DatasetSource.S3 + remote_root: str = config.AWS_S3_URL + + def __init__(self): + import s3fs + self.fs = s3fs.S3FileSystem( + anon=True, + client_kwargs={'region_name': 'us-west-2'} + ) + + def ls_all(self, dataset: str): + dataset_root_dir = pathlib.Path(self.remote_root, dataset) + log.info(f"listing dataset: {dataset_root_dir}") + names = self.fs.ls(dataset_root_dir) + for n in names: + log.info(n) + return names + + + def read(self, dataset: str, files: list[str], local_ds_root: pathlib.Path, check_etag: bool = True): + downloads = [] + if not local_ds_root.exists(): + log.info(f"local dataset root path not exist, creating it: {local_ds_root}") + local_ds_root.mkdir(parents=True) + downloads = [pathlib.Path(self.remote_root, dataset, f) for f in files] + + else: + for file in files: + remote_file = pathlib.Path(self.remote_root, dataset, file) + local_file = local_ds_root.joinpath(file) + + if (not local_file.exists()) or (not self.validate_file(remote_file, local_file, check_etag)): + log.info(f"local file: {local_file} not match with remote: {remote_file}; add to downloading list") + downloads.append(remote_file) + + if len(downloads) == 0: + return + + log.info(f"Start to downloading files, total count: {len(downloads)}") + for s3_file in tqdm(downloads): + log.debug(f"downloading file {s3_file} to {local_ds_root}") + self.fs.download(s3_file, local_ds_root.as_posix()) + + log.info(f"Succeed to download all files, downloaded file count = {len(downloads)}") + + + def validate_file(self, remote: pathlib.Path, local: pathlib.Path, check_etag: bool) -> bool: + # info() uses ls() inside, maybe we only need to ls once + info = self.fs.info(remote) + + # check size equal + remote_size, local_size = info.get("size"), os.path.getsize(local) + if remote_size != local_size: + log.info(f"local file: {local} size[{local_size}] not match with remote size[{remote_size}]") + return False + + # check etag equal + if check_etag: + return match_etag(info.get('ETag', "").strip('"'), local) + + return True + + +def match_etag(expected_etag: str, local_file) -> bool: + """Check if local files' etag match with S3""" + def factor_of_1MB(filesize, num_parts): + x = filesize / int(num_parts) + y = x % 1048576 + return int(x + 1048576 - y) + + def calc_etag(inputfile, partsize): + md5_digests = [] + with open(inputfile, 'rb') as f: + for chunk in iter(lambda: f.read(partsize), b''): + md5_digests.append(md5(chunk).digest()) + return md5(b''.join(md5_digests)).hexdigest() + '-' + str(len(md5_digests)) + + def possible_partsizes(filesize, num_parts): + return lambda partsize: partsize < filesize and (float(filesize) / float(partsize)) <= num_parts + + filesize = os.path.getsize(local_file) + le = "" + if '-' not in expected_etag: # no spliting uploading + with open(local_file, 'rb') as f: + le = md5(f.read()).hexdigest() + log.debug(f"calculated local etag {le}, expected etag: {expected_etag}") + return expected_etag == le + else: + num_parts = int(expected_etag.split('-')[-1]) + partsizes = [ ## Default Partsizes Map + 8388608, # aws_cli/boto3 + 15728640, # s3cmd + factor_of_1MB(filesize, num_parts) # Used by many clients to upload large files + ] + + for partsize in filter(possible_partsizes(filesize, num_parts), partsizes): + le = calc_etag(local_file, partsize) + log.debug(f"calculated local etag {le}, expected etag: {expected_etag}") + if expected_etag == le: + return True + return False diff --git a/vectordb_bench/backend/dataset.py b/vectordb_bench/backend/dataset.py index 8c2b0ceef..46c31bce5 100644 --- a/vectordb_bench/backend/dataset.py +++ b/vectordb_bench/backend/dataset.py @@ -4,14 +4,11 @@ >>> Dataset.Cohere.get(100_000) """ -import os +from collections import namedtuple import logging import pathlib -from hashlib import md5 from enum import Enum -import s3fs import pandas as pd -from tqdm import tqdm from pydantic import validator, PrivateAttr import polars as pl from pyarrow.parquet import ParquetFile @@ -20,17 +17,21 @@ from .. import config from ..backend.clients import MetricType from . import utils +from .data_source import DatasetSource, DatasetReader log = logging.getLogger(__name__) +SizeLabel = namedtuple('SizeLabel', ['size', 'label', 'files']) + + class BaseDataset(BaseModel): name: str size: int dim: int metric_type: MetricType use_shuffled: bool - _size_label: dict = PrivateAttr() + _size_label: dict[int, SizeLabel] = PrivateAttr() @validator("size") def verify_size(cls, v): @@ -40,19 +41,51 @@ def verify_size(cls, v): @property def label(self) -> str: - return self._size_label.get(self.size) + return self._size_label.get(self.size).label @property def dir_name(self) -> str: return f"{self.name}_{self.label}_{utils.numerize(self.size)}".lower() + @property + def files(self) -> str: + return self._size_label.get(self.size).files + + +def get_files(train_count: int, use_shuffled: bool, with_gt: bool = True) -> list[str]: + prefix = "shuffle_train" if use_shuffled else "train" + middle = f"of-{train_count}" + surfix = "parquet" + + train_files = [] + if train_count > 1: + just_size = len(str(train_count)) + for i in range(train_count): + sub_file = f"{prefix}-{str(i).rjust(just_size, '0')}-{middle}.{surfix}" + train_files.append(sub_file) + else: + train_files.append(f"{prefix}.{surfix}") + + files = ['test.parquet'] + if with_gt: + files.extend([ + 'neighbors.parquet', + 'neighbors_tail_1p.parquet', + 'neighbors_head_1p.parquet', + ]) + + files.extend(train_files) + return files + class LAION(BaseDataset): name: str = "LAION" dim: int = 768 metric_type: MetricType = MetricType.L2 use_shuffled: bool = False - _size_label: dict = {100_000_000: "LARGE"} + _size_label: dict = { + 100_000_000: SizeLabel(100_000_000, "LARGE", get_files(100, False)), + } class GIST(BaseDataset): @@ -61,8 +94,8 @@ class GIST(BaseDataset): metric_type: MetricType = MetricType.L2 use_shuffled: bool = False _size_label: dict = { - 100_000: "SMALL", - 1_000_000: "MEDIUM", + 100_000: SizeLabel(100_000, "SMALL", get_files(1, False, False)), + 1_000_000: SizeLabel(1_000_000, "MEDIUM", get_files(1, False, False)), } @@ -72,9 +105,9 @@ class Cohere(BaseDataset): metric_type: MetricType = MetricType.COSINE use_shuffled: bool = config.USE_SHUFFLED_DATA _size_label: dict = { - 100_000: "SMALL", - 1_000_000: "MEDIUM", - 10_000_000: "LARGE", + 100_000: SizeLabel(100_000, "SMALL", get_files(1, config.USE_SHUFFLED_DATA)), + 1_000_000: SizeLabel(1_000_000, "MEDIUM", get_files(1, config.USE_SHUFFLED_DATA)), + 10_000_000: SizeLabel(10_000_000, "LARGE", get_files(10, config.USE_SHUFFLED_DATA)), } @@ -83,7 +116,7 @@ class Glove(BaseDataset): dim: int = 200 metric_type: MetricType = MetricType.COSINE use_shuffled: bool = False - _size_label: dict = {1_000_000: "MEDIUM"} + _size_label: dict = {1_000_000: SizeLabel(1_000_000, "MEDIUM", get_files(1, False, False))} class SIFT(BaseDataset): @@ -92,25 +125,26 @@ class SIFT(BaseDataset): metric_type: MetricType = MetricType.L2 use_shuffled: bool = False _size_label: dict = { - 500_000: "SMALL", - 5_000_000: "MEDIUM", - 50_000_000: "LARGE", + 500_000: SizeLabel(500_000, "SMALL", get_files(1, False, False)), + 5_000_000: SizeLabel(5_000_000, "MEDIUM", get_files(1, False, False)), + # 50_000_000: SizeLabel(50_000_000, "LARGE", get_files(50, False, False)), } + class OpenAI(BaseDataset): name: str = "OpenAI" dim: int = 1536 metric_type: MetricType = MetricType.COSINE use_shuffled: bool = config.USE_SHUFFLED_DATA _size_label: dict = { - 50_000: "SMALL", - 500_000: "MEDIUM", - 5_000_000: "LARGE", + 50_000: SizeLabel(50_000, "SMALL", get_files(1, config.USE_SHUFFLED_DATA)), + 500_000: SizeLabel(500_000, "MEDIUM", get_files(1, config.USE_SHUFFLED_DATA)), + 5_000_000: SizeLabel(5_000_000, "LARGE", get_files(10, config.USE_SHUFFLED_DATA)), } class DatasetManager(BaseModel): - """Download dataset if not int the local directory. Provide data for cases. + """Download dataset if not in the local directory. Provide data for cases. DatasetManager is iterable, each iteration will return the next batch of data in pandas.DataFrame @@ -122,12 +156,16 @@ class DatasetManager(BaseModel): data: BaseDataset test_data: pd.DataFrame | None = None train_files : list[str] = [] + reader: DatasetReader | None = None def __eq__(self, obj): if isinstance(obj, DatasetManager): return self.data.name == obj.data.name and self.data.label == obj.data.label return False + def set_reader(self, reader: DatasetReader): + self.reader = reader + @property def data_dir(self) -> pathlib.Path: """ data local directory: config.DATASET_LOCAL_DIR/{dataset_name}/{dataset_dirname} @@ -139,108 +177,12 @@ def data_dir(self) -> pathlib.Path: """ return pathlib.Path(config.DATASET_LOCAL_DIR, self.data.name.lower(), self.data.dir_name.lower()) - @property - def download_dir(self) -> str: - """ data s3 directory: config.DEFAULT_DATASET_URL/{dataset_dirname} - - Examples: - >>> sift_s = Dataset.SIFT.manager(500_000) - >>> sift_s.download_dir - 'assets.zilliz.com/benchmark/sift_small_500k' - """ - return f"{config.DEFAULT_DATASET_URL}{self.data.dir_name}" - def __iter__(self): return DataSetIterator(self) - def _validate_local_file(self): - if not self.data_dir.exists(): - log.info(f"local file path not exist, creating it: {self.data_dir}") - self.data_dir.mkdir(parents=True) - - fs = s3fs.S3FileSystem( - anon=True, - client_kwargs={'region_name': 'us-west-2'} - ) - dataset_info = fs.ls(self.download_dir, detail=True) - if len(dataset_info) == 0: - raise ValueError(f"No data in s3 for dataset: {self.download_dir}") - path2etag = {info['Key']: info['ETag'].split('"')[1] for info in dataset_info} - - perfix_to_filter = "train" if self.data.use_shuffled else "shuffle_train" - filtered_keys = [key for key in path2etag.keys() if key.split("/")[-1].startswith(perfix_to_filter)] - for k in filtered_keys: - path2etag.pop(k) - - # get local files ended with '.parquet' - file_names = [p.name for p in self.data_dir.glob("*.parquet")] - log.info(f"local files: {file_names}") - log.info(f"s3 files: {path2etag.keys()}") - downloads = [] - if len(file_names) == 0: - log.info("no local files, set all to downloading lists") - downloads = path2etag.keys() - else: - # if local file exists, check the etag of local file with s3, - # make sure data files aren't corrupted. - for name in tqdm([key.split("/")[-1] for key in path2etag.keys()]): - s3_path = f"{self.download_dir}/{name}" - local_path = self.data_dir.joinpath(name) - log.debug(f"s3 path: {s3_path}, local_path: {local_path}") - if not local_path.exists(): - log.info(f"local file not exists: {local_path}, add to downloading lists") - downloads.append(s3_path) - - elif not self.match_etag(path2etag.get(s3_path), local_path): - log.info(f"local file etag not match with s3 file: {local_path}, add to downloading lists") - downloads.append(s3_path) - - for s3_file in tqdm(downloads): - log.debug(f"downloading file {s3_file} to {self.data_dir}") - fs.download(s3_file, self.data_dir.as_posix()) - - def match_etag(self, expected_etag: str, local_file) -> bool: - """Check if local files' etag match with S3""" - def factor_of_1MB(filesize, num_parts): - x = filesize / int(num_parts) - y = x % 1048576 - return int(x + 1048576 - y) - - def calc_etag(inputfile, partsize): - md5_digests = [] - with open(inputfile, 'rb') as f: - for chunk in iter(lambda: f.read(partsize), b''): - md5_digests.append(md5(chunk).digest()) - return md5(b''.join(md5_digests)).hexdigest() + '-' + str(len(md5_digests)) - - def possible_partsizes(filesize, num_parts): - return lambda partsize: partsize < filesize and (float(filesize) / float(partsize)) <= num_parts - - filesize = os.path.getsize(local_file) - le = "" - if '-' not in expected_etag: # no spliting uploading - with open(local_file, 'rb') as f: - le = md5(f.read()).hexdigest() - log.debug(f"calculated local etag {le}, expected etag: {expected_etag}") - return expected_etag == le - else: - num_parts = int(expected_etag.split('-')[-1]) - partsizes = [ ## Default Partsizes Map - 8388608, # aws_cli/boto3 - 15728640, # s3cmd - factor_of_1MB(filesize, num_parts) # Used by many clients to upload large files - ] - - for partsize in filter(possible_partsizes(filesize, num_parts), partsizes): - le = calc_etag(local_file, partsize) - log.debug(f"calculated local etag {le}, expected etag: {expected_etag}") - if expected_etag == le: - return True - return False - - def prepare(self, check=True) -> bool: - """Download the dataset from S3 - url = f"{config.DEFAULT_DATASET_URL}/{self.data.dir_name}" + def prepare(self, source: DatasetSource=DatasetSource.S3, check: bool=True) -> bool: + """Download the dataset from DatasetSource + url = f"{source}/{self.data.dir_name}" download files from url to self.data_dir, there'll be 4 types of files in the data_dir - train*.parquet: for training @@ -248,9 +190,20 @@ def prepare(self, check=True) -> bool: - neighbors.parquet: ground_truth of the test.parquet - neighbors_head_1p.parquet: ground_truth of the test.parquet after filtering 1% data - neighbors_99p.parquet: ground_truth of the test.parquet after filtering 99% data + + Args: + source(DatasetSource): S3 or AliyunOSS, default as S3 + check(bool): Whether to do etags check + + Returns: + bool: whether the dataset is successfully prepared + """ - if check: - self._validate_local_file() + source.reader().read( + dataset=self.data.dir_name.lower(), + files=self.data.files, + local_ds_root=self.data_dir, + ) prefix = "shuffle_train" if self.data.use_shuffled else "train" self.train_files = sorted([f.name for f in self.data_dir.glob(f'{prefix}*.parquet')]) diff --git a/vectordb_bench/backend/task_runner.py b/vectordb_bench/backend/task_runner.py index 46265297a..80c5ac1df 100644 --- a/vectordb_bench/backend/task_runner.py +++ b/vectordb_bench/backend/task_runner.py @@ -17,6 +17,7 @@ from ..metric import Metric from .runner import MultiProcessingSearchRunner from .runner import SerialSearchRunner, SerialInsertRunner +from .data_source import DatasetSource log = logging.getLogger(__name__) @@ -44,6 +45,7 @@ class CaseRunner(BaseModel): config: TaskConfig ca: Case status: RunningStatus + dataset_source: DatasetSource db: api.VectorDB | None = None test_emb: list[list[float]] | None = None @@ -59,7 +61,7 @@ def __eq__(self, obj): return False def display(self) -> dict: - c_dict = self.ca.dict(include={'label':True, 'filters': True,'dataset':{'data': True} }) + c_dict = self.ca.dict(include={'label':True, 'filters': True,'dataset':{'data': {'name': True, 'size': True, 'dim': True, 'metric_type': True, 'label': True}} }) c_dict['db'] = self.config.db_name return c_dict @@ -82,7 +84,7 @@ def init_db(self, drop_old: bool = True) -> None: def _pre_run(self, drop_old: bool = True): try: self.init_db(drop_old) - self.ca.dataset.prepare() + self.ca.dataset.prepare(self.dataset_source) except ModuleNotFoundError as e: log.warning(f"pre run case error: please install client for db: {self.config.db}, error={e}") raise e from None diff --git a/vectordb_bench/interface.py b/vectordb_bench/interface.py index 96d6265a8..c170c67dc 100644 --- a/vectordb_bench/interface.py +++ b/vectordb_bench/interface.py @@ -23,6 +23,7 @@ from .backend.result_collector import ResultCollector from .backend.assembler import Assembler from .backend.task_runner import TaskRunner +from .backend.data_source import DatasetSource log = logging.getLogger(__name__) @@ -39,13 +40,16 @@ def __init__(self): self.running_task: TaskRunner | None = None self.latest_error: str | None = None self.drop_old: bool = True - + self.dataset_source: DatasetSource = DatasetSource.S3 + def set_drop_old(self, drop_old: bool): self.drop_old = drop_old - + def set_download_address(self, use_aliyun: bool): - # todo - pass + if use_aliyun: + self.dataset_source = DatasetSource.AliyunOSS + else: + self.dataset_source = DatasetSource.S3 def run(self, tasks: list[TaskConfig], task_label: str | None = None) -> bool: """run all the tasks in the configs, write one result into the path""" @@ -58,7 +62,7 @@ def run(self, tasks: list[TaskConfig], task_label: str | None = None) -> bool: log.warning("Empty tasks submitted") return False - log.debug(f"tasks: {tasks}") + log.debug(f"tasks: {tasks}, task_label: {task_label}, dataset source: {self.dataset_source}") # Generate run_id run_id = uuid.uuid4().hex @@ -69,7 +73,7 @@ def run(self, tasks: list[TaskConfig], task_label: str | None = None) -> bool: self.latest_error = "" try: - self.running_task = Assembler.assemble_all(run_id, task_label, tasks) + self.running_task = Assembler.assemble_all(run_id, task_label, tasks, self.dataset_source) self.running_task.display() except ModuleNotFoundError as e: msg = f"Please install client for database, error={e}" diff --git a/vectordb_bench/log_util.py b/vectordb_bench/log_util.py index bbf3f4923..b923bdcd2 100644 --- a/vectordb_bench/log_util.py +++ b/vectordb_bench/log_util.py @@ -1,7 +1,6 @@ import logging from logging import config - def init(log_level): LOGGING = { 'version': 1, From e026d7d5ef23ce31a43e0997624a113ab8959762 Mon Sep 17 00:00:00 2001 From: "been.zino" Date: Tue, 31 Oct 2023 10:37:02 +0900 Subject: [PATCH 036/327] Replace sqlalchemy with psycopg2 SQLAlchemy makes postgresql slow, So replace it with psycopg2 --- .../backend/clients/pgvector/config.py | 20 ++- .../backend/clients/pgvector/pgvector.py | 145 +++++++++--------- 2 files changed, 86 insertions(+), 79 deletions(-) diff --git a/vectordb_bench/backend/clients/pgvector/config.py b/vectordb_bench/backend/clients/pgvector/config.py index d40004b4f..7d90e86d2 100644 --- a/vectordb_bench/backend/clients/pgvector/config.py +++ b/vectordb_bench/backend/clients/pgvector/config.py @@ -6,15 +6,19 @@ class PgVectorConfig(DBConfig): user_name: SecretStr = "postgres" password: SecretStr - url: SecretStr + host: str = "localhost" + port: int = 5432 db_name: str def to_dict(self) -> dict: user_str = self.user_name.get_secret_value() pwd_str = self.password.get_secret_value() - url_str = self.url.get_secret_value() return { - "url" : POSTGRE_URL_PLACEHOLDER%(user_str, pwd_str, url_str, self.db_name) + "host" : self.host, + "port" : self.port, + "dbname" : self.db_name, + "user" : user_str, + "password" : pwd_str } class PgVectorIndexConfig(BaseModel, DBCaseConfig): @@ -29,6 +33,13 @@ def parse_metric(self) -> str: return "vector_ip_ops" return "vector_cosine_ops" + def parse_metric_fun_op(self) -> str: + if self.metric_type == MetricType.L2: + return "<->" + elif self.metric_type == MetricType.IP: + return "<#>" + return "<=>" + def parse_metric_fun_str(self) -> str: if self.metric_type == MetricType.L2: return "l2_distance" @@ -45,5 +56,6 @@ def index_param(self) -> dict: def search_param(self) -> dict: return { "probes" : self.probes, - "metric_fun" : self.parse_metric_fun_str() + "metric_fun" : self.parse_metric_fun_str(), + "metric_fun_op" : self.parse_metric_fun_op(), } \ No newline at end of file diff --git a/vectordb_bench/backend/clients/pgvector/pgvector.py b/vectordb_bench/backend/clients/pgvector/pgvector.py index 7e634a8f5..d34a52e17 100644 --- a/vectordb_bench/backend/clients/pgvector/pgvector.py +++ b/vectordb_bench/backend/clients/pgvector/pgvector.py @@ -3,26 +3,10 @@ import logging from contextlib import contextmanager from typing import Any +import psycopg2 +import psycopg2.extras from ..api import VectorDB, DBCaseConfig -from pgvector.sqlalchemy import Vector -from sqlalchemy import ( - MetaData, - create_engine, - insert, - select, - Index, - Table, - text, - Column, - Float, - Integer -) -from sqlalchemy.orm import ( - declarative_base, - mapped_column, - Session -) log = logging.getLogger(__name__) @@ -47,22 +31,26 @@ def __init__( self._vector_field = "embedding" # construct basic units - pg_engine = create_engine(**self.db_config) - Base = declarative_base() - pq_metadata = Base.metadata - pq_metadata.reflect(pg_engine) + self.conn = psycopg2.connect(**self.db_config) + self.conn.autocommit = False + self.cursor = self.conn.cursor() # create vector extension - with pg_engine.connect() as conn: - conn.execute(text('CREATE EXTENSION IF NOT EXISTS vector')) - conn.commit() + self.cursor.execute('CREATE EXTENSION IF NOT EXISTS vector') + self.conn.commit() - self.pg_table = self._get_table_schema(pq_metadata) - if drop_old and self.table_name in pq_metadata.tables: + if drop_old : log.info(f"Pgvector client drop table : {self.table_name}") # self.pg_table.drop(pg_engine, checkfirst=True) - pq_metadata.drop_all(pg_engine) - self._create_table(dim, pg_engine) + self._drop_index() + self._drop_table() + self._create_table(dim) + self._create_index() + + self.cursor.close() + self.conn.close() + self.cursor = None + self.conn = None @contextmanager def init(self) -> None: @@ -72,18 +60,24 @@ def init(self) -> None: >>> self.insert_embeddings() >>> self.search_embedding() """ - self.pg_engine = create_engine(**self.db_config) - - Base = declarative_base() - pq_metadata = Base.metadata - pq_metadata.reflect(self.pg_engine) - self.pg_session = Session(self.pg_engine) - self.pg_table = self._get_table_schema(pq_metadata) - yield - self.pg_session = None - self.pg_engine = None - del (self.pg_session) - del (self.pg_engine) + self.conn = psycopg2.connect(**self.db_config) + self.conn.autocommit = False + self.cursor = self.conn.cursor() + + try: + yield + finally: + self.cursor.close() + self.conn.close() + self.cursor = None + self.conn = None + + def _drop_table(self): + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + self.cursor.execute(f'DROP TABLE IF EXISTS public."{self.table_name}"') + self.conn.commit() def ready_to_load(self): pass @@ -93,32 +87,31 @@ def optimize(self): def ready_to_search(self): pass - - def _get_table_schema(self, pq_metadata): - return Table( - self.table_name, - pq_metadata, - Column(self._primary_field, Integer, primary_key=True), - Column(self._vector_field, Vector(self.dim)), - extend_existing=True - ) + + def _drop_index(self): + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + self.cursor.execute(f'DROP INDEX IF EXISTS "{self._index_name}"') + self.conn.commit() - def _create_index(self, pg_engine): + def _create_index(self): + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + index_param = self.case_config.index_param() - index = Index(self._index_name, self.pg_table.c.embedding, - postgresql_using='ivfflat', - postgresql_with={'lists': index_param["lists"]}, - postgresql_ops={'embedding': index_param["metric"]} - ) - index.drop(pg_engine, checkfirst = True) - index.create(pg_engine) - - def _create_table(self, dim, pg_engine : int): + self.cursor.execute(f'CREATE INDEX IF NOT EXISTS {self._index_name} ON public."{self.table_name}" USING ivfflat (embedding {index_param["metric"]}) WITH (lists={index_param["lists"]});') + self.conn.commit() + + def _create_table(self, dim : int): + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + try: # create table - self.pg_table.create(bind = pg_engine, checkfirst = True) - # create vec index - self._create_index(pg_engine) + self.cursor.execute(f'CREATE TABLE IF NOT EXISTS public."{self.table_name}" (id BIGINT PRIMARY KEY, embedding vector({dim}));') + self.cursor.execute(f'ALTER TABLE public."{self.table_name}" ALTER COLUMN embedding SET STORAGE PLAIN;') + self.conn.commit() except Exception as e: log.warning(f"Failed to create pgvector table: {self.table_name} error: {e}") raise e from None @@ -129,10 +122,13 @@ def insert_embeddings( metadata: list[int], **kwargs: Any, ) -> (int, Exception): + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + try: items = [dict(id = metadata[i], embedding=embeddings[i]) for i in range(len(metadata))] - self.pg_session.execute(insert(self.pg_table), items) - self.pg_session.commit() + psycopg2.extras.execute_batch(self.cursor, f'INSERT INTO public."{self.table_name}" (id, embedding) VALUES (%(id)s, %(embedding)s)', items) + self.conn.commit() return len(metadata), None except Exception as e: log.warning(f"Failed to insert data into pgvector table ({self.table_name}), error: {e}") @@ -145,15 +141,14 @@ def search_embedding( filters: dict | None = None, timeout: int | None = None, ) -> list[int]: - assert self.pg_table is not None + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + search_param =self.case_config.search_param() - with self.pg_engine.connect() as conn: - conn.execute(text(f'SET ivfflat.probes = {search_param["probes"]}')) - conn.commit() - op_fun = getattr(self.pg_table.c.embedding, search_param["metric_fun"]) - if filters: - res = self.pg_session.scalars(select(self.pg_table).order_by(op_fun(query)).filter(self.pg_table.c.id > filters.get('id')).limit(k)) - else: - res = self.pg_session.scalars(select(self.pg_table).order_by(op_fun(query)).limit(k)) - return list(res) + self.cursor.execute(f'SET ivfflat.probes = {search_param["probes"]}') + self.cursor.execute(f"SELECT id FROM public.\"{self.table_name}\" ORDER BY embedding {search_param['metric_fun_op']} '{query}' LIMIT {k};") + self.conn.commit() + result = self.cursor.fetchall() + + return [int(i[0]) for i in result] From f85d4ab78c2a319788a37c684f93e968598ee268 Mon Sep 17 00:00:00 2001 From: "been.zino" Date: Tue, 31 Oct 2023 16:05:00 +0900 Subject: [PATCH 037/327] Replace INSERT with COPY Bulk insert is too slow. Using copy when it comes to bulk insert. --- vectordb_bench/backend/clients/pgvector/pgvector.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/vectordb_bench/backend/clients/pgvector/pgvector.py b/vectordb_bench/backend/clients/pgvector/pgvector.py index d34a52e17..a0eeefd25 100644 --- a/vectordb_bench/backend/clients/pgvector/pgvector.py +++ b/vectordb_bench/backend/clients/pgvector/pgvector.py @@ -1,8 +1,10 @@ """Wrapper around the Pgvector vector database over VectorDB""" +import io import logging from contextlib import contextmanager from typing import Any +import pandas as pd import psycopg2 import psycopg2.extras @@ -126,8 +128,15 @@ def insert_embeddings( assert self.cursor is not None, "Cursor is not initialized" try: - items = [dict(id = metadata[i], embedding=embeddings[i]) for i in range(len(metadata))] - psycopg2.extras.execute_batch(self.cursor, f'INSERT INTO public."{self.table_name}" (id, embedding) VALUES (%(id)s, %(embedding)s)', items) + items = { + "id": metadata, + "embedding": embeddings + } + df = pd.DataFrame(items) + csv_buffer = io.StringIO() + df.to_csv(csv_buffer, index=False, header=False) + csv_buffer.seek(0) + self.cursor.copy_expert(f"COPY public.\"{self.table_name}\" FROM STDIN WITH (FORMAT CSV)", csv_buffer) self.conn.commit() return len(metadata), None except Exception as e: From a756d2893607815ba1b1e3f641fb498ee316dad8 Mon Sep 17 00:00:00 2001 From: "been.zino" Date: Tue, 31 Oct 2023 16:31:11 +0900 Subject: [PATCH 038/327] REBUILD Index after INSERT REBUILD Index after INSERT like milvus --- vectordb_bench/backend/clients/pgvector/pgvector.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/vectordb_bench/backend/clients/pgvector/pgvector.py b/vectordb_bench/backend/clients/pgvector/pgvector.py index a0eeefd25..e0fc8d3b8 100644 --- a/vectordb_bench/backend/clients/pgvector/pgvector.py +++ b/vectordb_bench/backend/clients/pgvector/pgvector.py @@ -23,6 +23,7 @@ def __init__( drop_old: bool = False, **kwargs, ): + self.name = "PgVector" self.db_config = db_config self.case_config = db_case_config self.table_name = collection_name @@ -86,6 +87,11 @@ def ready_to_load(self): def optimize(self): pass + + def _post_insert(self): + log.info(f"{self.name} post insert before optimize") + self._drop_index() + self._create_index() def ready_to_search(self): pass @@ -138,6 +144,10 @@ def insert_embeddings( csv_buffer.seek(0) self.cursor.copy_expert(f"COPY public.\"{self.table_name}\" FROM STDIN WITH (FORMAT CSV)", csv_buffer) self.conn.commit() + + if kwargs.get("last_batch"): + self._post_insert() + return len(metadata), None except Exception as e: log.warning(f"Failed to insert data into pgvector table ({self.table_name}), error: {e}") From 450dea34cc6e5b0d935ab251f37490b70e6ccf52 Mon Sep 17 00:00:00 2001 From: yangxuan Date: Thu, 25 Jan 2024 11:07:35 +0800 Subject: [PATCH 039/327] test: fix UT unable to iter local cohere 10M Signed-off-by: yangxuan --- tests/test_dataset.py | 13 +++++++------ vectordb_bench/backend/data_source.py | 4 ++-- vectordb_bench/backend/dataset.py | 1 + 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/tests/test_dataset.py b/tests/test_dataset.py index 56c3715e5..f6ba05036 100644 --- a/tests/test_dataset.py +++ b/tests/test_dataset.py @@ -23,17 +23,18 @@ def test_cohere_error(self): with pytest.raises(ValidationError): Dataset.COHERE.get(9999) - def test_init_cohere(self): - coheres = [Dataset.COHERE.manager(i) for i in [100_000, 1_000_000, 10_000_000]] - for t in coheres: - t._validate_local_file() - def test_iter_cohere(self): cohere_10m = Dataset.COHERE.manager(10_000_000) - cohere_10m.prepare(False) + cohere_10m.prepare(check=False) + + import time + before = time.time() for i in cohere_10m: log.debug(i.head(1)) + dur_iter = time.time() - before + log.warning(f"iter through cohere_10m cost={dur_iter/60}min") + class TestGetFiles: @pytest.mark.parametrize("train_count", [ diff --git a/vectordb_bench/backend/data_source.py b/vectordb_bench/backend/data_source.py index 0398ec653..65926ff6b 100644 --- a/vectordb_bench/backend/data_source.py +++ b/vectordb_bench/backend/data_source.py @@ -69,7 +69,6 @@ def validate_file(self, remote: pathlib.Path, local: pathlib.Path, check_etag: b if check_etag: return match_etag(info.etag.strip('"').lower(), local) - return True def read(self, dataset: str, files: list[str], local_ds_root: pathlib.Path, check_etag: bool = False): @@ -84,7 +83,8 @@ def read(self, dataset: str, files: list[str], local_ds_root: pathlib.Path, chec remote_file = pathlib.Path("benchmark", dataset, file) local_file = local_ds_root.joinpath(file) - if (not local_file.exists()) or (not self.validate_file(remote_file, local_file, check_etag)): + # Don't check etags for Dataset from Aliyun OSS + if (not local_file.exists()) or (not self.validate_file(remote_file, local_file, False)): log.info(f"local file: {local_file} not match with remote: {remote_file}; add to downloading list") downloads.append((remote_file, local_file)) diff --git a/vectordb_bench/backend/dataset.py b/vectordb_bench/backend/dataset.py index 46c31bce5..a227b4147 100644 --- a/vectordb_bench/backend/dataset.py +++ b/vectordb_bench/backend/dataset.py @@ -203,6 +203,7 @@ def prepare(self, source: DatasetSource=DatasetSource.S3, check: bool=True) -> b dataset=self.data.dir_name.lower(), files=self.data.files, local_ds_root=self.data_dir, + check_etag=check, ) prefix = "shuffle_train" if self.data.use_shuffled else "train" From b4f3fc6c8ae510b7dcbf4a98d21056305a1362f3 Mon Sep 17 00:00:00 2001 From: "min.tian" Date: Thu, 22 Feb 2024 02:41:19 +0000 Subject: [PATCH 040/327] add new search_params for zillzcloud: level Signed-off-by: min.tian --- .../backend/clients/zilliz_cloud/config.py | 4 + .../components/run_test/caseSelector.py | 2 +- .../frontend/const/dbCaseConfigs.py | 104 +++++------------- vectordb_bench/models.py | 1 + 4 files changed, 34 insertions(+), 77 deletions(-) diff --git a/vectordb_bench/backend/clients/zilliz_cloud/config.py b/vectordb_bench/backend/clients/zilliz_cloud/config.py index 7c74512da..ee60b397f 100644 --- a/vectordb_bench/backend/clients/zilliz_cloud/config.py +++ b/vectordb_bench/backend/clients/zilliz_cloud/config.py @@ -19,6 +19,7 @@ def to_dict(self) -> dict: class AutoIndexConfig(MilvusIndexConfig, DBCaseConfig): index: IndexType = IndexType.AUTOINDEX + level: int = 1 def index_param(self) -> dict: return { @@ -30,6 +31,9 @@ def index_param(self) -> dict: def search_param(self) -> dict: return { "metric_type": self.parse_metric(), + "params": { + "level": self.level, + } } diff --git a/vectordb_bench/frontend/components/run_test/caseSelector.py b/vectordb_bench/frontend/components/run_test/caseSelector.py index 56521c831..9af023518 100644 --- a/vectordb_bench/frontend/components/run_test/caseSelector.py +++ b/vectordb_bench/frontend/components/run_test/caseSelector.py @@ -59,7 +59,7 @@ def caseConfigSetting(st, allCaseConfigs, case, activedDbList): ) caseConfig = allCaseConfigs[db][case] k = 0 - for config in CASE_CONFIG_MAP.get(db, {}).get(case, []): + for config in CASE_CONFIG_MAP.get(db, {}).get(case.case_cls().label, []): if config.isDisplayed(caseConfig): column = columns[1 + k % CASE_CONFIG_SETTING_COLUMNS] key = "%s-%s-%s" % (db, case, config.label.value) diff --git a/vectordb_bench/frontend/const/dbCaseConfigs.py b/vectordb_bench/frontend/const/dbCaseConfigs.py index f351ad9b2..2311efda8 100644 --- a/vectordb_bench/frontend/const/dbCaseConfigs.py +++ b/vectordb_bench/frontend/const/dbCaseConfigs.py @@ -1,7 +1,7 @@ from enum import IntEnum import typing from pydantic import BaseModel -from vectordb_bench.backend.cases import CaseType +from vectordb_bench.backend.cases import CaseLabel, CaseType from vectordb_bench.backend.clients import DB from vectordb_bench.backend.clients.api import IndexType @@ -406,6 +406,16 @@ class CaseConfigInput(BaseModel): == "product", ) +CaseConfigParamInput_ZillizLevel = CaseConfigInput( + label=CaseConfigParamType.level, + inputType=InputType.Number, + inputConfig={ + "min": 1, + "max": 3, + "value": 1, + }, +) + MilvusLoadConfig = [ CaseConfigParamInput_IndexType, CaseConfigParamInput_M, @@ -479,90 +489,32 @@ class CaseConfigInput(BaseModel): CaseConfigParamInput_QuantizationRatio_PgVectoRS, ] +ZillizCloudPerformanceConfig = [ + CaseConfigParamInput_ZillizLevel, +] + CASE_CONFIG_MAP = { DB.Milvus: { - CaseType.CapacityDim960: MilvusLoadConfig, - CaseType.CapacityDim128: MilvusLoadConfig, - CaseType.Performance768D100M: MilvusPerformanceConfig, - CaseType.Performance768D10M: MilvusPerformanceConfig, - CaseType.Performance768D1M: MilvusPerformanceConfig, - CaseType.Performance768D10M1P: MilvusPerformanceConfig, - CaseType.Performance768D1M1P: MilvusPerformanceConfig, - CaseType.Performance768D10M99P: MilvusPerformanceConfig, - CaseType.Performance768D1M99P: MilvusPerformanceConfig, - CaseType.Performance1536D5M: MilvusPerformanceConfig, - CaseType.Performance1536D500K: MilvusPerformanceConfig, - CaseType.Performance1536D5M1P: MilvusPerformanceConfig, - CaseType.Performance1536D500K1P: MilvusPerformanceConfig, - CaseType.Performance1536D5M99P: MilvusPerformanceConfig, - CaseType.Performance1536D500K99P: MilvusPerformanceConfig, + CaseLabel.Load: MilvusLoadConfig, + CaseLabel.Performance: MilvusPerformanceConfig, + }, + DB.ZillizCloud: { + CaseLabel.Performance: ZillizCloudPerformanceConfig, }, DB.WeaviateCloud: { - CaseType.CapacityDim960: WeaviateLoadConfig, - CaseType.CapacityDim128: WeaviateLoadConfig, - CaseType.Performance768D100M: WeaviatePerformanceConfig, - CaseType.Performance768D10M: WeaviatePerformanceConfig, - CaseType.Performance768D1M: WeaviatePerformanceConfig, - CaseType.Performance768D10M1P: WeaviatePerformanceConfig, - CaseType.Performance768D1M1P: WeaviatePerformanceConfig, - CaseType.Performance768D10M99P: WeaviatePerformanceConfig, - CaseType.Performance768D1M99P: WeaviatePerformanceConfig, - CaseType.Performance1536D5M: WeaviatePerformanceConfig, - CaseType.Performance1536D500K: WeaviatePerformanceConfig, - CaseType.Performance1536D5M1P: WeaviatePerformanceConfig, - CaseType.Performance1536D500K1P: WeaviatePerformanceConfig, - CaseType.Performance1536D5M99P: WeaviatePerformanceConfig, - CaseType.Performance1536D500K99P: WeaviatePerformanceConfig, + CaseLabel.Load: WeaviateLoadConfig, + CaseLabel.Performance: WeaviatePerformanceConfig, }, DB.ElasticCloud: { - CaseType.CapacityDim960: ESLoadingConfig, - CaseType.CapacityDim128: ESLoadingConfig, - CaseType.Performance768D100M: ESPerformanceConfig, - CaseType.Performance768D10M: ESPerformanceConfig, - CaseType.Performance768D1M: ESPerformanceConfig, - CaseType.Performance768D10M1P: ESPerformanceConfig, - CaseType.Performance768D1M1P: ESPerformanceConfig, - CaseType.Performance768D10M99P: ESPerformanceConfig, - CaseType.Performance768D1M99P: ESPerformanceConfig, - CaseType.Performance1536D5M: ESPerformanceConfig, - CaseType.Performance1536D500K: ESPerformanceConfig, - CaseType.Performance1536D5M1P: ESPerformanceConfig, - CaseType.Performance1536D500K1P: ESPerformanceConfig, - CaseType.Performance1536D5M99P: ESPerformanceConfig, - CaseType.Performance1536D500K99P: ESPerformanceConfig, + CaseLabel.Load: ESLoadingConfig, + CaseLabel.Performance: ESPerformanceConfig, }, DB.PgVector: { - CaseType.CapacityDim960: PgVectorLoadingConfig, - CaseType.CapacityDim128: PgVectorLoadingConfig, - CaseType.Performance768D100M: PgVectorPerformanceConfig, - CaseType.Performance768D10M: PgVectorPerformanceConfig, - CaseType.Performance768D1M: PgVectorPerformanceConfig, - CaseType.Performance768D10M1P: PgVectorPerformanceConfig, - CaseType.Performance768D1M1P: PgVectorPerformanceConfig, - CaseType.Performance768D10M99P: PgVectorPerformanceConfig, - CaseType.Performance768D1M99P: PgVectorPerformanceConfig, - CaseType.Performance1536D5M: PgVectorPerformanceConfig, - CaseType.Performance1536D500K: PgVectorPerformanceConfig, - CaseType.Performance1536D5M1P: PgVectorPerformanceConfig, - CaseType.Performance1536D500K1P: PgVectorPerformanceConfig, - CaseType.Performance1536D5M99P: PgVectorPerformanceConfig, - CaseType.Performance1536D500K99P: PgVectorPerformanceConfig, + CaseLabel.Load: PgVectorLoadingConfig, + CaseLabel.Performance: PgVectorPerformanceConfig, }, DB.PgVectoRS: { - CaseType.CapacityDim960: PgVectoRSLoadingConfig, - CaseType.CapacityDim128: PgVectoRSLoadingConfig, - CaseType.Performance768D100M: PgVectoRSPerformanceConfig, - CaseType.Performance768D10M: PgVectoRSPerformanceConfig, - CaseType.Performance768D1M: PgVectoRSPerformanceConfig, - CaseType.Performance768D10M1P: PgVectoRSPerformanceConfig, - CaseType.Performance768D1M1P: PgVectoRSPerformanceConfig, - CaseType.Performance768D10M99P: PgVectoRSPerformanceConfig, - CaseType.Performance768D1M99P: PgVectoRSPerformanceConfig, - CaseType.Performance1536D5M: PgVectoRSPerformanceConfig, - CaseType.Performance1536D500K: PgVectoRSPerformanceConfig, - CaseType.Performance1536D5M1P: PgVectoRSPerformanceConfig, - CaseType.Performance1536D500K1P: PgVectoRSPerformanceConfig, - CaseType.Performance1536D5M99P: PgVectorPerformanceConfig, - CaseType.Performance1536D500K99P: PgVectoRSPerformanceConfig, + CaseLabel.Load: PgVectoRSLoadingConfig, + CaseLabel.Performance: PgVectoRSPerformanceConfig, }, } diff --git a/vectordb_bench/models.py b/vectordb_bench/models.py index fc53d78db..3c2a5b9aa 100644 --- a/vectordb_bench/models.py +++ b/vectordb_bench/models.py @@ -59,6 +59,7 @@ class CaseConfigParamType(Enum): build_algo = "build_algo" cache_dataset_on_device = "cache_dataset_on_device" refine_ratio = "refine_ratio" + level = "level" class CustomizedCase(BaseModel): From 35c2beb78813f5e9318158292191c157c8e8ba16 Mon Sep 17 00:00:00 2001 From: Le Tan Dang Khoa Date: Mon, 15 Jan 2024 16:03:58 +0800 Subject: [PATCH 041/327] Fix normalization on data otherwise it returns ndarray which could not be serialized by json --- vectordb_bench/backend/runner/serial_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vectordb_bench/backend/runner/serial_runner.py b/vectordb_bench/backend/runner/serial_runner.py index d4ff45993..aeed0ec74 100644 --- a/vectordb_bench/backend/runner/serial_runner.py +++ b/vectordb_bench/backend/runner/serial_runner.py @@ -40,7 +40,7 @@ def task(self) -> int: emb_np = np.stack(data_df['emb']) if self.normalize: log.debug("normalize the 100k train data") - all_embeddings = emb_np / np.linalg.norm(emb_np, axis=1)[:, np.newaxis].tolist() + all_embeddings = (emb_np / np.linalg.norm(emb_np, axis=1)[:, np.newaxis]).tolist() else: all_embeddings = emb_np.tolist() del(emb_np) From 2c9e1045e1833831aa108d99ddfb1ee38bfbca8d Mon Sep 17 00:00:00 2001 From: "min.tian" Date: Mon, 26 Feb 2024 03:45:47 +0000 Subject: [PATCH 042/327] add new index_type for milvus: ivf_sq8 Signed-off-by: min.tian --- vectordb_bench/backend/clients/api.py | 1 + .../backend/clients/milvus/config.py | 19 +++++++++++++++++++ .../frontend/const/dbCaseConfigs.py | 3 +++ 3 files changed, 23 insertions(+) diff --git a/vectordb_bench/backend/clients/api.py b/vectordb_bench/backend/clients/api.py index 1e3455fd7..7d4e4f684 100644 --- a/vectordb_bench/backend/clients/api.py +++ b/vectordb_bench/backend/clients/api.py @@ -16,6 +16,7 @@ class IndexType(str, Enum): HNSW = "HNSW" DISKANN = "DISKANN" IVFFlat = "IVF_FLAT" + IVFSQ8 = "IVF_SQ8" Flat = "FLAT" AUTOINDEX = "AUTOINDEX" ES_HNSW = "hnsw" diff --git a/vectordb_bench/backend/clients/milvus/config.py b/vectordb_bench/backend/clients/milvus/config.py index 03ab1b42f..0e93e82b8 100644 --- a/vectordb_bench/backend/clients/milvus/config.py +++ b/vectordb_bench/backend/clients/milvus/config.py @@ -95,6 +95,24 @@ def search_param(self) -> dict: "metric_type": self.parse_metric(), "params": {"nprobe": self.nprobe}, } + +class IVFSQ8Config(MilvusIndexConfig, DBCaseConfig): + nlist: int + nprobe: int | None = None + index: IndexType = IndexType.IVFSQ8 + + def index_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "index_type": self.index.value, + "params": {"nlist": self.nlist}, + } + + def search_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "params": {"nprobe": self.nprobe}, + } class FLATConfig(MilvusIndexConfig, DBCaseConfig): @@ -210,6 +228,7 @@ def search_param(self) -> dict: IndexType.HNSW: HNSWConfig, IndexType.DISKANN: DISKANNConfig, IndexType.IVFFlat: IVFFlatConfig, + IndexType.IVFSQ8: IVFSQ8Config, IndexType.Flat: FLATConfig, IndexType.GPU_IVF_FLAT: GPUIVFFlatConfig, IndexType.GPU_IVF_PQ: GPUIVFPQConfig, diff --git a/vectordb_bench/frontend/const/dbCaseConfigs.py b/vectordb_bench/frontend/const/dbCaseConfigs.py index 2311efda8..fad5f362d 100644 --- a/vectordb_bench/frontend/const/dbCaseConfigs.py +++ b/vectordb_bench/frontend/const/dbCaseConfigs.py @@ -60,6 +60,7 @@ class CaseConfigInput(BaseModel): "options": [ IndexType.HNSW.value, IndexType.IVFFlat.value, + IndexType.IVFSQ8.value, IndexType.DISKANN.value, IndexType.Flat.value, IndexType.AUTOINDEX.value, @@ -197,6 +198,7 @@ class CaseConfigInput(BaseModel): isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) in [ IndexType.IVFFlat.value, + IndexType.IVFSQ8.value, IndexType.GPU_IVF_FLAT.value, IndexType.GPU_IVF_PQ.value, ], @@ -213,6 +215,7 @@ class CaseConfigInput(BaseModel): isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) in [ IndexType.IVFFlat.value, + IndexType.IVFSQ8.value, IndexType.GPU_IVF_FLAT.value, IndexType.GPU_IVF_PQ.value, ], From 67ba9a0393f73653933c846c98fc1991d089b370 Mon Sep 17 00:00:00 2001 From: ljqcodelove Date: Fri, 24 Nov 2023 13:31:00 +0800 Subject: [PATCH 043/327] Make Docker Images for VectorDB Bench --- Dockerfile | 18 +++++++++ install.py | 72 +++++++++++++++++++++++++++++++++ install/requirements_py3.11.txt | 23 +++++++++++ 3 files changed, 113 insertions(+) create mode 100644 Dockerfile create mode 100644 install.py create mode 100644 install/requirements_py3.11.txt diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..5c57ceb03 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +FROM python:3.11-buster as builder-image + +RUN apt-get update + +COPY install/requirements_py3.11.txt . +RUN pip3 install -U pip +RUN pip3 install --no-cache-dir -r requirements_py3.11.txt -i https://pypi.tuna.tsinghua.edu.cn/simple + +FROM python:3.11-slim-buster + +COPY --from=builder-image /usr/local/bin /usr/local/bin +COPY --from=builder-image /usr/local/lib/python3.11/site-packages /usr/local/lib/python3.11/site-packages + +WORKDIR /opt/code +COPY . . +ENV PYTHONPATH /opt/code + +ENTRYPOINT ["python3", "-m", "vectordb_bench"] diff --git a/install.py b/install.py new file mode 100644 index 000000000..f683a37b2 --- /dev/null +++ b/install.py @@ -0,0 +1,72 @@ +import os +import argparse +import subprocess + +def docker_tag_base(): + return 'vdbbench' + +def dockerfile_path_base(): + return os.path.join('vectordb_bench/', '../Dockerfile') + +def docker_tag(track, algo): + return docker_tag_base() + '-' + track + '-' + algo + + +def build(tag, args, dockerfile): + print('Building %s...' % tag) + if args is not None and len(args) != 0: + q = " ".join(["--build-arg " + x.replace(" ", "\\ ") for x in args]) + else: + q = "" + + try: + command = 'docker build %s --rm -t %s -f' \ + % (q, tag) + command += ' %s .' % dockerfile + print(command) + subprocess.check_call(command, shell=True) + return {tag: 'success'} + except subprocess.CalledProcessError: + return {tag: 'fail'} + +def build_multiprocess(args): + return build(*args) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument( + "--proc", + default=1, + type=int, + help="the number of process to build docker images") + parser.add_argument( + '--track', + choices=['none'], + default='none' + ) + parser.add_argument( + '--algorithm', + metavar='NAME', + help='build only the named algorithm image', + default=None) + parser.add_argument( + '--dockerfile', + metavar='PATH', + help='build only the image from a Dockerfile path', + default=None) + parser.add_argument( + '--build-arg', + help='pass given args to all docker builds', + nargs="+") + args = parser.parse_args() + + print('Building base image...') + + subprocess.check_call( + 'docker build \ + --rm -t %s -f %s .' % (docker_tag_base(), dockerfile_path_base()), shell=True) + + print('Building end.') + diff --git a/install/requirements_py3.11.txt b/install/requirements_py3.11.txt new file mode 100644 index 000000000..c55601934 --- /dev/null +++ b/install/requirements_py3.11.txt @@ -0,0 +1,23 @@ +grpcio==1.53.0 +grpcio-tools==1.53.0 +qdrant-client +pinecone-client +weaviate-client +elasticsearch +pgvector +sqlalchemy +redis +chromadb +pytz +streamlit-autorefresh +streamlit>=1.23.0 +streamlit_extras +tqdm +s3fs +psutil +polars +plotly +environs +pydantic Date: Wed, 28 Feb 2024 09:29:52 -0800 Subject: [PATCH 044/327] Merge branch 'main' of https://github.com/s-gruneberg/VectorDBBench --- tests/test_chroma.py | 19 +++---------------- .../backend/clients/chroma/chroma.py | 16 ++-------------- 2 files changed, 5 insertions(+), 30 deletions(-) diff --git a/tests/test_chroma.py b/tests/test_chroma.py index ff99b64fe..2b1b0596e 100644 --- a/tests/test_chroma.py +++ b/tests/test_chroma.py @@ -52,6 +52,7 @@ def test_insert_and_search(self): # insert with chrma.init(): + #chrma.client.delete_collection("example2") assert (chrma.client.heartbeat() is not None), "chroma client is not connected" res = chrma.insert_embeddings(embeddings=embeddings, metadata=range(count)) # bulk_insert return @@ -87,7 +88,7 @@ def test_insert_and_search(self): res = chrma.search_embedding( - query=q, k=100, filters={"metadata": filter_value} + query=q, k=100, filters={"id": filter_value} ) assert ( res[0] == int(test_id) @@ -101,18 +102,4 @@ def test_insert_and_search(self): break assert isFilter, f"Filter not working, id_list: {id_list}" - #Test id filter - res = chrma.search_embedding( - query=q, k=100, filters={"id": 9999} - ) - assert ( - res[0] == 9999 - ) - - #Test two filters, id and metadata - res = chrma.search_embedding( - query=q, k=100, filters={"metadata": filter_value, "id": 9999} - ) - assert ( - res[0] == 9999 and len(res) == 1, f"filters failed, got: ({res[0]}), expected ({9999})" - ) + \ No newline at end of file diff --git a/vectordb_bench/backend/clients/chroma/chroma.py b/vectordb_bench/backend/clients/chroma/chroma.py index 8325909c0..235cb595b 100644 --- a/vectordb_bench/backend/clients/chroma/chroma.py +++ b/vectordb_bench/backend/clients/chroma/chroma.py @@ -106,21 +106,9 @@ def search_embedding( """ if filters: # assumes benchmark test filters of format: {'metadata': '>=10000', 'id': 10000} - metadata_value = filters.get("metadata") id_value = filters.get("id") - if metadata_value and id_value: - results = self.collection.query( - query_embeddings=query, n_results=k, - where={"$and": [{"id": {"$eq": id_value}}, - {"id": {"$gt": metadata_value}} - ]} - ) - elif metadata_value: - results = self.collection.query(query_embeddings=query, n_results=k, - where={"id": {"$gt": metadata_value}}) - else: - results = self.collection.query(query_embeddings=query, n_results=k, - where={"id": {"$eq": id_value}}) + results = self.collection.query(query_embeddings=query, n_results=k, + where={"id": {"$gt": id_value}}) #return list of id's in results return [int(i) for i in results.get('ids')[0]] results = self.collection.query(query_embeddings=query, n_results=k) From b72c6ebd2d4e406f85a91313804dc636cf263ad2 Mon Sep 17 00:00:00 2001 From: yangxuan Date: Fri, 1 Mar 2024 11:18:56 +0800 Subject: [PATCH 045/327] fix: tell setuptools to find package by vectordb_bench Signed-off-by: yangxuan --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 075ec92be..b0e2d3cb1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,6 +5,9 @@ build-backend = "setuptools.build_meta" [tool.setuptools.package-data] "vectordb_bench.results" = ["*.json"] +[tool.setuptools.packages.find] +where = ["vectordb_bench"] + [project] name = "vectordb-bench" authors = [ From 1c2c9e945c8cef74018255fa2e08445b793b980b Mon Sep 17 00:00:00 2001 From: yangxuan Date: Tue, 5 Mar 2024 10:31:14 +0800 Subject: [PATCH 046/327] fix: Unable to find module "vectordb_bench" Fixes: #288 Signed-off-by: yangxuan --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index b0e2d3cb1..a7c5c892b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,8 @@ build-backend = "setuptools.build_meta" "vectordb_bench.results" = ["*.json"] [tool.setuptools.packages.find] -where = ["vectordb_bench"] +where = ["."] +include = ["vectordb_bench"] [project] name = "vectordb-bench" From ce7d7b594eec8ee74e8c6b67cdc65b02689dc9ff Mon Sep 17 00:00:00 2001 From: yangxuan Date: Mon, 4 Mar 2024 12:02:28 +0800 Subject: [PATCH 047/327] fix: Unable to download some files from AliyunOSS Fixes: #267, #275, #285 Signed-off-by: yangxuan --- tests/test_data_source.py | 84 ++++--------------- tests/test_dataset.py | 54 ++++++------ tests/test_utils.py | 27 ++++++ vectordb_bench/backend/dataset.py | 114 +++++++++++--------------- vectordb_bench/backend/task_runner.py | 4 +- vectordb_bench/backend/utils.py | 30 +++++++ 6 files changed, 149 insertions(+), 164 deletions(-) diff --git a/tests/test_data_source.py b/tests/test_data_source.py index a2b1d0ed8..07e5b1878 100644 --- a/tests/test_data_source.py +++ b/tests/test_data_source.py @@ -1,78 +1,28 @@ import logging -import pathlib import pytest -from vectordb_bench.backend.data_source import AliyunOSSReader, AwsS3Reader -from vectordb_bench.backend.dataset import Dataset, DatasetManager +from vectordb_bench.backend.data_source import DatasetSource +from vectordb_bench.backend.cases import type2case -log = logging.getLogger(__name__) +log = logging.getLogger("vectordb_bench") class TestReader: - @pytest.mark.parametrize("size", [ - 100_000, - 1_000_000, - 10_000_000, + @pytest.mark.parametrize("type_case", [ + (k, v) for k, v in type2case.items() ]) - def test_cohere(self, size): - cohere = Dataset.COHERE.manager(size) - self.per_dataset_test(cohere) + def test_type_cases(self, type_case): + self.per_case_test(type_case) - @pytest.mark.parametrize("size", [ - 100_000, - 1_000_000, - ]) - def test_gist(self, size): - gist = Dataset.GIST.manager(size) - self.per_dataset_test(gist) - - @pytest.mark.parametrize("size", [ - 1_000_000, - ]) - def test_glove(self, size): - glove = Dataset.GLOVE.manager(size) - self.per_dataset_test(glove) - - @pytest.mark.parametrize("size", [ - 500_000, - 5_000_000, - # 50_000_000, - ]) - def test_sift(self, size): - sift = Dataset.SIFT.manager(size) - self.per_dataset_test(sift) - - @pytest.mark.parametrize("size", [ - 50_000, - 500_000, - 5_000_000, - ]) - def test_openai(self, size): - openai = Dataset.OPENAI.manager(size) - self.per_dataset_test(openai) - - - def per_dataset_test(self, dataset: DatasetManager): - s3_reader = AwsS3Reader() - all_files = s3_reader.ls_all(dataset.data.dir_name) - - - remote_f_names = [] - for file in all_files: - remote_f = pathlib.Path(file).name - if dataset.data.use_shuffled and remote_f.startswith("train"): - continue - - elif (not dataset.data.use_shuffled) and remote_f.startswith("shuffle"): - continue - - remote_f_names.append(remote_f) + def per_case_test(self, type_case): + t, ca_cls = type_case + ca = ca_cls() + log.info(f"test case: {t.name}, {ca.name}") - assert set(dataset.data.files) == set(remote_f_names) + filters = ca.filter_rate + ca.dataset.prepare(source=DatasetSource.AliyunOSS, check=False, filters=filters) + ali_trains = ca.dataset.train_files - aliyun_reader = AliyunOSSReader() - for fname in dataset.data.files: - p = pathlib.Path("benchmark", dataset.data.dir_name, fname) - assert aliyun_reader.bucket.object_exists(p.as_posix()) + ca.dataset.prepare(check=False, filters=filters) + s3_trains = ca.dataset.train_files - log.info(f"downloading to {dataset.data_dir}") - aliyun_reader.read(dataset.data.dir_name.lower(), dataset.data.files, dataset.data_dir) + assert ali_trains == s3_trains diff --git a/tests/test_dataset.py b/tests/test_dataset.py index f6ba05036..60a219e25 100644 --- a/tests/test_dataset.py +++ b/tests/test_dataset.py @@ -1,7 +1,8 @@ -from vectordb_bench.backend.dataset import Dataset, get_files +from vectordb_bench.backend.dataset import Dataset import logging import pytest from pydantic import ValidationError +from vectordb_bench.backend.data_source import DatasetSource log = logging.getLogger("vectordb_bench") @@ -35,34 +36,31 @@ def test_iter_cohere(self): dur_iter = time.time() - before log.warning(f"iter through cohere_10m cost={dur_iter/60}min") + # pytest -sv tests/test_dataset.py::TestDataSet::test_iter_laion + def test_iter_laion(self): + laion_100m = Dataset.LAION.manager(100_000_000) + from vectordb_bench.backend.data_source import DatasetSource + laion_100m.prepare(source=DatasetSource.AliyunOSS, check=False) -class TestGetFiles: - @pytest.mark.parametrize("train_count", [ - 1, - 10, - 50, - 100, - ]) - @pytest.mark.parametrize("with_gt", [True, False]) - def test_train_count(self, train_count, with_gt): - files = get_files(train_count, True, with_gt) - log.info(files) + import time + before = time.time() + for i in laion_100m: + log.debug(i.head(1)) - if with_gt: - assert len(files) - 4 == train_count - else: - assert len(files) - 1 == train_count + dur_iter = time.time() - before + log.warning(f"iter through laion_100m cost={dur_iter/60}min") - @pytest.mark.parametrize("use_shuffled", [True, False]) - def test_use_shuffled(self, use_shuffled): - files = get_files(1, use_shuffled, True) - log.info(files) + # https://github.com/zilliztech/VectorDBBench/issues/285 + # TODO: ok + def test_iter_openai(self): + + openai_500k = Dataset.OPENAI.manager(500_000) + openai_500k.prepare(source=DatasetSource.AliyunOSS, check=False) - trains = [f for f in files if "train" in f] - if use_shuffled: - for t in trains: - assert "shuffle_train" in t - else: - for t in trains: - assert "shuffle" not in t - assert "train" in t + import time + before = time.time() + for i in openai_500k: + log.debug(i.head(1)) + + dur_iter = time.time() - before + log.warning(f"iter through openai 500K cost={dur_iter/60}min, source=AliyunOSS") diff --git a/tests/test_utils.py b/tests/test_utils.py index f1a8e1f3d..df3fa6ffe 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -37,3 +37,30 @@ def test_recall(self, got_expected): log.info(f"recall: {res}, expected: {expected}") assert res == expected + +class TestGetFiles: + @pytest.mark.parametrize("train_count", [ + 1, + 10, + 50, + 100, + ]) + def test_train_count(self, train_count): + files = utils.compose_train_files(train_count, True) + log.info(files) + + assert len(files) == train_count + + @pytest.mark.parametrize("use_shuffled", [True, False]) + def test_use_shuffled(self, use_shuffled): + files = utils.compose_train_files(1, use_shuffled) + log.info(files) + + trains = [f for f in files if "train" in f] + if use_shuffled: + for t in trains: + assert "shuffle_train" in t + else: + for t in trains: + assert "shuffle" not in t + assert "train" in t diff --git a/vectordb_bench/backend/dataset.py b/vectordb_bench/backend/dataset.py index a227b4147..ffe5b10f0 100644 --- a/vectordb_bench/backend/dataset.py +++ b/vectordb_bench/backend/dataset.py @@ -22,7 +22,7 @@ log = logging.getLogger(__name__) -SizeLabel = namedtuple('SizeLabel', ['size', 'label', 'files']) +SizeLabel = namedtuple('SizeLabel', ['size', 'label', 'file_count']) class BaseDataset(BaseModel): @@ -31,6 +31,7 @@ class BaseDataset(BaseModel): dim: int metric_type: MetricType use_shuffled: bool + with_gt: bool = False _size_label: dict[int, SizeLabel] = PrivateAttr() @validator("size") @@ -48,34 +49,8 @@ def dir_name(self) -> str: return f"{self.name}_{self.label}_{utils.numerize(self.size)}".lower() @property - def files(self) -> str: - return self._size_label.get(self.size).files - - -def get_files(train_count: int, use_shuffled: bool, with_gt: bool = True) -> list[str]: - prefix = "shuffle_train" if use_shuffled else "train" - middle = f"of-{train_count}" - surfix = "parquet" - - train_files = [] - if train_count > 1: - just_size = len(str(train_count)) - for i in range(train_count): - sub_file = f"{prefix}-{str(i).rjust(just_size, '0')}-{middle}.{surfix}" - train_files.append(sub_file) - else: - train_files.append(f"{prefix}.{surfix}") - - files = ['test.parquet'] - if with_gt: - files.extend([ - 'neighbors.parquet', - 'neighbors_tail_1p.parquet', - 'neighbors_head_1p.parquet', - ]) - - files.extend(train_files) - return files + def file_count(self) -> int: + return self._size_label.get(self.size).file_count class LAION(BaseDataset): @@ -83,8 +58,9 @@ class LAION(BaseDataset): dim: int = 768 metric_type: MetricType = MetricType.L2 use_shuffled: bool = False + with_gt: bool = True _size_label: dict = { - 100_000_000: SizeLabel(100_000_000, "LARGE", get_files(100, False)), + 100_000_000: SizeLabel(100_000_000, "LARGE", 100), } @@ -94,8 +70,8 @@ class GIST(BaseDataset): metric_type: MetricType = MetricType.L2 use_shuffled: bool = False _size_label: dict = { - 100_000: SizeLabel(100_000, "SMALL", get_files(1, False, False)), - 1_000_000: SizeLabel(1_000_000, "MEDIUM", get_files(1, False, False)), + 100_000: SizeLabel(100_000, "SMALL", 1), + 1_000_000: SizeLabel(1_000_000, "MEDIUM", 1), } @@ -104,10 +80,11 @@ class Cohere(BaseDataset): dim: int = 768 metric_type: MetricType = MetricType.COSINE use_shuffled: bool = config.USE_SHUFFLED_DATA + with_gt: bool = True, _size_label: dict = { - 100_000: SizeLabel(100_000, "SMALL", get_files(1, config.USE_SHUFFLED_DATA)), - 1_000_000: SizeLabel(1_000_000, "MEDIUM", get_files(1, config.USE_SHUFFLED_DATA)), - 10_000_000: SizeLabel(10_000_000, "LARGE", get_files(10, config.USE_SHUFFLED_DATA)), + 100_000: SizeLabel(100_000, "SMALL", 1), + 1_000_000: SizeLabel(1_000_000, "MEDIUM", 1), + 10_000_000: SizeLabel(10_000_000, "LARGE", 10), } @@ -116,7 +93,7 @@ class Glove(BaseDataset): dim: int = 200 metric_type: MetricType = MetricType.COSINE use_shuffled: bool = False - _size_label: dict = {1_000_000: SizeLabel(1_000_000, "MEDIUM", get_files(1, False, False))} + _size_label: dict = {1_000_000: SizeLabel(1_000_000, "MEDIUM", 1)} class SIFT(BaseDataset): @@ -125,9 +102,9 @@ class SIFT(BaseDataset): metric_type: MetricType = MetricType.L2 use_shuffled: bool = False _size_label: dict = { - 500_000: SizeLabel(500_000, "SMALL", get_files(1, False, False)), - 5_000_000: SizeLabel(5_000_000, "MEDIUM", get_files(1, False, False)), - # 50_000_000: SizeLabel(50_000_000, "LARGE", get_files(50, False, False)), + 500_000: SizeLabel(500_000, "SMALL", 1,), + 5_000_000: SizeLabel(5_000_000, "MEDIUM", 1), + # 50_000_000: SizeLabel(50_000_000, "LARGE", 50), } @@ -136,10 +113,11 @@ class OpenAI(BaseDataset): dim: int = 1536 metric_type: MetricType = MetricType.COSINE use_shuffled: bool = config.USE_SHUFFLED_DATA + with_gt: bool = True, _size_label: dict = { - 50_000: SizeLabel(50_000, "SMALL", get_files(1, config.USE_SHUFFLED_DATA)), - 500_000: SizeLabel(500_000, "MEDIUM", get_files(1, config.USE_SHUFFLED_DATA)), - 5_000_000: SizeLabel(5_000_000, "LARGE", get_files(10, config.USE_SHUFFLED_DATA)), + 50_000: SizeLabel(50_000, "SMALL", 1), + 500_000: SizeLabel(500_000, "MEDIUM", 1), + 5_000_000: SizeLabel(5_000_000, "LARGE", 10), } @@ -155,6 +133,7 @@ class DatasetManager(BaseModel): """ data: BaseDataset test_data: pd.DataFrame | None = None + gt_data: pd.DataFrame | None = None train_files : list[str] = [] reader: DatasetReader | None = None @@ -180,50 +159,51 @@ def data_dir(self) -> pathlib.Path: def __iter__(self): return DataSetIterator(self) - def prepare(self, source: DatasetSource=DatasetSource.S3, check: bool=True) -> bool: + # TODO passing use_shuffle from outside + def prepare(self, + source: DatasetSource=DatasetSource.S3, + check: bool=True, + filters: int | float | str | None = None, + ) -> bool: """Download the dataset from DatasetSource url = f"{source}/{self.data.dir_name}" - download files from url to self.data_dir, there'll be 4 types of files in the data_dir - - train*.parquet: for training - - test.parquet: for testing - - neighbors.parquet: ground_truth of the test.parquet - - neighbors_head_1p.parquet: ground_truth of the test.parquet after filtering 1% data - - neighbors_99p.parquet: ground_truth of the test.parquet after filtering 99% data - Args: source(DatasetSource): S3 or AliyunOSS, default as S3 - check(bool): Whether to do etags check + check(bool): Whether to do etags check, default as ture + filters(Optional[int | float | str]): combined with dataset's with_gt to + compose the correct ground_truth file Returns: bool: whether the dataset is successfully prepared """ + file_count, use_shuffled = self.data.file_count, self.data.use_shuffled + + train_files = utils.compose_train_files(file_count, use_shuffled) + all_files = train_files + + gt_file, test_file = None, None + if self.data.with_gt: + gt_file, test_file = utils.compose_gt_file(filters), "test.parquet" + all_files.extend([gt_file, test_file]) + source.reader().read( dataset=self.data.dir_name.lower(), - files=self.data.files, + files=all_files, local_ds_root=self.data_dir, check_etag=check, ) - prefix = "shuffle_train" if self.data.use_shuffled else "train" + if gt_file is not None and test_file is not None: + self.test_data = self._read_file(test_file) + self.gt_data = self._read_file(gt_file) + + prefix = "shuffle_train" if use_shuffled else "train" self.train_files = sorted([f.name for f in self.data_dir.glob(f'{prefix}*.parquet')]) log.debug(f"{self.data.name}: available train files {self.train_files}") - self.test_data = self._read_file("test.parquet") - return True - def get_ground_truth(self, filters: int | float | None = None) -> pd.DataFrame: - - file_name = "" - if filters is None: - file_name = "neighbors.parquet" - elif filters == 0.01: - file_name = "neighbors_head_1p.parquet" - elif filters == 0.99: - file_name = "neighbors_tail_1p.parquet" - else: - raise ValueError(f"Filters not supported: {filters}") - return self._read_file(file_name) + return True def _read_file(self, file_name: str) -> pd.DataFrame: """read one file from disk into memory""" diff --git a/vectordb_bench/backend/task_runner.py b/vectordb_bench/backend/task_runner.py index 80c5ac1df..a5f4e317f 100644 --- a/vectordb_bench/backend/task_runner.py +++ b/vectordb_bench/backend/task_runner.py @@ -84,7 +84,7 @@ def init_db(self, drop_old: bool = True) -> None: def _pre_run(self, drop_old: bool = True): try: self.init_db(drop_old) - self.ca.dataset.prepare(self.dataset_source) + self.ca.dataset.prepare(self.dataset_source, filters=self.ca.filter_rate) except ModuleNotFoundError as e: log.warning(f"pre run case error: please install client for db: {self.config.db}, error={e}") raise e from None @@ -215,7 +215,7 @@ def _init_search_runner(self): test_emb = test_emb / np.linalg.norm(test_emb, axis=1)[:, np.newaxis] self.test_emb = test_emb.tolist() - gt_df = self.ca.dataset.get_ground_truth(self.ca.filter_rate) + gt_df = self.ca.dataset.gt_data self.serial_search_runner = SerialSearchRunner( db=self.db, diff --git a/vectordb_bench/backend/utils.py b/vectordb_bench/backend/utils.py index d53da31a6..690cff3e8 100644 --- a/vectordb_bench/backend/utils.py +++ b/vectordb_bench/backend/utils.py @@ -42,3 +42,33 @@ def inner(*args, **kwargs): delta = time.perf_counter() - pref return result, delta return inner + + +def compose_train_files(train_count: int, use_shuffled: bool) -> list[str]: + prefix = "shuffle_train" if use_shuffled else "train" + middle = f"of-{train_count}" + surfix = "parquet" + + train_files = [] + if train_count > 1: + just_size = 2 + for i in range(train_count): + sub_file = f"{prefix}-{str(i).rjust(just_size, '0')}-{middle}.{surfix}" + train_files.append(sub_file) + else: + train_files.append(f"{prefix}.{surfix}") + + return train_files + + +def compose_gt_file(filters: int | float | str | None = None) -> str: + if filters is None: + return "neighbors.parquet" + + if filters == 0.01: + return "neighbors_head_1p.parquet" + + if filters == 0.99: + return "neighbors_tail_1p.parquet" + + raise ValueError(f"Filters not supported: {filters}") From 661f0aeab6cf092ec4570e739f6098c52d0d9c01 Mon Sep 17 00:00:00 2001 From: Zhao Junwang Date: Thu, 29 Feb 2024 14:01:48 +0800 Subject: [PATCH 048/327] add dev container support Signed-off-by: Zhao Junwang --- .devcontainer/Dockerfile | 10 +++++++ .devcontainer/devcontainer.json | 47 +++++++++++++++++++++++++++++++++ README.md | 13 +++++++++ 3 files changed, 70 insertions(+) create mode 100644 .devcontainer/Dockerfile create mode 100644 .devcontainer/devcontainer.json diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 000000000..e44b68006 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3.11-buster as builder-image + +RUN apt-get update + +COPY ../install/requirements_py3.11.txt . +RUN pip3 install -U pip +RUN pip3 install --no-cache-dir -r requirements_py3.11.txt -i https://pypi.tuna.tsinghua.edu.cn/simple + +WORKDIR /opt/code +ENV PYTHONPATH /opt/code/VectorDBBench diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 000000000..1b13439e5 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,47 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the +// README at: https://github.com/devcontainers/templates/tree/main/src/docker-existing-dockerfile +{ + "name": "VectorDBBench dev container", + "build": { + // Sets the run context to one level up instead of the .devcontainer folder. + "context": "..", + // Update the 'dockerFile' property if you aren't using the standard 'Dockerfile' filename. + "dockerfile": "./Dockerfile" + }, + "runArgs": [ + "--privileged", + "--cap-add=SYS_PTRACE" + ], + "mounts": [ + // You have to make sure source directory is avaliable on your host file system. + "source=${localEnv:HOME}/vectordb_bench/dataset,target=/tmp/vectordb_bench/dataset,type=bind,consistency=cached" + ], + "workspaceMount": "source=${localWorkspaceFolder},target=/opt/code/VectorDBBench,type=bind,consistency=cached", + "workspaceFolder": "/opt/code/VectorDBBench", + + // Features to add to the dev container. More info: https://containers.dev/features. + // "features": {}, + + // Use 'forwardPorts' to make a list of ports inside the container available locally. + "forwardPorts": [ + 8501 + ], + + // Uncomment the next line to run commands after the container is created. + // "postCreateCommand": "cat /etc/os-release", + + // Configure tool-specific properties. + "customizations": { + "vscode": { + "extensions": [ + "eamodio.gitlens", + "ms-python.python", + "ms-python.debugpy", + "ms-azuretools.vscode-docker" + ] + } + } + + // Uncomment to connect as an existing user other than the container default. More info: https://aka.ms/dev-containers-non-root. + // "remoteUser": "devcontainer" +} diff --git a/README.md b/README.md index 2cdeeb460..4ff2f9121 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,19 @@ OR: ```shell $ init_bench ``` + +OR: + +If you are using [dev container](https://code.visualstudio.com/docs/devcontainers/containers), create +the following dataset directory first: + +```shell +# Mount local ~/vectordb_bench/dataset to contain's /tmp/vectordb_bench/dataset. +# If you are not comfortable with the path name, feel free to change it in devcontainer.json +mkdir -p ~/vectordb_bench/dataset +``` +After reopen the repository in container, run `python -m vectordb_bench` in the container's bash. + ### Check coding styles ```shell $ ruff check vectordb_bench From 1c163214434321943fbd6c933c591cde057ccc30 Mon Sep 17 00:00:00 2001 From: yangxuan Date: Wed, 13 Mar 2024 16:11:23 +0800 Subject: [PATCH 049/327] enhance: Support for windows Add github actions to make sure about it See also: #285 Signed-off-by: yangxuan --- .github/workflows/pull_request.yml | 36 +++++++++++++++++++++++++ Makefile | 2 ++ tests/test_dataset.py | 39 ++++++++++++++++++--------- vectordb_bench/backend/data_source.py | 12 ++++----- 4 files changed, 70 insertions(+), 19 deletions(-) create mode 100644 .github/workflows/pull_request.yml create mode 100644 Makefile diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml new file mode 100644 index 000000000..7c133cd5f --- /dev/null +++ b/.github/workflows/pull_request.yml @@ -0,0 +1,36 @@ +name: Test on pull request + +on: + pull_request: + branches: + - main + +jobs: + build: + name: Run Python Tests + strategy: + matrix: + python-version: [3.11, 3.12] + os: [ubuntu-latest, windows-latest] + runs-on: ${{ matrix.os }} + + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Fetch tags + run: | + git fetch --prune --unshallow --tags + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e ".[test]" + + - name: Test with pytest + run: | + make unittest diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..562615f6d --- /dev/null +++ b/Makefile @@ -0,0 +1,2 @@ +unittest: + PYTHONPATH=`pwd` python3 -m pytest tests/test_dataset.py::TestDataSet::test_download_small -svv diff --git a/tests/test_dataset.py b/tests/test_dataset.py index 60a219e25..c7678c206 100644 --- a/tests/test_dataset.py +++ b/tests/test_dataset.py @@ -36,7 +36,7 @@ def test_iter_cohere(self): dur_iter = time.time() - before log.warning(f"iter through cohere_10m cost={dur_iter/60}min") - # pytest -sv tests/test_dataset.py::TestDataSet::test_iter_laion + # pytest -sv tests/test_dataset.py::TestDataSet::test_iter_laion def test_iter_laion(self): laion_100m = Dataset.LAION.manager(100_000_000) from vectordb_bench.backend.data_source import DatasetSource @@ -50,17 +50,30 @@ def test_iter_laion(self): dur_iter = time.time() - before log.warning(f"iter through laion_100m cost={dur_iter/60}min") - # https://github.com/zilliztech/VectorDBBench/issues/285 - # TODO: ok - def test_iter_openai(self): - - openai_500k = Dataset.OPENAI.manager(500_000) - openai_500k.prepare(source=DatasetSource.AliyunOSS, check=False) + def test_download_small(self): + openai_50k = Dataset.OPENAI.manager(50_000) + files = [ + "test.parquet", + "neighbors.parquet", + "neighbors_head_1p.parquet", + "neighbors_tail_1p.parquet", + ] - import time - before = time.time() - for i in openai_500k: - log.debug(i.head(1)) + file_path = openai_50k.data_dir.joinpath("test.parquet") + import os + + DatasetSource.S3.reader().read( + openai_50k.data.dir_name.lower(), + files=files, + local_ds_root=openai_50k.data_dir, + check_etag=False, + ) + + os.remove(file_path) + DatasetSource.AliyunOSS.reader().read( + openai_50k.data.dir_name.lower(), + files=files, + local_ds_root=openai_50k.data_dir, + check_etag=False, + ) - dur_iter = time.time() - before - log.warning(f"iter through openai 500K cost={dur_iter/60}min, source=AliyunOSS") diff --git a/vectordb_bench/backend/data_source.py b/vectordb_bench/backend/data_source.py index 65926ff6b..28e3c3636 100644 --- a/vectordb_bench/backend/data_source.py +++ b/vectordb_bench/backend/data_source.py @@ -76,11 +76,11 @@ def read(self, dataset: str, files: list[str], local_ds_root: pathlib.Path, chec if not local_ds_root.exists(): log.info(f"local dataset root path not exist, creating it: {local_ds_root}") local_ds_root.mkdir(parents=True) - downloads = [(pathlib.Path("benchmark", dataset, f), local_ds_root.joinpath(f)) for f in files] + downloads = [(pathlib.PurePosixPath("benchmark", dataset, f), local_ds_root.joinpath(f)) for f in files] else: for file in files: - remote_file = pathlib.Path("benchmark", dataset, file) + remote_file = pathlib.PurePosixPath("benchmark", dataset, file) local_file = local_ds_root.joinpath(file) # Don't check etags for Dataset from Aliyun OSS @@ -93,8 +93,8 @@ def read(self, dataset: str, files: list[str], local_ds_root: pathlib.Path, chec log.info(f"Start to downloading files, total count: {len(downloads)}") for remote_file, local_file in tqdm(downloads): - log.debug(f"downloading file {remote_file} to {local_ds_root}") - self.bucket.get_object_to_file(remote_file.as_posix(), local_file.as_posix()) + log.debug(f"downloading file {remote_file} to {local_file}") + self.bucket.get_object_to_file(remote_file.as_posix(), local_file.absolute()) log.info(f"Succeed to download all files, downloaded file count = {len(downloads)}") @@ -125,11 +125,11 @@ def read(self, dataset: str, files: list[str], local_ds_root: pathlib.Path, chec if not local_ds_root.exists(): log.info(f"local dataset root path not exist, creating it: {local_ds_root}") local_ds_root.mkdir(parents=True) - downloads = [pathlib.Path(self.remote_root, dataset, f) for f in files] + downloads = [pathlib.PurePosixPath(self.remote_root, dataset, f) for f in files] else: for file in files: - remote_file = pathlib.Path(self.remote_root, dataset, file) + remote_file = pathlib.PurePosixPath(self.remote_root, dataset, file) local_file = local_ds_root.joinpath(file) if (not local_file.exists()) or (not self.validate_file(remote_file, local_file, check_etag)): From bf2ba4176e4915dc7449e7fa6a10538c399c08cd Mon Sep 17 00:00:00 2001 From: yangxuan Date: Wed, 13 Mar 2024 17:35:21 +0800 Subject: [PATCH 050/327] enhance: Remove etag checks Etag checks cost a lot, check the size, file_name, and, file count is enough Signed-off-by: yangxuan --- tests/test_data_source.py | 4 +- tests/test_dataset.py | 6 +-- vectordb_bench/backend/data_source.py | 65 +++------------------------ vectordb_bench/backend/dataset.py | 3 -- 4 files changed, 11 insertions(+), 67 deletions(-) diff --git a/tests/test_data_source.py b/tests/test_data_source.py index 07e5b1878..302ee3a1c 100644 --- a/tests/test_data_source.py +++ b/tests/test_data_source.py @@ -19,10 +19,10 @@ def per_case_test(self, type_case): log.info(f"test case: {t.name}, {ca.name}") filters = ca.filter_rate - ca.dataset.prepare(source=DatasetSource.AliyunOSS, check=False, filters=filters) + ca.dataset.prepare(source=DatasetSource.AliyunOSS, filters=filters) ali_trains = ca.dataset.train_files - ca.dataset.prepare(check=False, filters=filters) + ca.dataset.prepare(filters=filters) s3_trains = ca.dataset.train_files assert ali_trains == s3_trains diff --git a/tests/test_dataset.py b/tests/test_dataset.py index c7678c206..d4ccb283d 100644 --- a/tests/test_dataset.py +++ b/tests/test_dataset.py @@ -26,7 +26,7 @@ def test_cohere_error(self): def test_iter_cohere(self): cohere_10m = Dataset.COHERE.manager(10_000_000) - cohere_10m.prepare(check=False) + cohere_10m.prepare() import time before = time.time() @@ -40,7 +40,7 @@ def test_iter_cohere(self): def test_iter_laion(self): laion_100m = Dataset.LAION.manager(100_000_000) from vectordb_bench.backend.data_source import DatasetSource - laion_100m.prepare(source=DatasetSource.AliyunOSS, check=False) + laion_100m.prepare(source=DatasetSource.AliyunOSS) import time before = time.time() @@ -66,7 +66,6 @@ def test_download_small(self): openai_50k.data.dir_name.lower(), files=files, local_ds_root=openai_50k.data_dir, - check_etag=False, ) os.remove(file_path) @@ -74,6 +73,5 @@ def test_download_small(self): openai_50k.data.dir_name.lower(), files=files, local_ds_root=openai_50k.data_dir, - check_etag=False, ) diff --git a/vectordb_bench/backend/data_source.py b/vectordb_bench/backend/data_source.py index 28e3c3636..9e2f172b4 100644 --- a/vectordb_bench/backend/data_source.py +++ b/vectordb_bench/backend/data_source.py @@ -3,7 +3,6 @@ import typing from enum import Enum from tqdm import tqdm -from hashlib import md5 import os from abc import ABC, abstractmethod @@ -32,14 +31,13 @@ class DatasetReader(ABC): remote_root: str @abstractmethod - def read(self, dataset: str, files: list[str], local_ds_root: pathlib.Path, check_etag: bool = True): + def read(self, dataset: str, files: list[str], local_ds_root: pathlib.Path): """read dataset files from remote_root to local_ds_root, Args: dataset(str): for instance "sift_small_500k" files(list[str]): all filenames of the dataset local_ds_root(pathlib.Path): whether to write the remote data. - check_etag(bool): whether to check the etag """ pass @@ -56,7 +54,7 @@ def __init__(self): import oss2 self.bucket = oss2.Bucket(oss2.AnonymousAuth(), self.remote_root, "benchmark", True) - def validate_file(self, remote: pathlib.Path, local: pathlib.Path, check_etag: bool) -> bool: + def validate_file(self, remote: pathlib.Path, local: pathlib.Path) -> bool: info = self.bucket.get_object_meta(remote.as_posix()) # check size equal @@ -65,13 +63,9 @@ def validate_file(self, remote: pathlib.Path, local: pathlib.Path, check_etag: b log.info(f"local file: {local} size[{local_size}] not match with remote size[{remote_size}]") return False - # check etag equal - if check_etag: - return match_etag(info.etag.strip('"').lower(), local) - return True - def read(self, dataset: str, files: list[str], local_ds_root: pathlib.Path, check_etag: bool = False): + def read(self, dataset: str, files: list[str], local_ds_root: pathlib.Path): downloads = [] if not local_ds_root.exists(): log.info(f"local dataset root path not exist, creating it: {local_ds_root}") @@ -83,8 +77,7 @@ def read(self, dataset: str, files: list[str], local_ds_root: pathlib.Path, chec remote_file = pathlib.PurePosixPath("benchmark", dataset, file) local_file = local_ds_root.joinpath(file) - # Don't check etags for Dataset from Aliyun OSS - if (not local_file.exists()) or (not self.validate_file(remote_file, local_file, False)): + if (not local_file.exists()) or (not self.validate_file(remote_file, local_file)): log.info(f"local file: {local_file} not match with remote: {remote_file}; add to downloading list") downloads.append((remote_file, local_file)) @@ -120,7 +113,7 @@ def ls_all(self, dataset: str): return names - def read(self, dataset: str, files: list[str], local_ds_root: pathlib.Path, check_etag: bool = True): + def read(self, dataset: str, files: list[str], local_ds_root: pathlib.Path): downloads = [] if not local_ds_root.exists(): log.info(f"local dataset root path not exist, creating it: {local_ds_root}") @@ -132,7 +125,7 @@ def read(self, dataset: str, files: list[str], local_ds_root: pathlib.Path, chec remote_file = pathlib.PurePosixPath(self.remote_root, dataset, file) local_file = local_ds_root.joinpath(file) - if (not local_file.exists()) or (not self.validate_file(remote_file, local_file, check_etag)): + if (not local_file.exists()) or (not self.validate_file(remote_file, local_file)): log.info(f"local file: {local_file} not match with remote: {remote_file}; add to downloading list") downloads.append(remote_file) @@ -147,7 +140,7 @@ def read(self, dataset: str, files: list[str], local_ds_root: pathlib.Path, chec log.info(f"Succeed to download all files, downloaded file count = {len(downloads)}") - def validate_file(self, remote: pathlib.Path, local: pathlib.Path, check_etag: bool) -> bool: + def validate_file(self, remote: pathlib.Path, local: pathlib.Path) -> bool: # info() uses ls() inside, maybe we only need to ls once info = self.fs.info(remote) @@ -157,48 +150,4 @@ def validate_file(self, remote: pathlib.Path, local: pathlib.Path, check_etag: b log.info(f"local file: {local} size[{local_size}] not match with remote size[{remote_size}]") return False - # check etag equal - if check_etag: - return match_etag(info.get('ETag', "").strip('"'), local) - return True - - -def match_etag(expected_etag: str, local_file) -> bool: - """Check if local files' etag match with S3""" - def factor_of_1MB(filesize, num_parts): - x = filesize / int(num_parts) - y = x % 1048576 - return int(x + 1048576 - y) - - def calc_etag(inputfile, partsize): - md5_digests = [] - with open(inputfile, 'rb') as f: - for chunk in iter(lambda: f.read(partsize), b''): - md5_digests.append(md5(chunk).digest()) - return md5(b''.join(md5_digests)).hexdigest() + '-' + str(len(md5_digests)) - - def possible_partsizes(filesize, num_parts): - return lambda partsize: partsize < filesize and (float(filesize) / float(partsize)) <= num_parts - - filesize = os.path.getsize(local_file) - le = "" - if '-' not in expected_etag: # no spliting uploading - with open(local_file, 'rb') as f: - le = md5(f.read()).hexdigest() - log.debug(f"calculated local etag {le}, expected etag: {expected_etag}") - return expected_etag == le - else: - num_parts = int(expected_etag.split('-')[-1]) - partsizes = [ ## Default Partsizes Map - 8388608, # aws_cli/boto3 - 15728640, # s3cmd - factor_of_1MB(filesize, num_parts) # Used by many clients to upload large files - ] - - for partsize in filter(possible_partsizes(filesize, num_parts), partsizes): - le = calc_etag(local_file, partsize) - log.debug(f"calculated local etag {le}, expected etag: {expected_etag}") - if expected_etag == le: - return True - return False diff --git a/vectordb_bench/backend/dataset.py b/vectordb_bench/backend/dataset.py index ffe5b10f0..2b630eae3 100644 --- a/vectordb_bench/backend/dataset.py +++ b/vectordb_bench/backend/dataset.py @@ -162,7 +162,6 @@ def __iter__(self): # TODO passing use_shuffle from outside def prepare(self, source: DatasetSource=DatasetSource.S3, - check: bool=True, filters: int | float | str | None = None, ) -> bool: """Download the dataset from DatasetSource @@ -170,7 +169,6 @@ def prepare(self, Args: source(DatasetSource): S3 or AliyunOSS, default as S3 - check(bool): Whether to do etags check, default as ture filters(Optional[int | float | str]): combined with dataset's with_gt to compose the correct ground_truth file @@ -192,7 +190,6 @@ def prepare(self, dataset=self.data.dir_name.lower(), files=all_files, local_ds_root=self.data_dir, - check_etag=check, ) if gt_file is not None and test_file is not None: From 225ba20e2a374eb50fd01623ecd691fca38314b6 Mon Sep 17 00:00:00 2001 From: "min.tian" Date: Fri, 22 Mar 2024 09:58:46 +0800 Subject: [PATCH 051/327] update zillizcloud results (2024-01) Signed-off-by: min.tian --- ...> result_20240105_202401_zillizcloud.json} | 82 +++++++++---------- vectordb_bench/results/leaderboard.json | 2 +- 2 files changed, 42 insertions(+), 42 deletions(-) rename vectordb_bench/results/ZillizCloud/{result_20240105_beta_202401_zillizcloud.json => result_20240105_202401_zillizcloud.json} (92%) diff --git a/vectordb_bench/results/ZillizCloud/result_20240105_beta_202401_zillizcloud.json b/vectordb_bench/results/ZillizCloud/result_20240105_202401_zillizcloud.json similarity index 92% rename from vectordb_bench/results/ZillizCloud/result_20240105_beta_202401_zillizcloud.json rename to vectordb_bench/results/ZillizCloud/result_20240105_202401_zillizcloud.json index 7c1f3667c..2d4bd7ea6 100644 --- a/vectordb_bench/results/ZillizCloud/result_20240105_beta_202401_zillizcloud.json +++ b/vectordb_bench/results/ZillizCloud/result_20240105_202401_zillizcloud.json @@ -1,6 +1,6 @@ { "run_id": "0ae10e14e34c4c3c9a7116e7a9591d01", - "task_label": "zilliz-beta-202401", + "task_label": "standard", "results": [ { "metrics": { @@ -13,7 +13,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "8cu-perf-beta-202401", + "db_label": "8cu-perf-202401", "uri": "**********", "user": "db_admin", "password": "**********" @@ -40,7 +40,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "8cu-perf-beta-202401", + "db_label": "8cu-perf-202401", "uri": "**********", "user": "db_admin", "password": "**********" @@ -67,7 +67,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "8cu-perf-beta-202401", + "db_label": "8cu-perf-202401", "uri": "**********", "user": "db_admin", "password": "**********" @@ -94,7 +94,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "8cu-perf-beta-202401", + "db_label": "8cu-perf-202401", "uri": "**********", "user": "db_admin", "password": "**********" @@ -121,7 +121,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "8cu-perf-beta-202401", + "db_label": "8cu-perf-202401", "uri": "**********", "user": "db_admin", "password": "**********" @@ -148,7 +148,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "8cu-perf-beta-202401", + "db_label": "8cu-perf-202401", "uri": "**********", "user": "db_admin", "password": "**********" @@ -175,7 +175,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "8cu-perf-beta-202401", + "db_label": "8cu-perf-202401", "uri": "**********", "user": "db_admin", "password": "**********" @@ -202,7 +202,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "8cu-perf-beta-202401", + "db_label": "8cu-perf-202401", "uri": "**********", "user": "db_admin", "password": "**********" @@ -229,7 +229,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "8cu-perf-beta-202401", + "db_label": "8cu-perf-202401", "uri": "**********", "user": "db_admin", "password": "**********" @@ -256,7 +256,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "8cu-perf-beta-202401", + "db_label": "8cu-perf-202401", "uri": "**********", "user": "db_admin", "password": "**********" @@ -283,7 +283,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "8cu-perf-beta-202401", + "db_label": "8cu-perf-202401", "uri": "**********", "user": "db_admin", "password": "**********" @@ -310,7 +310,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "8cu-perf-beta-202401", + "db_label": "8cu-perf-202401", "uri": "**********", "user": "db_admin", "password": "**********" @@ -337,7 +337,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-cap-beta-202401", + "db_label": "1cu-cap-202401", "uri": "**********", "user": "db_admin", "password": "**********" @@ -364,7 +364,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-cap-beta-202401", + "db_label": "1cu-cap-202401", "uri": "**********", "user": "db_admin", "password": "**********" @@ -391,7 +391,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-cap-beta-202401", + "db_label": "1cu-cap-202401", "uri": "**********", "user": "db_admin", "password": "**********" @@ -418,7 +418,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-cap-beta-202401", + "db_label": "1cu-cap-202401", "uri": "**********", "user": "db_admin", "password": "**********" @@ -445,7 +445,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-cap-beta-202401", + "db_label": "1cu-cap-202401", "uri": "**********", "user": "db_admin", "password": "**********" @@ -472,7 +472,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-cap-beta-202401", + "db_label": "1cu-cap-202401", "uri": "**********", "user": "db_admin", "password": "**********" @@ -499,7 +499,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "2cu-cap-beta-202401", + "db_label": "2cu-cap-202401", "uri": "**********", "user": "db_admin", "password": "**********" @@ -526,7 +526,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "2cu-cap-beta-202401", + "db_label": "2cu-cap-202401", "uri": "**********", "user": "db_admin", "password": "**********" @@ -553,7 +553,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "2cu-cap-beta-202401", + "db_label": "2cu-cap-202401", "uri": "**********", "user": "db_admin", "password": "**********" @@ -580,7 +580,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "2cu-cap-beta-202401", + "db_label": "2cu-cap-202401", "uri": "**********", "user": "db_admin", "password": "**********" @@ -607,7 +607,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "2cu-cap-beta-202401", + "db_label": "2cu-cap-202401", "uri": "**********", "user": "db_admin", "password": "**********" @@ -634,7 +634,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "2cu-cap-beta-202401", + "db_label": "2cu-cap-202401", "uri": "**********", "user": "db_admin", "password": "**********" @@ -661,7 +661,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "2cu-cap-beta-202401", + "db_label": "2cu-cap-202401", "uri": "**********", "user": "db_admin", "password": "**********" @@ -688,7 +688,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "2cu-cap-beta-202401", + "db_label": "2cu-cap-202401", "uri": "**********", "user": "db_admin", "password": "**********" @@ -715,7 +715,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "2cu-cap-beta-202401", + "db_label": "2cu-cap-202401", "uri": "**********", "user": "db_admin", "password": "**********" @@ -742,7 +742,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "2cu-cap-beta-202401", + "db_label": "2cu-cap-202401", "uri": "**********", "user": "db_admin", "password": "**********" @@ -769,7 +769,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "2cu-cap-beta-202401", + "db_label": "2cu-cap-202401", "uri": "**********", "user": "db_admin", "password": "**********" @@ -796,7 +796,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "2cu-cap-beta-202401", + "db_label": "2cu-cap-202401", "uri": "**********", "user": "db_admin", "password": "**********" @@ -823,7 +823,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-perf-beta-202401", + "db_label": "1cu-perf-202401", "uri": "**********", "user": "db_admin", "password": "**********" @@ -850,7 +850,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-perf-beta-202401", + "db_label": "1cu-perf-202401", "uri": "**********", "user": "db_admin", "password": "**********" @@ -877,7 +877,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-perf-beta-202401", + "db_label": "1cu-perf-202401", "uri": "**********", "user": "db_admin", "password": "**********" @@ -896,15 +896,15 @@ { "metrics": { "max_load_count": 0, - "load_duration": 960.7296, - "qps": 871.5513, - "serial_latency_p99": 0.0213, - "recall": 0.9471 + "load_duration": 963.5924, + "qps": 873.3712, + "serial_latency_p99": 0.0067, + "recall": 0.9477 }, "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-perf-beta-202401", + "db_label": "1cu-perf-202401", "uri": "**********", "user": "db_admin", "password": "**********" @@ -931,7 +931,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-perf-beta-202401", + "db_label": "1cu-perf-202401", "uri": "**********", "user": "db_admin", "password": "**********" @@ -958,7 +958,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-perf-beta-202401", + "db_label": "1cu-perf-202401", "uri": "**********", "user": "db_admin", "password": "**********" diff --git a/vectordb_bench/results/leaderboard.json b/vectordb_bench/results/leaderboard.json index 1fa4a495a..0d9d2e1d9 100644 --- a/vectordb_bench/results/leaderboard.json +++ b/vectordb_bench/results/leaderboard.json @@ -1 +1 @@ -[{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":0.8836,"latency":2523.0,"recall":0.8528,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":0.8937,"latency":3720.2000000000003,"recall":0.8525,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":1.2145,"latency":3622.3999999999996,"recall":0.7487,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":10.6271,"latency":730.7,"recall":0.8898,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":10.8507,"latency":733.1999999999999,"recall":0.8897,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":75.7055,"latency":121.2,"recall":0.9999,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":11.2945,"latency":3611.2000000000003,"recall":0.996,"label":1,"qp$":84832.4640100146},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":17.3271,"latency":3774.7999999999997,"recall":0.9961,"label":1,"qp$":130143.04193615691},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":26.26,"latency":556.1,"recall":0.9999,"label":1,"qp$":197237.6382224077},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":15.2269,"latency":861.8,"recall":0.9888,"label":1,"qp$":114368.53745044857},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":15.1749,"latency":774.3,"recall":0.989,"label":1,"qp$":113977.96786981013},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":27.6181,"latency":305.5,"recall":0.9999,"label":1,"qp$":207438.26413519715},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":228.4,"latency":22.2,"recall":0.9348,"label":1,"qp$":5631780.821917809},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":181.5,"latency":26.1,"recall":0.9345,"label":1,"qp$":4475342.465753425},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":205.7,"latency":24.2,"recall":0.9586,"label":1,"qp$":5072054.794520548},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":67.63,"latency":36.0,"recall":0.8064,"label":1,"qp$":2502240.4933196297},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":63.35,"latency":38.4,"recall":0.8065,"label":1,"qp$":2343884.892086331},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":176.7,"latency":27.599999999999998,"recall":1.0,"label":1,"qp$":6537718.396711202},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":15.33,"latency":84.9,"recall":0.8064,"label":1,"qp$":567194.2446043165},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":15.13,"latency":86.7,"recall":0.8065,"label":1,"qp$":559794.4501541625},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":17.41,"latency":74.30000000000001,"recall":1.0,"label":1,"qp$":644152.1068859198},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":16.34,"latency":88.7,"recall":0.879,"label":1,"qp$":301661.53846153844},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":10.45,"latency":126.8,"recall":0.8208,"label":1,"qp$":192923.0769230769},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":16.18,"latency":87.5,"recall":0.8793,"label":1,"qp$":298707.69230769225},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":9.8,"latency":130.89999999999998,"recall":0.8212,"label":1,"qp$":180923.0769230769},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":36.11,"latency":55.1,"recall":1.0,"label":1,"qp$":666646.1538461539},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":14.84,"latency":92.7,"recall":0.96,"label":1,"qp$":273969.23076923075},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":322.7,"latency":26.4,"recall":0.9478,"label":1,"qp$":994623.287671233},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":265.5,"latency":26.9,"recall":0.9332,"label":1,"qp$":818321.9178082192},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":303.8,"latency":27.3,"recall":0.9478,"label":1,"qp$":936369.8630136987},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":180.2,"latency":28.2,"recall":0.9335,"label":1,"qp$":555410.9589041097},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":730.7,"latency":24.6,"recall":0.9586,"label":1,"qp$":2252157.5342465756},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":104.3,"latency":31.7,"recall":0.9563,"label":1,"qp$":321472.602739726},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":147.7,"latency":35.3,"recall":0.9707,"label":1,"qp$":682567.3940949935},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":782.5,"latency":25.9,"recall":1.0,"label":1,"qp$":3616174.5827984596},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":18.7634,"latency":153.70000000000002,"recall":0.8737,"label":1,"qp$":694226.515930113},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":18.3619,"latency":79.8,"recall":0.8741,"label":1,"qp$":679371.4285714285},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":25.2744,"latency":61.199999999999996,"recall":0.9979,"label":1,"qp$":935126.8242548818},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":261.798,"latency":23.099999999999998,"recall":0.9262,"label":1,"qp$":6455293.150684931},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":166.1851,"latency":23.900000000000002,"recall":0.9264,"label":1,"qp$":4097714.7945205485},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":121.7169,"latency":29.0,"recall":0.9693,"label":1,"qp$":3001238.6301369863},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":46.6189,"latency":43.1,"recall":0.8737,"label":1,"qp$":1724851.3874614593},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":42.4856,"latency":44.0,"recall":0.8741,"label":1,"qp$":1571923.5354573485},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":138.9479,"latency":26.200000000000003,"recall":0.9979,"label":1,"qp$":5140929.496402878},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":20.7437,"latency":75.80000000000001,"recall":0.9291,"label":1,"qp$":382960.6153846154},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":20.2993,"latency":76.5,"recall":0.9293,"label":1,"qp$":374756.30769230763},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":26.4719,"latency":67.0,"recall":1.0,"label":1,"qp$":488712.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":8.6675,"latency":180.2,"recall":0.8369,"label":1,"qp$":160015.38461538462},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":7.8121,"latency":167.7,"recall":0.8369,"label":1,"qp$":144223.3846153846},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":16.869,"latency":87.8,"recall":0.9814,"label":1,"qp$":311427.6923076923},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":365.0835,"latency":23.599999999999998,"recall":0.945,"label":1,"qp$":1125257.3630136987},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":325.5271,"latency":25.1,"recall":0.9452,"label":1,"qp$":1003336.9520547946},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":596.7942,"latency":24.2,"recall":0.9693,"label":1,"qp$":1839434.1780821919},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":303.2044,"latency":27.400000000000002,"recall":0.9246,"label":1,"qp$":934534.1095890411},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":136.0345,"latency":31.9,"recall":0.9244,"label":1,"qp$":419284.4178082192},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":66.7221,"latency":42.1,"recall":0.963,"label":1,"qp$":205650.30821917808},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":131.2549,"latency":30.200000000000003,"recall":0.9867,"label":1,"qp$":606569.4993581515},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":127.9337,"latency":30.099999999999998,"recall":0.9869,"label":1,"qp$":591221.2066752247},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":595.8462,"latency":23.400000000000002,"recall":1.0,"label":1,"qp$":2753589.627727856},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":330.0144,"latency":9.0,"recall":0.9507,"label":1,"qp$":7472024.150943397},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":271.6585,"latency":10.1,"recall":0.9678,"label":1,"qp$":6150758.490566038},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":216.5226,"latency":12.9,"recall":1.0,"label":1,"qp$":4902398.490566038},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":123.9553,"latency":23.0,"recall":0.971,"label":1,"qp$":1403267.5471698113},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":59.1479,"latency":44.5,"recall":0.9906,"label":1,"qp$":669598.8679245282},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":40.999,"latency":55.300000000000004,"recall":1.0,"label":1,"qp$":464139.62264150946},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":579.9416,"latency":9.4,"recall":0.9213,"label":1,"qp$":6565376.603773585},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":425.2529,"latency":11.299999999999999,"recall":0.9686,"label":1,"qp$":4814183.773584905},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":397.0539,"latency":13.799999999999999,"recall":1.0,"label":1,"qp$":4494949.811320755},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":516.27,"latency":7.0,"recall":0.9463,"label":1,"qp$":11689132.075471697},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":354.8416,"latency":10.0,"recall":0.9802,"label":1,"qp$":8034149.433962264},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":427.5229,"latency":8.7,"recall":1.0,"label":1,"qp$":9679763.773584906},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":2884.689,"latency":5.3,"recall":0.8801,"label":1,"qp$":8164214.150943396},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":1689.5799,"latency":6.6,"recall":0.9493,"label":1,"qp$":4781829.905660377},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1517.6792,"latency":10.0,"recall":1.0,"label":1,"qp$":4295318.4905660385},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":822.5318,"latency":5.6,"recall":0.9294,"label":1,"qp$":2327920.1886792453},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":378.9146,"latency":10.3,"recall":0.9758,"label":1,"qp$":1072399.8113207547},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":218.6854,"latency":16.2,"recall":1.0,"label":1,"qp$":618920.9433962263},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":297.5,"latency":7.2,"recall":0.974,"label":1,"qp$":6735849.0566037735},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":228.3,"latency":10.6,"recall":0.994,"label":1,"qp$":5169056.603773585},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":584.0,"latency":4.6,"recall":1.0,"label":1,"qp$":13222641.509433962},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":1871.0,"latency":7.0,"recall":0.9602,"label":1,"qp$":5295283.018867925},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":556.7,"latency":6.7,"recall":0.9723,"label":1,"qp$":1575566.0377358492},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1583.0,"latency":6.8,"recall":0.9836,"label":1,"qp$":4480188.679245283},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":294.3,"latency":10.9,"recall":0.9939,"label":1,"qp$":832924.5283018869},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":2345.0,"latency":8.9,"recall":1.0,"label":1,"qp$":6636792.452830189},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":295.6,"latency":12.3,"recall":1.0,"label":1,"qp$":836603.7735849057},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":143.0,"latency":33.5,"recall":0.9818,"label":1,"qp$":3237735.8490566034},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":106.0,"latency":20.7,"recall":0.9887,"label":1,"qp$":2400000.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":189.0,"latency":11.6,"recall":1.0,"label":1,"qp$":4279245.2830188675},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":379.9721,"latency":12.4,"recall":0.982,"label":1,"qp$":4301570.9433962265},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":71.74,"latency":50.8,"recall":0.9883,"label":1,"qp$":812150.9433962264},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":287.0,"latency":14.9,"recall":0.9865,"label":1,"qp$":3249056.603773585},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":34.6654,"latency":64.69999999999999,"recall":0.9961,"label":1,"qp$":392438.4905660377},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":412.0,"latency":10.3,"recall":1.0,"label":1,"qp$":4664150.9433962265},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":42.169,"latency":46.800000000000004,"recall":1.0,"label":1,"qp$":477384.9056603773},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":537.4975,"latency":18.9,"recall":0.8903,"label":1,"qp$":1356936.1851332397},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":372.0466,"latency":17.8,"recall":0.8904,"label":1,"qp$":939248.0785413746},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1156.2898,"latency":14.4,"recall":0.9989,"label":1,"qp$":2919104.684431978},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":110.248,"latency":69.0,"recall":0.898,"label":1,"qp$":278325.94670406735},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":87.2601,"latency":27.799999999999997,"recall":0.898,"label":1,"qp$":220291.97755960727},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":125.7846,"latency":23.099999999999998,"recall":0.975,"label":1,"qp$":317548.77980364655},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":240.7209,"latency":17.4,"recall":0.8887,"label":1,"qp$":3038552.734922861},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":189.4399,"latency":17.5,"recall":0.8889,"label":1,"qp$":2391246.98457223},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":313.5116,"latency":16.1,"recall":0.9999,"label":1,"qp$":3957369.424964937},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":78.7196,"latency":49.4,"recall":0.9203,"label":1,"qp$":1707172.048192771},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":68.3111,"latency":35.5,"recall":0.9202,"label":1,"qp$":1481445.5421686745},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":210.2147,"latency":26.8,"recall":0.9996,"label":1,"qp$":4558873.012048192},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":188.6436,"latency":917.5,"recall":0.9175,"label":1,"qp$":2381195.5119214584},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":155.6991,"latency":917.1,"recall":0.9171,"label":1,"qp$":1965346.283309958},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":445.3289,"latency":14.1,"recall":0.9999,"label":1,"qp$":5621262.412342216},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":633.6033,"latency":24.6,"recall":0.919,"label":1,"qp$":1599559.5231416551},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":95.5682,"latency":58.7,"recall":0.9463,"label":1,"qp$":241266.14305750353},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":434.4062,"latency":17.4,"recall":0.9181,"label":1,"qp$":1096677.643758766},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":80.3192,"latency":26.599999999999998,"recall":0.9462,"label":1,"qp$":202769.3688639551},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":1509.3293,"latency":18.5,"recall":0.9995,"label":1,"qp$":3810368.4992987383},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":166.7252,"latency":19.599999999999998,"recall":0.9988,"label":1,"qp$":420905.1332398317},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":46.8622,"latency":123.0,"recall":0.9957,"label":1,"qp$":16703.358415841583},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1.8388,"latency":1063.5,"recall":0.9957,"label":1,"qp$":655.4138613861386},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":45.0666,"latency":109.2,"recall":1.0,"label":1,"qp$":16063.342574257427},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":43.5017,"latency":228.7,"recall":0.9957,"label":1,"qp$":4803.8687116564415},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1.5668,"latency":1114.4,"recall":0.9957,"label":1,"qp$":173.02085889570552},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":41.5443,"latency":159.0,"recall":1.0,"label":1,"qp$":4587.714110429448},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":67.9121,"latency":179.5,"recall":0.9909,"label":1,"qp$":7499.495705521471},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.7636,"latency":1921.3,"recall":0.9908,"label":1,"qp$":84.32392638036809},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":32.0,"latency":124.5,"recall":1.0,"label":1,"qp$":3533.7423312883434},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":63.1365,"latency":145.7,"recall":0.991,"label":1,"qp$":22504.09900990099},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.7512,"latency":1983.8,"recall":0.9908,"label":1,"qp$":267.7544554455445},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":30.1358,"latency":129.8,"recall":1.0,"label":1,"qp$":10741.473267326734},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":37.432,"latency":75.0,"recall":0.9975,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":37.0696,"latency":73.5,"recall":0.9976,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":81.1915,"latency":53.0,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":321.6048,"latency":13.4,"recall":0.989,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":22.1467,"latency":86.8,"recall":0.9972,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":303.2551,"latency":10.9,"recall":0.9876,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":21.5388,"latency":81.69999999999999,"recall":0.997,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":394.5418,"latency":8.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":37.878,"latency":45.0,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":180.2757,"latency":6.0,"recall":0.9942,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":179.0033,"latency":6.4,"recall":0.9943,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":526.8846,"latency":3.6,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":626.5243,"latency":6.2,"recall":0.9954,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":78.4227,"latency":25.3,"recall":0.9982,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":599.4213,"latency":6.6,"recall":0.9955,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":78.5351,"latency":26.3,"recall":0.9982,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":2098.2113,"latency":3.4,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":275.6292,"latency":10.0,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":100.6667,"latency":21.1,"recall":0.9909,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":101.1399,"latency":19.7,"recall":0.9907,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":52.2606,"latency":18.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":61.0661,"latency":49.8,"recall":0.9911,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":58.9326,"latency":44.6,"recall":0.9911,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":42.5977,"latency":54.9,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":536.0726,"latency":8.200000000000001,"recall":0.9728,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":467.179,"latency":7.0,"recall":0.9697,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":431.7512,"latency":8.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":274.5407,"latency":4.8999999999999995,"recall":0.9807,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":236.5672,"latency":10.3,"recall":0.981,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":309.4833,"latency":4.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":178.6585,"latency":13.700000000000001,"recall":0.9843,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":178.3732,"latency":15.0,"recall":0.9844,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":229.3526,"latency":12.5,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":1258.7043,"latency":4.8999999999999995,"recall":0.9799,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":1075.8776,"latency":5.3,"recall":0.98,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1494.8493,"latency":4.7,"recall":1.0,"label":1,"qp$":0.0}] \ No newline at end of file +[{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":0.8836,"latency":2523.0,"recall":0.8528,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":0.8937,"latency":3720.2000000000003,"recall":0.8525,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":1.2145,"latency":3622.3999999999996,"recall":0.7487,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":10.6271,"latency":730.7,"recall":0.8898,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":10.8507,"latency":733.1999999999999,"recall":0.8897,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":75.7055,"latency":121.2,"recall":0.9999,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":11.2945,"latency":3611.2000000000003,"recall":0.996,"label":1,"qp$":84832.4640100146},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":17.3271,"latency":3774.7999999999997,"recall":0.9961,"label":1,"qp$":130143.04193615691},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":26.26,"latency":556.1,"recall":0.9999,"label":1,"qp$":197237.6382224077},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":15.2269,"latency":861.8,"recall":0.9888,"label":1,"qp$":114368.53745044857},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":15.1749,"latency":774.3,"recall":0.989,"label":1,"qp$":113977.96786981013},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":27.6181,"latency":305.5,"recall":0.9999,"label":1,"qp$":207438.26413519715},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":228.4,"latency":22.2,"recall":0.9348,"label":1,"qp$":5631780.821917809},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":181.5,"latency":26.1,"recall":0.9345,"label":1,"qp$":4475342.465753425},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":205.7,"latency":24.2,"recall":0.9586,"label":1,"qp$":5072054.794520548},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":67.63,"latency":36.0,"recall":0.8064,"label":1,"qp$":2502240.4933196297},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":63.35,"latency":38.4,"recall":0.8065,"label":1,"qp$":2343884.892086331},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":176.7,"latency":27.599999999999998,"recall":1.0,"label":1,"qp$":6537718.396711202},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":15.33,"latency":84.9,"recall":0.8064,"label":1,"qp$":567194.2446043165},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":15.13,"latency":86.7,"recall":0.8065,"label":1,"qp$":559794.4501541625},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":17.41,"latency":74.30000000000001,"recall":1.0,"label":1,"qp$":644152.1068859198},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":16.34,"latency":88.7,"recall":0.879,"label":1,"qp$":301661.53846153844},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":10.45,"latency":126.8,"recall":0.8208,"label":1,"qp$":192923.0769230769},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":16.18,"latency":87.5,"recall":0.8793,"label":1,"qp$":298707.69230769225},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":9.8,"latency":130.89999999999998,"recall":0.8212,"label":1,"qp$":180923.0769230769},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":36.11,"latency":55.1,"recall":1.0,"label":1,"qp$":666646.1538461539},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":14.84,"latency":92.7,"recall":0.96,"label":1,"qp$":273969.23076923075},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":322.7,"latency":26.4,"recall":0.9478,"label":1,"qp$":994623.287671233},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":265.5,"latency":26.9,"recall":0.9332,"label":1,"qp$":818321.9178082192},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":303.8,"latency":27.3,"recall":0.9478,"label":1,"qp$":936369.8630136987},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":180.2,"latency":28.2,"recall":0.9335,"label":1,"qp$":555410.9589041097},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":730.7,"latency":24.6,"recall":0.9586,"label":1,"qp$":2252157.5342465756},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":104.3,"latency":31.7,"recall":0.9563,"label":1,"qp$":321472.602739726},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":147.7,"latency":35.3,"recall":0.9707,"label":1,"qp$":682567.3940949935},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":782.5,"latency":25.9,"recall":1.0,"label":1,"qp$":3616174.5827984596},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":18.7634,"latency":153.70000000000002,"recall":0.8737,"label":1,"qp$":694226.515930113},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":18.3619,"latency":79.8,"recall":0.8741,"label":1,"qp$":679371.4285714285},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":25.2744,"latency":61.199999999999996,"recall":0.9979,"label":1,"qp$":935126.8242548818},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":261.798,"latency":23.099999999999998,"recall":0.9262,"label":1,"qp$":6455293.150684931},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":166.1851,"latency":23.900000000000002,"recall":0.9264,"label":1,"qp$":4097714.7945205485},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":121.7169,"latency":29.0,"recall":0.9693,"label":1,"qp$":3001238.6301369863},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":46.6189,"latency":43.1,"recall":0.8737,"label":1,"qp$":1724851.3874614593},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":42.4856,"latency":44.0,"recall":0.8741,"label":1,"qp$":1571923.5354573485},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":138.9479,"latency":26.200000000000003,"recall":0.9979,"label":1,"qp$":5140929.496402878},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":20.7437,"latency":75.80000000000001,"recall":0.9291,"label":1,"qp$":382960.6153846154},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":20.2993,"latency":76.5,"recall":0.9293,"label":1,"qp$":374756.30769230763},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":26.4719,"latency":67.0,"recall":1.0,"label":1,"qp$":488712.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":8.6675,"latency":180.2,"recall":0.8369,"label":1,"qp$":160015.38461538462},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":7.8121,"latency":167.7,"recall":0.8369,"label":1,"qp$":144223.3846153846},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":16.869,"latency":87.8,"recall":0.9814,"label":1,"qp$":311427.6923076923},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":365.0835,"latency":23.599999999999998,"recall":0.945,"label":1,"qp$":1125257.3630136987},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":325.5271,"latency":25.1,"recall":0.9452,"label":1,"qp$":1003336.9520547946},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":596.7942,"latency":24.2,"recall":0.9693,"label":1,"qp$":1839434.1780821919},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":303.2044,"latency":27.400000000000002,"recall":0.9246,"label":1,"qp$":934534.1095890411},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":136.0345,"latency":31.9,"recall":0.9244,"label":1,"qp$":419284.4178082192},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":66.7221,"latency":42.1,"recall":0.963,"label":1,"qp$":205650.30821917808},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":131.2549,"latency":30.200000000000003,"recall":0.9867,"label":1,"qp$":606569.4993581515},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":127.9337,"latency":30.099999999999998,"recall":0.9869,"label":1,"qp$":591221.2066752247},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":595.8462,"latency":23.400000000000002,"recall":1.0,"label":1,"qp$":2753589.627727856},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":330.0144,"latency":9.0,"recall":0.9507,"label":1,"qp$":7472024.150943397},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":271.6585,"latency":10.1,"recall":0.9678,"label":1,"qp$":6150758.490566038},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":216.5226,"latency":12.9,"recall":1.0,"label":1,"qp$":4902398.490566038},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":123.9553,"latency":23.0,"recall":0.971,"label":1,"qp$":1403267.5471698113},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":59.1479,"latency":44.5,"recall":0.9906,"label":1,"qp$":669598.8679245282},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":40.999,"latency":55.300000000000004,"recall":1.0,"label":1,"qp$":464139.62264150946},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":579.9416,"latency":9.4,"recall":0.9213,"label":1,"qp$":6565376.603773585},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":425.2529,"latency":11.299999999999999,"recall":0.9686,"label":1,"qp$":4814183.773584905},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":397.0539,"latency":13.799999999999999,"recall":1.0,"label":1,"qp$":4494949.811320755},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":516.27,"latency":7.0,"recall":0.9463,"label":1,"qp$":11689132.075471697},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":354.8416,"latency":10.0,"recall":0.9802,"label":1,"qp$":8034149.433962264},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":427.5229,"latency":8.7,"recall":1.0,"label":1,"qp$":9679763.773584906},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":2884.689,"latency":5.3,"recall":0.8801,"label":1,"qp$":8164214.150943396},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":1689.5799,"latency":6.6,"recall":0.9493,"label":1,"qp$":4781829.905660377},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1517.6792,"latency":10.0,"recall":1.0,"label":1,"qp$":4295318.4905660385},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":822.5318,"latency":5.6,"recall":0.9294,"label":1,"qp$":2327920.1886792453},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":378.9146,"latency":10.3,"recall":0.9758,"label":1,"qp$":1072399.8113207547},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":218.6854,"latency":16.2,"recall":1.0,"label":1,"qp$":618920.9433962263},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":297.5,"latency":7.2,"recall":0.974,"label":1,"qp$":6735849.0566037735},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":228.3,"latency":10.6,"recall":0.994,"label":1,"qp$":5169056.603773585},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":584.0,"latency":4.6,"recall":1.0,"label":1,"qp$":13222641.509433962},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":1871.0,"latency":7.0,"recall":0.9602,"label":1,"qp$":5295283.018867925},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":556.7,"latency":6.7,"recall":0.9723,"label":1,"qp$":1575566.0377358492},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1583.0,"latency":6.8,"recall":0.9836,"label":1,"qp$":4480188.679245283},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":294.3,"latency":10.9,"recall":0.9939,"label":1,"qp$":832924.5283018869},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":2345.0,"latency":8.9,"recall":1.0,"label":1,"qp$":6636792.452830189},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":295.6,"latency":12.3,"recall":1.0,"label":1,"qp$":836603.7735849057},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":143.0,"latency":33.5,"recall":0.9818,"label":1,"qp$":3237735.8490566034},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":106.0,"latency":20.7,"recall":0.9887,"label":1,"qp$":2400000.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":189.0,"latency":11.6,"recall":1.0,"label":1,"qp$":4279245.2830188675},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":379.9721,"latency":12.4,"recall":0.982,"label":1,"qp$":4301570.9433962265},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":71.74,"latency":50.8,"recall":0.9883,"label":1,"qp$":812150.9433962264},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":287.0,"latency":14.9,"recall":0.9865,"label":1,"qp$":3249056.603773585},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":34.6654,"latency":64.69999999999999,"recall":0.9961,"label":1,"qp$":392438.4905660377},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":412.0,"latency":10.3,"recall":1.0,"label":1,"qp$":4664150.9433962265},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":42.169,"latency":46.800000000000004,"recall":1.0,"label":1,"qp$":477384.9056603773},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":537.4975,"latency":18.9,"recall":0.8903,"label":1,"qp$":1356936.1851332397},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":372.0466,"latency":17.8,"recall":0.8904,"label":1,"qp$":939248.0785413746},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1156.2898,"latency":14.4,"recall":0.9989,"label":1,"qp$":2919104.684431978},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":110.248,"latency":69.0,"recall":0.898,"label":1,"qp$":278325.94670406735},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":87.2601,"latency":27.799999999999997,"recall":0.898,"label":1,"qp$":220291.97755960727},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":125.7846,"latency":23.099999999999998,"recall":0.975,"label":1,"qp$":317548.77980364655},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":240.7209,"latency":17.4,"recall":0.8887,"label":1,"qp$":3038552.734922861},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":189.4399,"latency":17.5,"recall":0.8889,"label":1,"qp$":2391246.98457223},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":313.5116,"latency":16.1,"recall":0.9999,"label":1,"qp$":3957369.424964937},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":78.7196,"latency":49.4,"recall":0.9203,"label":1,"qp$":1707172.048192771},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":68.3111,"latency":35.5,"recall":0.9202,"label":1,"qp$":1481445.5421686745},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":210.2147,"latency":26.8,"recall":0.9996,"label":1,"qp$":4558873.012048192},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":188.6436,"latency":917.5,"recall":0.9175,"label":1,"qp$":2381195.5119214584},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":155.6991,"latency":917.1,"recall":0.9171,"label":1,"qp$":1965346.283309958},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":445.3289,"latency":14.1,"recall":0.9999,"label":1,"qp$":5621262.412342216},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":633.6033,"latency":24.6,"recall":0.919,"label":1,"qp$":1599559.5231416551},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":95.5682,"latency":58.7,"recall":0.9463,"label":1,"qp$":241266.14305750353},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":434.4062,"latency":17.4,"recall":0.9181,"label":1,"qp$":1096677.643758766},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":80.3192,"latency":26.599999999999998,"recall":0.9462,"label":1,"qp$":202769.3688639551},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":1509.3293,"latency":18.5,"recall":0.9995,"label":1,"qp$":3810368.4992987383},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":166.7252,"latency":19.599999999999998,"recall":0.9988,"label":1,"qp$":420905.1332398317},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":46.8622,"latency":123.0,"recall":0.9957,"label":1,"qp$":16703.358415841583},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1.8388,"latency":1063.5,"recall":0.9957,"label":1,"qp$":655.4138613861386},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":45.0666,"latency":109.2,"recall":1.0,"label":1,"qp$":16063.342574257427},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":43.5017,"latency":228.7,"recall":0.9957,"label":1,"qp$":4803.8687116564415},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1.5668,"latency":1114.4,"recall":0.9957,"label":1,"qp$":173.02085889570552},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":41.5443,"latency":159.0,"recall":1.0,"label":1,"qp$":4587.714110429448},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":67.9121,"latency":179.5,"recall":0.9909,"label":1,"qp$":7499.495705521471},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.7636,"latency":1921.3,"recall":0.9908,"label":1,"qp$":84.32392638036809},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":32.0,"latency":124.5,"recall":1.0,"label":1,"qp$":3533.7423312883434},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":63.1365,"latency":145.7,"recall":0.991,"label":1,"qp$":22504.09900990099},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.7512,"latency":1983.8,"recall":0.9908,"label":1,"qp$":267.7544554455445},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":30.1358,"latency":129.8,"recall":1.0,"label":1,"qp$":10741.473267326734},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":37.432,"latency":75.0,"recall":0.9975,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":37.0696,"latency":73.5,"recall":0.9976,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":81.1915,"latency":53.0,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":321.6048,"latency":13.4,"recall":0.989,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":22.1467,"latency":86.8,"recall":0.9972,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":303.2551,"latency":10.9,"recall":0.9876,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":21.5388,"latency":81.69999999999999,"recall":0.997,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":394.5418,"latency":8.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":37.878,"latency":45.0,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":180.2757,"latency":6.0,"recall":0.9942,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":179.0033,"latency":6.4,"recall":0.9943,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":526.8846,"latency":3.6,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":626.5243,"latency":6.2,"recall":0.9954,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":78.4227,"latency":25.3,"recall":0.9982,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":599.4213,"latency":6.6,"recall":0.9955,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":78.5351,"latency":26.3,"recall":0.9982,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":2098.2113,"latency":3.4,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":275.6292,"latency":10.0,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":100.6667,"latency":21.1,"recall":0.9909,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":101.1399,"latency":19.7,"recall":0.9907,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":52.2606,"latency":18.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":61.0661,"latency":49.8,"recall":0.9911,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":58.9326,"latency":44.6,"recall":0.9911,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":42.5977,"latency":54.9,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":536.0726,"latency":8.200000000000001,"recall":0.9728,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":467.179,"latency":7.0,"recall":0.9697,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":431.7512,"latency":8.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":274.5407,"latency":4.8999999999999995,"recall":0.9807,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":236.5672,"latency":10.3,"recall":0.981,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":309.4833,"latency":4.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":178.6585,"latency":13.700000000000001,"recall":0.9843,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":178.3732,"latency":15.0,"recall":0.9844,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":229.3526,"latency":12.5,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":1258.7043,"latency":4.8999999999999995,"recall":0.9799,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":1075.8776,"latency":5.3,"recall":0.98,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1494.8493,"latency":4.7,"recall":1.0,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-202401","db_name":"ZillizCloud-8cu-perf-202401","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":5115.5303,"latency":8.7,"recall":0.9469,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-202401","db_name":"ZillizCloud-8cu-perf-202401","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":3685.0767,"latency":9.1,"recall":0.9736,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-202401","db_name":"ZillizCloud-8cu-perf-202401","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":4742.1617,"latency":9.1,"recall":0.9936,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-202401","db_name":"ZillizCloud-8cu-perf-202401","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":6054.4428,"latency":8.1,"recall":0.9155,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-202401","db_name":"ZillizCloud-8cu-perf-202401","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":4104.2598,"latency":10.6,"recall":0.9506,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-202401","db_name":"ZillizCloud-8cu-perf-202401","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":4252.1267,"latency":9.2,"recall":0.9964,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-202401","db_name":"ZillizCloud-8cu-perf-202401","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":1685.3091,"latency":13.299999999999999,"recall":0.9718,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-202401","db_name":"ZillizCloud-8cu-perf-202401","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":769.8991,"latency":10.7,"recall":0.9884,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-202401","db_name":"ZillizCloud-8cu-perf-202401","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":945.4061,"latency":10.9,"recall":0.9941,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-202401","db_name":"ZillizCloud-8cu-perf-202401","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":2214.9028,"latency":8.4,"recall":0.9249,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-202401","db_name":"ZillizCloud-8cu-perf-202401","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":827.975,"latency":12.0,"recall":0.9692,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-202401","db_name":"ZillizCloud-8cu-perf-202401","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":776.9454,"latency":11.4,"recall":0.9966,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-202401","db_name":"ZillizCloud-1cu-cap-202401","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":269.5464,"latency":9.799999999999999,"recall":0.9776,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-202401","db_name":"ZillizCloud-1cu-cap-202401","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":240.0363,"latency":10.6,"recall":0.9822,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-202401","db_name":"ZillizCloud-1cu-cap-202401","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":218.0627,"latency":11.4,"recall":0.9936,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-202401","db_name":"ZillizCloud-1cu-cap-202401","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":392.8825,"latency":6.8999999999999995,"recall":0.9581,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-202401","db_name":"ZillizCloud-1cu-cap-202401","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":343.8204,"latency":8.4,"recall":0.968,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-202401","db_name":"ZillizCloud-1cu-cap-202401","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":216.6773,"latency":10.1,"recall":0.9968,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-202401","db_name":"ZillizCloud-2cu-cap-202401","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":503.2284,"latency":9.0,"recall":0.9677,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-202401","db_name":"ZillizCloud-2cu-cap-202401","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":413.3232,"latency":9.6,"recall":0.981,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-202401","db_name":"ZillizCloud-2cu-cap-202401","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":425.5492,"latency":10.200000000000001,"recall":0.9936,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-202401","db_name":"ZillizCloud-2cu-cap-202401","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":789.1229,"latency":5.6,"recall":0.9396,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-202401","db_name":"ZillizCloud-2cu-cap-202401","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":571.4257,"latency":7.7,"recall":0.9668,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-202401","db_name":"ZillizCloud-2cu-cap-202401","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":411.7653,"latency":9.1,"recall":0.9968,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-202401","db_name":"ZillizCloud-2cu-cap-202401","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":98.0448,"latency":16.1,"recall":0.9803,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-202401","db_name":"ZillizCloud-2cu-cap-202401","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":58.3152,"latency":28.0,"recall":0.9891,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-202401","db_name":"ZillizCloud-2cu-cap-202401","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":46.8304,"latency":28.6,"recall":0.9941,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-202401","db_name":"ZillizCloud-2cu-cap-202401","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":170.5693,"latency":8.9,"recall":0.9605,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-202401","db_name":"ZillizCloud-2cu-cap-202401","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":94.7766,"latency":19.599999999999998,"recall":0.9843,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-202401","db_name":"ZillizCloud-2cu-cap-202401","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":44.8695,"latency":30.900000000000002,"recall":0.9966,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-202401","db_name":"ZillizCloud-1cu-perf-202401","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":722.0315,"latency":7.7,"recall":0.976,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-202401","db_name":"ZillizCloud-1cu-perf-202401","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":467.5795,"latency":8.8,"recall":0.9898,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-202401","db_name":"ZillizCloud-1cu-perf-202401","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":975.2503,"latency":8.200000000000001,"recall":0.9936,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-202401","db_name":"ZillizCloud-1cu-perf-202401","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":873.3712,"latency":6.7,"recall":0.9477,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-202401","db_name":"ZillizCloud-1cu-perf-202401","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":544.6203,"latency":8.4,"recall":0.977,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-202401","db_name":"ZillizCloud-1cu-perf-202401","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":930.9164,"latency":9.6,"recall":0.9968,"label":1,"qp$":0.0}] \ No newline at end of file From 41d57d817a7ef68434063ab15c6dd477906259cb Mon Sep 17 00:00:00 2001 From: "min.tian" Date: Fri, 22 Mar 2024 11:44:53 +0800 Subject: [PATCH 052/327] add zillizcloud failed results Signed-off-by: min.tian --- .../result_20240105_202401_zillizcloud.json | 324 ++++++++++++++++++ vectordb_bench/results/leaderboard.json | 2 +- 2 files changed, 325 insertions(+), 1 deletion(-) diff --git a/vectordb_bench/results/ZillizCloud/result_20240105_202401_zillizcloud.json b/vectordb_bench/results/ZillizCloud/result_20240105_202401_zillizcloud.json index 2d4bd7ea6..afd6068a1 100644 --- a/vectordb_bench/results/ZillizCloud/result_20240105_202401_zillizcloud.json +++ b/vectordb_bench/results/ZillizCloud/result_20240105_202401_zillizcloud.json @@ -973,6 +973,330 @@ } }, "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-perf-202401", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 4, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-perf-202401", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 6, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-perf-202401", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 8, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-perf-202401", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 11, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-perf-202401", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 13, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-perf-202401", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 15, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-cap-202401", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 4, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-cap-202401", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 6, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-cap-202401", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 8, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-cap-202401", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 11, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-cap-202401", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 13, + "custom_case": {} + } + }, + "label": "x" + }, + { + "metrics": { + "max_load_count": 0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0 + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "1cu-cap-202401", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE" + }, + "case_config": { + "case_id": 15, + "custom_case": {} + } + }, + "label": "x" } ], "file_fmt": "result_{}_{}_{}.json" diff --git a/vectordb_bench/results/leaderboard.json b/vectordb_bench/results/leaderboard.json index 0d9d2e1d9..5d8705571 100644 --- a/vectordb_bench/results/leaderboard.json +++ b/vectordb_bench/results/leaderboard.json @@ -1 +1 @@ -[{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":0.8836,"latency":2523.0,"recall":0.8528,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":0.8937,"latency":3720.2000000000003,"recall":0.8525,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":1.2145,"latency":3622.3999999999996,"recall":0.7487,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":10.6271,"latency":730.7,"recall":0.8898,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":10.8507,"latency":733.1999999999999,"recall":0.8897,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":75.7055,"latency":121.2,"recall":0.9999,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":11.2945,"latency":3611.2000000000003,"recall":0.996,"label":1,"qp$":84832.4640100146},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":17.3271,"latency":3774.7999999999997,"recall":0.9961,"label":1,"qp$":130143.04193615691},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":26.26,"latency":556.1,"recall":0.9999,"label":1,"qp$":197237.6382224077},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":15.2269,"latency":861.8,"recall":0.9888,"label":1,"qp$":114368.53745044857},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":15.1749,"latency":774.3,"recall":0.989,"label":1,"qp$":113977.96786981013},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":27.6181,"latency":305.5,"recall":0.9999,"label":1,"qp$":207438.26413519715},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":228.4,"latency":22.2,"recall":0.9348,"label":1,"qp$":5631780.821917809},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":181.5,"latency":26.1,"recall":0.9345,"label":1,"qp$":4475342.465753425},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":205.7,"latency":24.2,"recall":0.9586,"label":1,"qp$":5072054.794520548},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":67.63,"latency":36.0,"recall":0.8064,"label":1,"qp$":2502240.4933196297},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":63.35,"latency":38.4,"recall":0.8065,"label":1,"qp$":2343884.892086331},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":176.7,"latency":27.599999999999998,"recall":1.0,"label":1,"qp$":6537718.396711202},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":15.33,"latency":84.9,"recall":0.8064,"label":1,"qp$":567194.2446043165},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":15.13,"latency":86.7,"recall":0.8065,"label":1,"qp$":559794.4501541625},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":17.41,"latency":74.30000000000001,"recall":1.0,"label":1,"qp$":644152.1068859198},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":16.34,"latency":88.7,"recall":0.879,"label":1,"qp$":301661.53846153844},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":10.45,"latency":126.8,"recall":0.8208,"label":1,"qp$":192923.0769230769},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":16.18,"latency":87.5,"recall":0.8793,"label":1,"qp$":298707.69230769225},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":9.8,"latency":130.89999999999998,"recall":0.8212,"label":1,"qp$":180923.0769230769},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":36.11,"latency":55.1,"recall":1.0,"label":1,"qp$":666646.1538461539},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":14.84,"latency":92.7,"recall":0.96,"label":1,"qp$":273969.23076923075},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":322.7,"latency":26.4,"recall":0.9478,"label":1,"qp$":994623.287671233},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":265.5,"latency":26.9,"recall":0.9332,"label":1,"qp$":818321.9178082192},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":303.8,"latency":27.3,"recall":0.9478,"label":1,"qp$":936369.8630136987},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":180.2,"latency":28.2,"recall":0.9335,"label":1,"qp$":555410.9589041097},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":730.7,"latency":24.6,"recall":0.9586,"label":1,"qp$":2252157.5342465756},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":104.3,"latency":31.7,"recall":0.9563,"label":1,"qp$":321472.602739726},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":147.7,"latency":35.3,"recall":0.9707,"label":1,"qp$":682567.3940949935},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":782.5,"latency":25.9,"recall":1.0,"label":1,"qp$":3616174.5827984596},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":18.7634,"latency":153.70000000000002,"recall":0.8737,"label":1,"qp$":694226.515930113},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":18.3619,"latency":79.8,"recall":0.8741,"label":1,"qp$":679371.4285714285},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":25.2744,"latency":61.199999999999996,"recall":0.9979,"label":1,"qp$":935126.8242548818},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":261.798,"latency":23.099999999999998,"recall":0.9262,"label":1,"qp$":6455293.150684931},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":166.1851,"latency":23.900000000000002,"recall":0.9264,"label":1,"qp$":4097714.7945205485},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":121.7169,"latency":29.0,"recall":0.9693,"label":1,"qp$":3001238.6301369863},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":46.6189,"latency":43.1,"recall":0.8737,"label":1,"qp$":1724851.3874614593},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":42.4856,"latency":44.0,"recall":0.8741,"label":1,"qp$":1571923.5354573485},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":138.9479,"latency":26.200000000000003,"recall":0.9979,"label":1,"qp$":5140929.496402878},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":20.7437,"latency":75.80000000000001,"recall":0.9291,"label":1,"qp$":382960.6153846154},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":20.2993,"latency":76.5,"recall":0.9293,"label":1,"qp$":374756.30769230763},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":26.4719,"latency":67.0,"recall":1.0,"label":1,"qp$":488712.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":8.6675,"latency":180.2,"recall":0.8369,"label":1,"qp$":160015.38461538462},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":7.8121,"latency":167.7,"recall":0.8369,"label":1,"qp$":144223.3846153846},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":16.869,"latency":87.8,"recall":0.9814,"label":1,"qp$":311427.6923076923},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":365.0835,"latency":23.599999999999998,"recall":0.945,"label":1,"qp$":1125257.3630136987},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":325.5271,"latency":25.1,"recall":0.9452,"label":1,"qp$":1003336.9520547946},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":596.7942,"latency":24.2,"recall":0.9693,"label":1,"qp$":1839434.1780821919},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":303.2044,"latency":27.400000000000002,"recall":0.9246,"label":1,"qp$":934534.1095890411},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":136.0345,"latency":31.9,"recall":0.9244,"label":1,"qp$":419284.4178082192},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":66.7221,"latency":42.1,"recall":0.963,"label":1,"qp$":205650.30821917808},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":131.2549,"latency":30.200000000000003,"recall":0.9867,"label":1,"qp$":606569.4993581515},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":127.9337,"latency":30.099999999999998,"recall":0.9869,"label":1,"qp$":591221.2066752247},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":595.8462,"latency":23.400000000000002,"recall":1.0,"label":1,"qp$":2753589.627727856},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":330.0144,"latency":9.0,"recall":0.9507,"label":1,"qp$":7472024.150943397},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":271.6585,"latency":10.1,"recall":0.9678,"label":1,"qp$":6150758.490566038},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":216.5226,"latency":12.9,"recall":1.0,"label":1,"qp$":4902398.490566038},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":123.9553,"latency":23.0,"recall":0.971,"label":1,"qp$":1403267.5471698113},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":59.1479,"latency":44.5,"recall":0.9906,"label":1,"qp$":669598.8679245282},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":40.999,"latency":55.300000000000004,"recall":1.0,"label":1,"qp$":464139.62264150946},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":579.9416,"latency":9.4,"recall":0.9213,"label":1,"qp$":6565376.603773585},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":425.2529,"latency":11.299999999999999,"recall":0.9686,"label":1,"qp$":4814183.773584905},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":397.0539,"latency":13.799999999999999,"recall":1.0,"label":1,"qp$":4494949.811320755},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":516.27,"latency":7.0,"recall":0.9463,"label":1,"qp$":11689132.075471697},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":354.8416,"latency":10.0,"recall":0.9802,"label":1,"qp$":8034149.433962264},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":427.5229,"latency":8.7,"recall":1.0,"label":1,"qp$":9679763.773584906},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":2884.689,"latency":5.3,"recall":0.8801,"label":1,"qp$":8164214.150943396},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":1689.5799,"latency":6.6,"recall":0.9493,"label":1,"qp$":4781829.905660377},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1517.6792,"latency":10.0,"recall":1.0,"label":1,"qp$":4295318.4905660385},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":822.5318,"latency":5.6,"recall":0.9294,"label":1,"qp$":2327920.1886792453},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":378.9146,"latency":10.3,"recall":0.9758,"label":1,"qp$":1072399.8113207547},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":218.6854,"latency":16.2,"recall":1.0,"label":1,"qp$":618920.9433962263},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":297.5,"latency":7.2,"recall":0.974,"label":1,"qp$":6735849.0566037735},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":228.3,"latency":10.6,"recall":0.994,"label":1,"qp$":5169056.603773585},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":584.0,"latency":4.6,"recall":1.0,"label":1,"qp$":13222641.509433962},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":1871.0,"latency":7.0,"recall":0.9602,"label":1,"qp$":5295283.018867925},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":556.7,"latency":6.7,"recall":0.9723,"label":1,"qp$":1575566.0377358492},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1583.0,"latency":6.8,"recall":0.9836,"label":1,"qp$":4480188.679245283},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":294.3,"latency":10.9,"recall":0.9939,"label":1,"qp$":832924.5283018869},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":2345.0,"latency":8.9,"recall":1.0,"label":1,"qp$":6636792.452830189},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":295.6,"latency":12.3,"recall":1.0,"label":1,"qp$":836603.7735849057},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":143.0,"latency":33.5,"recall":0.9818,"label":1,"qp$":3237735.8490566034},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":106.0,"latency":20.7,"recall":0.9887,"label":1,"qp$":2400000.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":189.0,"latency":11.6,"recall":1.0,"label":1,"qp$":4279245.2830188675},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":379.9721,"latency":12.4,"recall":0.982,"label":1,"qp$":4301570.9433962265},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":71.74,"latency":50.8,"recall":0.9883,"label":1,"qp$":812150.9433962264},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":287.0,"latency":14.9,"recall":0.9865,"label":1,"qp$":3249056.603773585},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":34.6654,"latency":64.69999999999999,"recall":0.9961,"label":1,"qp$":392438.4905660377},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":412.0,"latency":10.3,"recall":1.0,"label":1,"qp$":4664150.9433962265},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":42.169,"latency":46.800000000000004,"recall":1.0,"label":1,"qp$":477384.9056603773},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":537.4975,"latency":18.9,"recall":0.8903,"label":1,"qp$":1356936.1851332397},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":372.0466,"latency":17.8,"recall":0.8904,"label":1,"qp$":939248.0785413746},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1156.2898,"latency":14.4,"recall":0.9989,"label":1,"qp$":2919104.684431978},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":110.248,"latency":69.0,"recall":0.898,"label":1,"qp$":278325.94670406735},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":87.2601,"latency":27.799999999999997,"recall":0.898,"label":1,"qp$":220291.97755960727},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":125.7846,"latency":23.099999999999998,"recall":0.975,"label":1,"qp$":317548.77980364655},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":240.7209,"latency":17.4,"recall":0.8887,"label":1,"qp$":3038552.734922861},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":189.4399,"latency":17.5,"recall":0.8889,"label":1,"qp$":2391246.98457223},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":313.5116,"latency":16.1,"recall":0.9999,"label":1,"qp$":3957369.424964937},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":78.7196,"latency":49.4,"recall":0.9203,"label":1,"qp$":1707172.048192771},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":68.3111,"latency":35.5,"recall":0.9202,"label":1,"qp$":1481445.5421686745},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":210.2147,"latency":26.8,"recall":0.9996,"label":1,"qp$":4558873.012048192},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":188.6436,"latency":917.5,"recall":0.9175,"label":1,"qp$":2381195.5119214584},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":155.6991,"latency":917.1,"recall":0.9171,"label":1,"qp$":1965346.283309958},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":445.3289,"latency":14.1,"recall":0.9999,"label":1,"qp$":5621262.412342216},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":633.6033,"latency":24.6,"recall":0.919,"label":1,"qp$":1599559.5231416551},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":95.5682,"latency":58.7,"recall":0.9463,"label":1,"qp$":241266.14305750353},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":434.4062,"latency":17.4,"recall":0.9181,"label":1,"qp$":1096677.643758766},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":80.3192,"latency":26.599999999999998,"recall":0.9462,"label":1,"qp$":202769.3688639551},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":1509.3293,"latency":18.5,"recall":0.9995,"label":1,"qp$":3810368.4992987383},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":166.7252,"latency":19.599999999999998,"recall":0.9988,"label":1,"qp$":420905.1332398317},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":46.8622,"latency":123.0,"recall":0.9957,"label":1,"qp$":16703.358415841583},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1.8388,"latency":1063.5,"recall":0.9957,"label":1,"qp$":655.4138613861386},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":45.0666,"latency":109.2,"recall":1.0,"label":1,"qp$":16063.342574257427},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":43.5017,"latency":228.7,"recall":0.9957,"label":1,"qp$":4803.8687116564415},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1.5668,"latency":1114.4,"recall":0.9957,"label":1,"qp$":173.02085889570552},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":41.5443,"latency":159.0,"recall":1.0,"label":1,"qp$":4587.714110429448},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":67.9121,"latency":179.5,"recall":0.9909,"label":1,"qp$":7499.495705521471},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.7636,"latency":1921.3,"recall":0.9908,"label":1,"qp$":84.32392638036809},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":32.0,"latency":124.5,"recall":1.0,"label":1,"qp$":3533.7423312883434},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":63.1365,"latency":145.7,"recall":0.991,"label":1,"qp$":22504.09900990099},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.7512,"latency":1983.8,"recall":0.9908,"label":1,"qp$":267.7544554455445},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":30.1358,"latency":129.8,"recall":1.0,"label":1,"qp$":10741.473267326734},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":37.432,"latency":75.0,"recall":0.9975,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":37.0696,"latency":73.5,"recall":0.9976,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":81.1915,"latency":53.0,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":321.6048,"latency":13.4,"recall":0.989,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":22.1467,"latency":86.8,"recall":0.9972,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":303.2551,"latency":10.9,"recall":0.9876,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":21.5388,"latency":81.69999999999999,"recall":0.997,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":394.5418,"latency":8.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":37.878,"latency":45.0,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":180.2757,"latency":6.0,"recall":0.9942,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":179.0033,"latency":6.4,"recall":0.9943,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":526.8846,"latency":3.6,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":626.5243,"latency":6.2,"recall":0.9954,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":78.4227,"latency":25.3,"recall":0.9982,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":599.4213,"latency":6.6,"recall":0.9955,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":78.5351,"latency":26.3,"recall":0.9982,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":2098.2113,"latency":3.4,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":275.6292,"latency":10.0,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":100.6667,"latency":21.1,"recall":0.9909,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":101.1399,"latency":19.7,"recall":0.9907,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":52.2606,"latency":18.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":61.0661,"latency":49.8,"recall":0.9911,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":58.9326,"latency":44.6,"recall":0.9911,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":42.5977,"latency":54.9,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":536.0726,"latency":8.200000000000001,"recall":0.9728,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":467.179,"latency":7.0,"recall":0.9697,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":431.7512,"latency":8.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":274.5407,"latency":4.8999999999999995,"recall":0.9807,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":236.5672,"latency":10.3,"recall":0.981,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":309.4833,"latency":4.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":178.6585,"latency":13.700000000000001,"recall":0.9843,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":178.3732,"latency":15.0,"recall":0.9844,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":229.3526,"latency":12.5,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":1258.7043,"latency":4.8999999999999995,"recall":0.9799,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":1075.8776,"latency":5.3,"recall":0.98,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1494.8493,"latency":4.7,"recall":1.0,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-202401","db_name":"ZillizCloud-8cu-perf-202401","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":5115.5303,"latency":8.7,"recall":0.9469,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-202401","db_name":"ZillizCloud-8cu-perf-202401","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":3685.0767,"latency":9.1,"recall":0.9736,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-202401","db_name":"ZillizCloud-8cu-perf-202401","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":4742.1617,"latency":9.1,"recall":0.9936,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-202401","db_name":"ZillizCloud-8cu-perf-202401","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":6054.4428,"latency":8.1,"recall":0.9155,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-202401","db_name":"ZillizCloud-8cu-perf-202401","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":4104.2598,"latency":10.6,"recall":0.9506,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-202401","db_name":"ZillizCloud-8cu-perf-202401","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":4252.1267,"latency":9.2,"recall":0.9964,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-202401","db_name":"ZillizCloud-8cu-perf-202401","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":1685.3091,"latency":13.299999999999999,"recall":0.9718,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-202401","db_name":"ZillizCloud-8cu-perf-202401","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":769.8991,"latency":10.7,"recall":0.9884,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-202401","db_name":"ZillizCloud-8cu-perf-202401","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":945.4061,"latency":10.9,"recall":0.9941,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-202401","db_name":"ZillizCloud-8cu-perf-202401","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":2214.9028,"latency":8.4,"recall":0.9249,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-202401","db_name":"ZillizCloud-8cu-perf-202401","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":827.975,"latency":12.0,"recall":0.9692,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-202401","db_name":"ZillizCloud-8cu-perf-202401","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":776.9454,"latency":11.4,"recall":0.9966,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-202401","db_name":"ZillizCloud-1cu-cap-202401","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":269.5464,"latency":9.799999999999999,"recall":0.9776,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-202401","db_name":"ZillizCloud-1cu-cap-202401","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":240.0363,"latency":10.6,"recall":0.9822,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-202401","db_name":"ZillizCloud-1cu-cap-202401","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":218.0627,"latency":11.4,"recall":0.9936,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-202401","db_name":"ZillizCloud-1cu-cap-202401","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":392.8825,"latency":6.8999999999999995,"recall":0.9581,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-202401","db_name":"ZillizCloud-1cu-cap-202401","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":343.8204,"latency":8.4,"recall":0.968,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-202401","db_name":"ZillizCloud-1cu-cap-202401","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":216.6773,"latency":10.1,"recall":0.9968,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-202401","db_name":"ZillizCloud-2cu-cap-202401","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":503.2284,"latency":9.0,"recall":0.9677,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-202401","db_name":"ZillizCloud-2cu-cap-202401","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":413.3232,"latency":9.6,"recall":0.981,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-202401","db_name":"ZillizCloud-2cu-cap-202401","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":425.5492,"latency":10.200000000000001,"recall":0.9936,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-202401","db_name":"ZillizCloud-2cu-cap-202401","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":789.1229,"latency":5.6,"recall":0.9396,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-202401","db_name":"ZillizCloud-2cu-cap-202401","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":571.4257,"latency":7.7,"recall":0.9668,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-202401","db_name":"ZillizCloud-2cu-cap-202401","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":411.7653,"latency":9.1,"recall":0.9968,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-202401","db_name":"ZillizCloud-2cu-cap-202401","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":98.0448,"latency":16.1,"recall":0.9803,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-202401","db_name":"ZillizCloud-2cu-cap-202401","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":58.3152,"latency":28.0,"recall":0.9891,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-202401","db_name":"ZillizCloud-2cu-cap-202401","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":46.8304,"latency":28.6,"recall":0.9941,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-202401","db_name":"ZillizCloud-2cu-cap-202401","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":170.5693,"latency":8.9,"recall":0.9605,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-202401","db_name":"ZillizCloud-2cu-cap-202401","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":94.7766,"latency":19.599999999999998,"recall":0.9843,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-202401","db_name":"ZillizCloud-2cu-cap-202401","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":44.8695,"latency":30.900000000000002,"recall":0.9966,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-202401","db_name":"ZillizCloud-1cu-perf-202401","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":722.0315,"latency":7.7,"recall":0.976,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-202401","db_name":"ZillizCloud-1cu-perf-202401","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":467.5795,"latency":8.8,"recall":0.9898,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-202401","db_name":"ZillizCloud-1cu-perf-202401","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":975.2503,"latency":8.200000000000001,"recall":0.9936,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-202401","db_name":"ZillizCloud-1cu-perf-202401","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":873.3712,"latency":6.7,"recall":0.9477,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-202401","db_name":"ZillizCloud-1cu-perf-202401","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":544.6203,"latency":8.4,"recall":0.977,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-202401","db_name":"ZillizCloud-1cu-perf-202401","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":930.9164,"latency":9.6,"recall":0.9968,"label":1,"qp$":0.0}] \ No newline at end of file +[{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":0.8836,"latency":2523.0,"recall":0.8528,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":0.8937,"latency":3720.2000000000003,"recall":0.8525,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":1.2145,"latency":3622.3999999999996,"recall":0.7487,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":10.6271,"latency":730.7,"recall":0.8898,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":10.8507,"latency":733.1999999999999,"recall":0.8897,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":75.7055,"latency":121.2,"recall":0.9999,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":11.2945,"latency":3611.2000000000003,"recall":0.996,"label":1,"qp$":84832.4640100146},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":17.3271,"latency":3774.7999999999997,"recall":0.9961,"label":1,"qp$":130143.04193615691},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":26.26,"latency":556.1,"recall":0.9999,"label":1,"qp$":197237.6382224077},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":15.2269,"latency":861.8,"recall":0.9888,"label":1,"qp$":114368.53745044857},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":15.1749,"latency":774.3,"recall":0.989,"label":1,"qp$":113977.96786981013},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":27.6181,"latency":305.5,"recall":0.9999,"label":1,"qp$":207438.26413519715},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":228.4,"latency":22.2,"recall":0.9348,"label":1,"qp$":5631780.821917809},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":181.5,"latency":26.1,"recall":0.9345,"label":1,"qp$":4475342.465753425},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":205.7,"latency":24.2,"recall":0.9586,"label":1,"qp$":5072054.794520548},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":67.63,"latency":36.0,"recall":0.8064,"label":1,"qp$":2502240.4933196297},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":63.35,"latency":38.4,"recall":0.8065,"label":1,"qp$":2343884.892086331},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":176.7,"latency":27.599999999999998,"recall":1.0,"label":1,"qp$":6537718.396711202},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":15.33,"latency":84.9,"recall":0.8064,"label":1,"qp$":567194.2446043165},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":15.13,"latency":86.7,"recall":0.8065,"label":1,"qp$":559794.4501541625},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":17.41,"latency":74.30000000000001,"recall":1.0,"label":1,"qp$":644152.1068859198},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":16.34,"latency":88.7,"recall":0.879,"label":1,"qp$":301661.53846153844},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":10.45,"latency":126.8,"recall":0.8208,"label":1,"qp$":192923.0769230769},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":16.18,"latency":87.5,"recall":0.8793,"label":1,"qp$":298707.69230769225},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":9.8,"latency":130.89999999999998,"recall":0.8212,"label":1,"qp$":180923.0769230769},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":36.11,"latency":55.1,"recall":1.0,"label":1,"qp$":666646.1538461539},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":14.84,"latency":92.7,"recall":0.96,"label":1,"qp$":273969.23076923075},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":322.7,"latency":26.4,"recall":0.9478,"label":1,"qp$":994623.287671233},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":265.5,"latency":26.9,"recall":0.9332,"label":1,"qp$":818321.9178082192},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":303.8,"latency":27.3,"recall":0.9478,"label":1,"qp$":936369.8630136987},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":180.2,"latency":28.2,"recall":0.9335,"label":1,"qp$":555410.9589041097},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":730.7,"latency":24.6,"recall":0.9586,"label":1,"qp$":2252157.5342465756},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":104.3,"latency":31.7,"recall":0.9563,"label":1,"qp$":321472.602739726},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":147.7,"latency":35.3,"recall":0.9707,"label":1,"qp$":682567.3940949935},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":782.5,"latency":25.9,"recall":1.0,"label":1,"qp$":3616174.5827984596},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":18.7634,"latency":153.70000000000002,"recall":0.8737,"label":1,"qp$":694226.515930113},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":18.3619,"latency":79.8,"recall":0.8741,"label":1,"qp$":679371.4285714285},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":25.2744,"latency":61.199999999999996,"recall":0.9979,"label":1,"qp$":935126.8242548818},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":261.798,"latency":23.099999999999998,"recall":0.9262,"label":1,"qp$":6455293.150684931},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":166.1851,"latency":23.900000000000002,"recall":0.9264,"label":1,"qp$":4097714.7945205485},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":121.7169,"latency":29.0,"recall":0.9693,"label":1,"qp$":3001238.6301369863},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":46.6189,"latency":43.1,"recall":0.8737,"label":1,"qp$":1724851.3874614593},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":42.4856,"latency":44.0,"recall":0.8741,"label":1,"qp$":1571923.5354573485},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":138.9479,"latency":26.200000000000003,"recall":0.9979,"label":1,"qp$":5140929.496402878},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":20.7437,"latency":75.80000000000001,"recall":0.9291,"label":1,"qp$":382960.6153846154},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":20.2993,"latency":76.5,"recall":0.9293,"label":1,"qp$":374756.30769230763},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":26.4719,"latency":67.0,"recall":1.0,"label":1,"qp$":488712.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":8.6675,"latency":180.2,"recall":0.8369,"label":1,"qp$":160015.38461538462},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":7.8121,"latency":167.7,"recall":0.8369,"label":1,"qp$":144223.3846153846},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":16.869,"latency":87.8,"recall":0.9814,"label":1,"qp$":311427.6923076923},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":365.0835,"latency":23.599999999999998,"recall":0.945,"label":1,"qp$":1125257.3630136987},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":325.5271,"latency":25.1,"recall":0.9452,"label":1,"qp$":1003336.9520547946},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":596.7942,"latency":24.2,"recall":0.9693,"label":1,"qp$":1839434.1780821919},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":303.2044,"latency":27.400000000000002,"recall":0.9246,"label":1,"qp$":934534.1095890411},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":136.0345,"latency":31.9,"recall":0.9244,"label":1,"qp$":419284.4178082192},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":66.7221,"latency":42.1,"recall":0.963,"label":1,"qp$":205650.30821917808},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":131.2549,"latency":30.200000000000003,"recall":0.9867,"label":1,"qp$":606569.4993581515},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":127.9337,"latency":30.099999999999998,"recall":0.9869,"label":1,"qp$":591221.2066752247},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":595.8462,"latency":23.400000000000002,"recall":1.0,"label":1,"qp$":2753589.627727856},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":330.0144,"latency":9.0,"recall":0.9507,"label":1,"qp$":7472024.150943397},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":271.6585,"latency":10.1,"recall":0.9678,"label":1,"qp$":6150758.490566038},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":216.5226,"latency":12.9,"recall":1.0,"label":1,"qp$":4902398.490566038},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":123.9553,"latency":23.0,"recall":0.971,"label":1,"qp$":1403267.5471698113},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":59.1479,"latency":44.5,"recall":0.9906,"label":1,"qp$":669598.8679245282},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":40.999,"latency":55.300000000000004,"recall":1.0,"label":1,"qp$":464139.62264150946},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":579.9416,"latency":9.4,"recall":0.9213,"label":1,"qp$":6565376.603773585},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":425.2529,"latency":11.299999999999999,"recall":0.9686,"label":1,"qp$":4814183.773584905},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":397.0539,"latency":13.799999999999999,"recall":1.0,"label":1,"qp$":4494949.811320755},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":516.27,"latency":7.0,"recall":0.9463,"label":1,"qp$":11689132.075471697},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":354.8416,"latency":10.0,"recall":0.9802,"label":1,"qp$":8034149.433962264},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":427.5229,"latency":8.7,"recall":1.0,"label":1,"qp$":9679763.773584906},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":2884.689,"latency":5.3,"recall":0.8801,"label":1,"qp$":8164214.150943396},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":1689.5799,"latency":6.6,"recall":0.9493,"label":1,"qp$":4781829.905660377},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1517.6792,"latency":10.0,"recall":1.0,"label":1,"qp$":4295318.4905660385},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":822.5318,"latency":5.6,"recall":0.9294,"label":1,"qp$":2327920.1886792453},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":378.9146,"latency":10.3,"recall":0.9758,"label":1,"qp$":1072399.8113207547},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":218.6854,"latency":16.2,"recall":1.0,"label":1,"qp$":618920.9433962263},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":297.5,"latency":7.2,"recall":0.974,"label":1,"qp$":6735849.0566037735},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":228.3,"latency":10.6,"recall":0.994,"label":1,"qp$":5169056.603773585},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":584.0,"latency":4.6,"recall":1.0,"label":1,"qp$":13222641.509433962},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":1871.0,"latency":7.0,"recall":0.9602,"label":1,"qp$":5295283.018867925},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":556.7,"latency":6.7,"recall":0.9723,"label":1,"qp$":1575566.0377358492},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1583.0,"latency":6.8,"recall":0.9836,"label":1,"qp$":4480188.679245283},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":294.3,"latency":10.9,"recall":0.9939,"label":1,"qp$":832924.5283018869},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":2345.0,"latency":8.9,"recall":1.0,"label":1,"qp$":6636792.452830189},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":295.6,"latency":12.3,"recall":1.0,"label":1,"qp$":836603.7735849057},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":143.0,"latency":33.5,"recall":0.9818,"label":1,"qp$":3237735.8490566034},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":106.0,"latency":20.7,"recall":0.9887,"label":1,"qp$":2400000.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":189.0,"latency":11.6,"recall":1.0,"label":1,"qp$":4279245.2830188675},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":379.9721,"latency":12.4,"recall":0.982,"label":1,"qp$":4301570.9433962265},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":71.74,"latency":50.8,"recall":0.9883,"label":1,"qp$":812150.9433962264},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":287.0,"latency":14.9,"recall":0.9865,"label":1,"qp$":3249056.603773585},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":34.6654,"latency":64.69999999999999,"recall":0.9961,"label":1,"qp$":392438.4905660377},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":412.0,"latency":10.3,"recall":1.0,"label":1,"qp$":4664150.9433962265},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":42.169,"latency":46.800000000000004,"recall":1.0,"label":1,"qp$":477384.9056603773},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":537.4975,"latency":18.9,"recall":0.8903,"label":1,"qp$":1356936.1851332397},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":372.0466,"latency":17.8,"recall":0.8904,"label":1,"qp$":939248.0785413746},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1156.2898,"latency":14.4,"recall":0.9989,"label":1,"qp$":2919104.684431978},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":110.248,"latency":69.0,"recall":0.898,"label":1,"qp$":278325.94670406735},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":87.2601,"latency":27.799999999999997,"recall":0.898,"label":1,"qp$":220291.97755960727},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":125.7846,"latency":23.099999999999998,"recall":0.975,"label":1,"qp$":317548.77980364655},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":240.7209,"latency":17.4,"recall":0.8887,"label":1,"qp$":3038552.734922861},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":189.4399,"latency":17.5,"recall":0.8889,"label":1,"qp$":2391246.98457223},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":313.5116,"latency":16.1,"recall":0.9999,"label":1,"qp$":3957369.424964937},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":78.7196,"latency":49.4,"recall":0.9203,"label":1,"qp$":1707172.048192771},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":68.3111,"latency":35.5,"recall":0.9202,"label":1,"qp$":1481445.5421686745},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":210.2147,"latency":26.8,"recall":0.9996,"label":1,"qp$":4558873.012048192},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":188.6436,"latency":917.5,"recall":0.9175,"label":1,"qp$":2381195.5119214584},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":155.6991,"latency":917.1,"recall":0.9171,"label":1,"qp$":1965346.283309958},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":445.3289,"latency":14.1,"recall":0.9999,"label":1,"qp$":5621262.412342216},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":633.6033,"latency":24.6,"recall":0.919,"label":1,"qp$":1599559.5231416551},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":95.5682,"latency":58.7,"recall":0.9463,"label":1,"qp$":241266.14305750353},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":434.4062,"latency":17.4,"recall":0.9181,"label":1,"qp$":1096677.643758766},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":80.3192,"latency":26.599999999999998,"recall":0.9462,"label":1,"qp$":202769.3688639551},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":1509.3293,"latency":18.5,"recall":0.9995,"label":1,"qp$":3810368.4992987383},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":166.7252,"latency":19.599999999999998,"recall":0.9988,"label":1,"qp$":420905.1332398317},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":46.8622,"latency":123.0,"recall":0.9957,"label":1,"qp$":16703.358415841583},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1.8388,"latency":1063.5,"recall":0.9957,"label":1,"qp$":655.4138613861386},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":45.0666,"latency":109.2,"recall":1.0,"label":1,"qp$":16063.342574257427},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":43.5017,"latency":228.7,"recall":0.9957,"label":1,"qp$":4803.8687116564415},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1.5668,"latency":1114.4,"recall":0.9957,"label":1,"qp$":173.02085889570552},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":41.5443,"latency":159.0,"recall":1.0,"label":1,"qp$":4587.714110429448},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":67.9121,"latency":179.5,"recall":0.9909,"label":1,"qp$":7499.495705521471},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.7636,"latency":1921.3,"recall":0.9908,"label":1,"qp$":84.32392638036809},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":32.0,"latency":124.5,"recall":1.0,"label":1,"qp$":3533.7423312883434},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":63.1365,"latency":145.7,"recall":0.991,"label":1,"qp$":22504.09900990099},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.7512,"latency":1983.8,"recall":0.9908,"label":1,"qp$":267.7544554455445},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":30.1358,"latency":129.8,"recall":1.0,"label":1,"qp$":10741.473267326734},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":37.432,"latency":75.0,"recall":0.9975,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":37.0696,"latency":73.5,"recall":0.9976,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":81.1915,"latency":53.0,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":321.6048,"latency":13.4,"recall":0.989,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":22.1467,"latency":86.8,"recall":0.9972,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":303.2551,"latency":10.9,"recall":0.9876,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":21.5388,"latency":81.69999999999999,"recall":0.997,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":394.5418,"latency":8.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":37.878,"latency":45.0,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":180.2757,"latency":6.0,"recall":0.9942,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":179.0033,"latency":6.4,"recall":0.9943,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":526.8846,"latency":3.6,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":626.5243,"latency":6.2,"recall":0.9954,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":78.4227,"latency":25.3,"recall":0.9982,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":599.4213,"latency":6.6,"recall":0.9955,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":78.5351,"latency":26.3,"recall":0.9982,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":2098.2113,"latency":3.4,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":275.6292,"latency":10.0,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":100.6667,"latency":21.1,"recall":0.9909,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":101.1399,"latency":19.7,"recall":0.9907,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":52.2606,"latency":18.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":61.0661,"latency":49.8,"recall":0.9911,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":58.9326,"latency":44.6,"recall":0.9911,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":42.5977,"latency":54.9,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":536.0726,"latency":8.200000000000001,"recall":0.9728,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":467.179,"latency":7.0,"recall":0.9697,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":431.7512,"latency":8.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":274.5407,"latency":4.8999999999999995,"recall":0.9807,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":236.5672,"latency":10.3,"recall":0.981,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":309.4833,"latency":4.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":178.6585,"latency":13.700000000000001,"recall":0.9843,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":178.3732,"latency":15.0,"recall":0.9844,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":229.3526,"latency":12.5,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":1258.7043,"latency":4.8999999999999995,"recall":0.9799,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":1075.8776,"latency":5.3,"recall":0.98,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1494.8493,"latency":4.7,"recall":1.0,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-202401","db_name":"ZillizCloud-8cu-perf-202401","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":5115.5303,"latency":8.7,"recall":0.9469,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-202401","db_name":"ZillizCloud-8cu-perf-202401","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":3685.0767,"latency":9.1,"recall":0.9736,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-202401","db_name":"ZillizCloud-8cu-perf-202401","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":4742.1617,"latency":9.1,"recall":0.9936,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-202401","db_name":"ZillizCloud-8cu-perf-202401","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":6054.4428,"latency":8.1,"recall":0.9155,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-202401","db_name":"ZillizCloud-8cu-perf-202401","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":4104.2598,"latency":10.6,"recall":0.9506,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-202401","db_name":"ZillizCloud-8cu-perf-202401","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":4252.1267,"latency":9.2,"recall":0.9964,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-202401","db_name":"ZillizCloud-8cu-perf-202401","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":1685.3091,"latency":13.299999999999999,"recall":0.9718,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-202401","db_name":"ZillizCloud-8cu-perf-202401","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":769.8991,"latency":10.7,"recall":0.9884,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-202401","db_name":"ZillizCloud-8cu-perf-202401","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":945.4061,"latency":10.9,"recall":0.9941,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-202401","db_name":"ZillizCloud-8cu-perf-202401","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":2214.9028,"latency":8.4,"recall":0.9249,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-202401","db_name":"ZillizCloud-8cu-perf-202401","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":827.975,"latency":12.0,"recall":0.9692,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-202401","db_name":"ZillizCloud-8cu-perf-202401","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":776.9454,"latency":11.4,"recall":0.9966,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-202401","db_name":"ZillizCloud-1cu-cap-202401","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":269.5464,"latency":9.799999999999999,"recall":0.9776,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-202401","db_name":"ZillizCloud-1cu-cap-202401","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":240.0363,"latency":10.6,"recall":0.9822,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-202401","db_name":"ZillizCloud-1cu-cap-202401","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":218.0627,"latency":11.4,"recall":0.9936,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-202401","db_name":"ZillizCloud-1cu-cap-202401","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":392.8825,"latency":6.8999999999999995,"recall":0.9581,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-202401","db_name":"ZillizCloud-1cu-cap-202401","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":343.8204,"latency":8.4,"recall":0.968,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-202401","db_name":"ZillizCloud-1cu-cap-202401","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":216.6773,"latency":10.1,"recall":0.9968,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-202401","db_name":"ZillizCloud-2cu-cap-202401","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":503.2284,"latency":9.0,"recall":0.9677,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-202401","db_name":"ZillizCloud-2cu-cap-202401","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":413.3232,"latency":9.6,"recall":0.981,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-202401","db_name":"ZillizCloud-2cu-cap-202401","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":425.5492,"latency":10.200000000000001,"recall":0.9936,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-202401","db_name":"ZillizCloud-2cu-cap-202401","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":789.1229,"latency":5.6,"recall":0.9396,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-202401","db_name":"ZillizCloud-2cu-cap-202401","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":571.4257,"latency":7.7,"recall":0.9668,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-202401","db_name":"ZillizCloud-2cu-cap-202401","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":411.7653,"latency":9.1,"recall":0.9968,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-202401","db_name":"ZillizCloud-2cu-cap-202401","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":98.0448,"latency":16.1,"recall":0.9803,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-202401","db_name":"ZillizCloud-2cu-cap-202401","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":58.3152,"latency":28.0,"recall":0.9891,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-202401","db_name":"ZillizCloud-2cu-cap-202401","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":46.8304,"latency":28.6,"recall":0.9941,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-202401","db_name":"ZillizCloud-2cu-cap-202401","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":170.5693,"latency":8.9,"recall":0.9605,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-202401","db_name":"ZillizCloud-2cu-cap-202401","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":94.7766,"latency":19.599999999999998,"recall":0.9843,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-202401","db_name":"ZillizCloud-2cu-cap-202401","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":44.8695,"latency":30.900000000000002,"recall":0.9966,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-202401","db_name":"ZillizCloud-1cu-perf-202401","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":722.0315,"latency":7.7,"recall":0.976,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-202401","db_name":"ZillizCloud-1cu-perf-202401","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":467.5795,"latency":8.8,"recall":0.9898,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-202401","db_name":"ZillizCloud-1cu-perf-202401","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":975.2503,"latency":8.200000000000001,"recall":0.9936,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-202401","db_name":"ZillizCloud-1cu-perf-202401","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":873.3712,"latency":6.7,"recall":0.9477,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-202401","db_name":"ZillizCloud-1cu-perf-202401","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":544.6203,"latency":8.4,"recall":0.977,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-202401","db_name":"ZillizCloud-1cu-perf-202401","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":930.9164,"latency":9.6,"recall":0.9968,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-202401","db_name":"ZillizCloud-1cu-perf-202401","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-202401","db_name":"ZillizCloud-1cu-perf-202401","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-202401","db_name":"ZillizCloud-1cu-perf-202401","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-202401","db_name":"ZillizCloud-1cu-perf-202401","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-202401","db_name":"ZillizCloud-1cu-perf-202401","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-202401","db_name":"ZillizCloud-1cu-perf-202401","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-202401","db_name":"ZillizCloud-1cu-cap-202401","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-202401","db_name":"ZillizCloud-1cu-cap-202401","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-202401","db_name":"ZillizCloud-1cu-cap-202401","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-202401","db_name":"ZillizCloud-1cu-cap-202401","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-202401","db_name":"ZillizCloud-1cu-cap-202401","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-202401","db_name":"ZillizCloud-1cu-cap-202401","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0}] \ No newline at end of file From 3ebdcc826233228bccea7276d2b04b4912206c7a Mon Sep 17 00:00:00 2001 From: Wahaj Ali Date: Tue, 19 Mar 2024 10:41:25 +0500 Subject: [PATCH 053/327] Add support for HNSW in pgvector --- vectordb_bench/backend/clients/__init__.py | 4 +- .../backend/clients/pgvector/config.py | 47 +++++++++++++++++-- .../backend/clients/pgvector/pgvector.py | 22 +++++++-- 3 files changed, 64 insertions(+), 9 deletions(-) diff --git a/vectordb_bench/backend/clients/__init__.py b/vectordb_bench/backend/clients/__init__.py index 3df11610b..bd5211587 100644 --- a/vectordb_bench/backend/clients/__init__.py +++ b/vectordb_bench/backend/clients/__init__.py @@ -142,8 +142,8 @@ def case_config_cls(self, index_type: IndexType | None = None) -> Type[DBCaseCon return WeaviateIndexConfig if self == DB.PgVector: - from .pgvector.config import PgVectorIndexConfig - return PgVectorIndexConfig + from .pgvector.config import _pgvector_case_config + return _pgvector_case_config.get(index_type) if self == DB.PgVectoRS: from .pgvecto_rs.config import _pgvecto_rs_case_config diff --git a/vectordb_bench/backend/clients/pgvector/config.py b/vectordb_bench/backend/clients/pgvector/config.py index 7d90e86d2..3cf19993c 100644 --- a/vectordb_bench/backend/clients/pgvector/config.py +++ b/vectordb_bench/backend/clients/pgvector/config.py @@ -1,5 +1,5 @@ from pydantic import BaseModel, SecretStr -from ..api import DBConfig, DBCaseConfig, MetricType +from ..api import DBConfig, DBCaseConfig, IndexType, MetricType POSTGRE_URL_PLACEHOLDER = "postgresql://%s:%s@%s/%s" @@ -23,6 +23,7 @@ def to_dict(self) -> dict: class PgVectorIndexConfig(BaseModel, DBCaseConfig): metric_type: MetricType | None = None + index: IndexType lists: int | None = 1000 probes: int | None = 10 @@ -47,15 +48,55 @@ def parse_metric_fun_str(self) -> str: return "max_inner_product" return "cosine_distance" + + +class HNSWConfig(PgVectorIndexConfig): + M: int + efConstruction: int + ef: int | None = None + index: IndexType = IndexType.HNSW + + def index_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "index_type": self.index.value, + "params": {"M": self.M, "efConstruction": self.efConstruction}, + } + + def index_param(self) -> dict: + return { + "m" : self.M, + "efConstruction" : self.efConstruction, + "metric" : self.parse_metric() + } + + def search_param(self) -> dict: + return { + "ef" : self.ef, + "metric_fun" : self.parse_metric_fun_str(), + "metric_fun_op" : self.parse_metric_fun_op(), + } + + +class IVFFlatConfig(PgVectorIndexConfig): + lists: int + probes: int | None = None + index: IndexType = IndexType.IVFFlat + def index_param(self) -> dict: return { "lists" : self.lists, "metric" : self.parse_metric() } - + def search_param(self) -> dict: return { "probes" : self.probes, "metric_fun" : self.parse_metric_fun_str(), "metric_fun_op" : self.parse_metric_fun_op(), - } \ No newline at end of file + } + +_pgvector_case_config = { + IndexType.HNSW: HNSWConfig, + IndexType.IVFFlat: IVFFlatConfig, +} \ No newline at end of file diff --git a/vectordb_bench/backend/clients/pgvector/pgvector.py b/vectordb_bench/backend/clients/pgvector/pgvector.py index e0fc8d3b8..a69b7cbde 100644 --- a/vectordb_bench/backend/clients/pgvector/pgvector.py +++ b/vectordb_bench/backend/clients/pgvector/pgvector.py @@ -8,7 +8,7 @@ import psycopg2 import psycopg2.extras -from ..api import VectorDB, DBCaseConfig +from ..api import IndexType, VectorDB, DBCaseConfig log = logging.getLogger(__name__) @@ -108,7 +108,14 @@ def _create_index(self): assert self.cursor is not None, "Cursor is not initialized" index_param = self.case_config.index_param() - self.cursor.execute(f'CREATE INDEX IF NOT EXISTS {self._index_name} ON public."{self.table_name}" USING ivfflat (embedding {index_param["metric"]}) WITH (lists={index_param["lists"]});') + if self.case_config.index == IndexType.HNSW: + log.debug(f'Creating HNSW index. m={index_param["m"]}, ef_construction={index_param["ef_construction"]}') + self.cursor.execute(f'CREATE INDEX IF NOT EXISTS {self._index_name} ON public."{self.table_name}" USING hnsw (embedding {index_param["metric"]}) WITH (m={index_param["m"]}, ef_construction={index_param["ef_construction"]});') + elif self.case_config.index == IndexType.IVFFlat: + log.debug(f'Creating IVFFLAT index. list={index_param["lists"]}') + self.cursor.execute(f'CREATE INDEX IF NOT EXISTS {self._index_name} ON public."{self.table_name}" USING ivfflat (embedding {index_param["metric"]}) WITH (lists={index_param["lists"]});') + else: + assert "Invalid index type {self.case_config.index}" self.conn.commit() def _create_table(self, dim : int): @@ -164,8 +171,15 @@ def search_embedding( assert self.cursor is not None, "Cursor is not initialized" search_param =self.case_config.search_param() - self.cursor.execute(f'SET ivfflat.probes = {search_param["probes"]}') - self.cursor.execute(f"SELECT id FROM public.\"{self.table_name}\" ORDER BY embedding {search_param['metric_fun_op']} '{query}' LIMIT {k};") + + if self.case_config.index == IndexType.HNSW: + self.cursor.execute(f'SET hnsw.ef_search = {search_param["ef"]}') + self.cursor.execute(f"SELECT id FROM public.\"{self.table_name}\" ORDER BY embedding {search_param['metric_fun_op']} '{query}' LIMIT {k};") + elif self.case_config.index == IndexType.IVFFlat: + self.cursor.execute(f'SET ivfflat.probes = {search_param["probes"]}') + self.cursor.execute(f"SELECT id FROM public.\"{self.table_name}\" ORDER BY embedding {search_param['metric_fun_op']} '{query}' LIMIT {k};") + else: + assert "Invalid index type {self.case_config.index}" self.conn.commit() result = self.cursor.fetchall() From 3c340485c0c69f816ba6510fbd7d3bada5041d6c Mon Sep 17 00:00:00 2001 From: Wahaj Ali Date: Wed, 20 Mar 2024 03:10:19 +0500 Subject: [PATCH 054/327] Add index type in results for pgvector --- .../PgVector/result_20230727_standard_pgvector.json | 8 ++++++++ .../PgVector/result_20230808_standard_pgvector.json | 12 +++++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/vectordb_bench/results/PgVector/result_20230727_standard_pgvector.json b/vectordb_bench/results/PgVector/result_20230727_standard_pgvector.json index 4a5b9d5fe..aacecb4c5 100644 --- a/vectordb_bench/results/PgVector/result_20230727_standard_pgvector.json +++ b/vectordb_bench/results/PgVector/result_20230727_standard_pgvector.json @@ -20,6 +20,7 @@ "db_name": "**********" }, "db_case_config": { + "index": "IVF_FLAT", "metric_type": "L2", "lists": 10, "probes": 2 @@ -49,6 +50,7 @@ "db_name": "**********" }, "db_case_config": { + "index": "IVF_FLAT", "metric_type": "L2", "lists": 10, "probes": 2 @@ -78,6 +80,7 @@ "db_name": "**********" }, "db_case_config": { + "index": "IVF_FLAT", "metric_type": "COSINE", "lists": 10, "probes": 2 @@ -107,6 +110,7 @@ "db_name": "**********" }, "db_case_config": { + "index": "IVF_FLAT", "metric_type": "COSINE", "lists": 10, "probes": 2 @@ -136,6 +140,7 @@ "db_name": "**********" }, "db_case_config": { + "index": "IVF_FLAT", "metric_type": "COSINE", "lists": 10, "probes": 2 @@ -165,6 +170,7 @@ "db_name": "**********" }, "db_case_config": { + "index": "IVF_FLAT", "metric_type": "COSINE", "lists": 10, "probes": 2 @@ -194,6 +200,7 @@ "db_name": "**********" }, "db_case_config": { + "index": "IVF_FLAT", "metric_type": "COSINE", "lists": 10, "probes": 2 @@ -223,6 +230,7 @@ "db_name": "**********" }, "db_case_config": { + "index": "IVF_FLAT", "metric_type": "COSINE", "lists": 10, "probes": 2 diff --git a/vectordb_bench/results/PgVector/result_20230808_standard_pgvector.json b/vectordb_bench/results/PgVector/result_20230808_standard_pgvector.json index 5e8a15c98..2172fe5f2 100644 --- a/vectordb_bench/results/PgVector/result_20230808_standard_pgvector.json +++ b/vectordb_bench/results/PgVector/result_20230808_standard_pgvector.json @@ -20,6 +20,7 @@ "db_name": "**********" }, "db_case_config": { + "index": "IVF_FLAT", "metric_type": "L2", "lists": 10, "probes": 2 @@ -51,7 +52,8 @@ "db_case_config": { "metric_type": "L2", "lists": 10, - "probes": 2 + "probes": 2, + "index": "IVF_FLAT" }, "case_config": { "case_id": 11, @@ -80,7 +82,8 @@ "db_case_config": { "metric_type": "L2", "lists": 10, - "probes": 2 + "probes": 2, + "index": "IVF_FLAT" }, "case_config": { "case_id": 12, @@ -107,6 +110,7 @@ "db_name": "**********" }, "db_case_config": { + "index": "IVF_FLAT", "metric_type": "L2", "lists": 10, "probes": 2 @@ -136,6 +140,7 @@ "db_name": "**********" }, "db_case_config": { + "index": "IVF_FLAT", "metric_type": "L2", "lists": 10, "probes": 2 @@ -165,6 +170,7 @@ "db_name": "**********" }, "db_case_config": { + "index": "IVF_FLAT", "metric_type": "L2", "lists": 10, "probes": 2 @@ -178,4 +184,4 @@ } ], "file_fmt": "result_{}_{}_{}.json" -} \ No newline at end of file +} From 2b1fd495d86aa401527b1d68ecc8ac829e735dbe Mon Sep 17 00:00:00 2001 From: Wahaj Ali Date: Wed, 20 Mar 2024 08:37:08 +0500 Subject: [PATCH 055/327] Add defaults for ivfflat --- .../backend/clients/pgvector/config.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/vectordb_bench/backend/clients/pgvector/config.py b/vectordb_bench/backend/clients/pgvector/config.py index 3cf19993c..ec9b1eed0 100644 --- a/vectordb_bench/backend/clients/pgvector/config.py +++ b/vectordb_bench/backend/clients/pgvector/config.py @@ -24,31 +24,29 @@ def to_dict(self) -> dict: class PgVectorIndexConfig(BaseModel, DBCaseConfig): metric_type: MetricType | None = None index: IndexType - lists: int | None = 1000 - probes: int | None = 10 - def parse_metric(self) -> str: + def parse_metric(self) -> str: if self.metric_type == MetricType.L2: return "vector_l2_ops" elif self.metric_type == MetricType.IP: return "vector_ip_ops" return "vector_cosine_ops" - + def parse_metric_fun_op(self) -> str: if self.metric_type == MetricType.L2: return "<->" elif self.metric_type == MetricType.IP: return "<#>" return "<=>" - - def parse_metric_fun_str(self) -> str: + + def parse_metric_fun_str(self) -> str: if self.metric_type == MetricType.L2: return "l2_distance" elif self.metric_type == MetricType.IP: return "max_inner_product" return "cosine_distance" - + class HNSWConfig(PgVectorIndexConfig): M: int @@ -79,8 +77,8 @@ def search_param(self) -> dict: class IVFFlatConfig(PgVectorIndexConfig): - lists: int - probes: int | None = None + lists: int | None = 1000 + probes: int | None = 10 index: IndexType = IndexType.IVFFlat def index_param(self) -> dict: @@ -99,4 +97,4 @@ def search_param(self) -> dict: _pgvector_case_config = { IndexType.HNSW: HNSWConfig, IndexType.IVFFlat: IVFFlatConfig, -} \ No newline at end of file +} From 2df2cfd2684129250551d1cd335b61a6053887f2 Mon Sep 17 00:00:00 2001 From: "min.tian" Date: Mon, 8 Apr 2024 10:59:35 +0000 Subject: [PATCH 056/327] fix qdrant_cloud import bugs Signed-off-by: min.tian --- vectordb_bench/backend/clients/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vectordb_bench/backend/clients/__init__.py b/vectordb_bench/backend/clients/__init__.py index bd5211587..aa6641a93 100644 --- a/vectordb_bench/backend/clients/__init__.py +++ b/vectordb_bench/backend/clients/__init__.py @@ -54,8 +54,8 @@ def init_cls(self) -> Type[VectorDB]: return ElasticCloud if self == DB.QdrantCloud: - from .qdrant_cloud.qdrant_cloud import QdrantClient - return QdrantClient + from .qdrant_cloud.qdrant_cloud import QdrantCloud + return QdrantCloud if self == DB.WeaviateCloud: from .weaviate_cloud.weaviate_cloud import WeaviateCloud From 3686f7a908cc124362ea04597f5d275535224237 Mon Sep 17 00:00:00 2001 From: "min.tian" Date: Tue, 16 Apr 2024 10:01:57 +0000 Subject: [PATCH 057/327] fix the label of zillizcloud test results (Jan 2024) Signed-off-by: min.tian --- ...20240105_standard_202401_zillizcloud.json} | 98 +++++++++---------- vectordb_bench/results/getLeaderboardData.py | 2 +- vectordb_bench/results/leaderboard.json | 2 +- 3 files changed, 51 insertions(+), 51 deletions(-) rename vectordb_bench/results/ZillizCloud/{result_20240105_202401_zillizcloud.json => result_20240105_standard_202401_zillizcloud.json} (93%) diff --git a/vectordb_bench/results/ZillizCloud/result_20240105_202401_zillizcloud.json b/vectordb_bench/results/ZillizCloud/result_20240105_standard_202401_zillizcloud.json similarity index 93% rename from vectordb_bench/results/ZillizCloud/result_20240105_202401_zillizcloud.json rename to vectordb_bench/results/ZillizCloud/result_20240105_standard_202401_zillizcloud.json index afd6068a1..f37ed02a6 100644 --- a/vectordb_bench/results/ZillizCloud/result_20240105_202401_zillizcloud.json +++ b/vectordb_bench/results/ZillizCloud/result_20240105_standard_202401_zillizcloud.json @@ -1,6 +1,6 @@ { "run_id": "0ae10e14e34c4c3c9a7116e7a9591d01", - "task_label": "standard", + "task_label": "standard_202401", "results": [ { "metrics": { @@ -13,7 +13,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "8cu-perf-202401", + "db_label": "8cu-perf-(Jan-2024)", "uri": "**********", "user": "db_admin", "password": "**********" @@ -40,7 +40,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "8cu-perf-202401", + "db_label": "8cu-perf-(Jan-2024)", "uri": "**********", "user": "db_admin", "password": "**********" @@ -67,7 +67,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "8cu-perf-202401", + "db_label": "8cu-perf-(Jan-2024)", "uri": "**********", "user": "db_admin", "password": "**********" @@ -94,7 +94,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "8cu-perf-202401", + "db_label": "8cu-perf-(Jan-2024)", "uri": "**********", "user": "db_admin", "password": "**********" @@ -121,7 +121,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "8cu-perf-202401", + "db_label": "8cu-perf-(Jan-2024)", "uri": "**********", "user": "db_admin", "password": "**********" @@ -148,7 +148,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "8cu-perf-202401", + "db_label": "8cu-perf-(Jan-2024)", "uri": "**********", "user": "db_admin", "password": "**********" @@ -175,7 +175,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "8cu-perf-202401", + "db_label": "8cu-perf-(Jan-2024)", "uri": "**********", "user": "db_admin", "password": "**********" @@ -202,7 +202,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "8cu-perf-202401", + "db_label": "8cu-perf-(Jan-2024)", "uri": "**********", "user": "db_admin", "password": "**********" @@ -229,7 +229,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "8cu-perf-202401", + "db_label": "8cu-perf-(Jan-2024)", "uri": "**********", "user": "db_admin", "password": "**********" @@ -256,7 +256,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "8cu-perf-202401", + "db_label": "8cu-perf-(Jan-2024)", "uri": "**********", "user": "db_admin", "password": "**********" @@ -283,7 +283,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "8cu-perf-202401", + "db_label": "8cu-perf-(Jan-2024)", "uri": "**********", "user": "db_admin", "password": "**********" @@ -310,7 +310,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "8cu-perf-202401", + "db_label": "8cu-perf-(Jan-2024)", "uri": "**********", "user": "db_admin", "password": "**********" @@ -337,7 +337,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-cap-202401", + "db_label": "1cu-cap-(Jan-2024)", "uri": "**********", "user": "db_admin", "password": "**********" @@ -364,7 +364,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-cap-202401", + "db_label": "1cu-cap-(Jan-2024)", "uri": "**********", "user": "db_admin", "password": "**********" @@ -391,7 +391,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-cap-202401", + "db_label": "1cu-cap-(Jan-2024)", "uri": "**********", "user": "db_admin", "password": "**********" @@ -418,7 +418,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-cap-202401", + "db_label": "1cu-cap-(Jan-2024)", "uri": "**********", "user": "db_admin", "password": "**********" @@ -445,7 +445,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-cap-202401", + "db_label": "1cu-cap-(Jan-2024)", "uri": "**********", "user": "db_admin", "password": "**********" @@ -472,7 +472,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-cap-202401", + "db_label": "1cu-cap-(Jan-2024)", "uri": "**********", "user": "db_admin", "password": "**********" @@ -499,7 +499,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "2cu-cap-202401", + "db_label": "2cu-cap-(Jan-2024)", "uri": "**********", "user": "db_admin", "password": "**********" @@ -526,7 +526,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "2cu-cap-202401", + "db_label": "2cu-cap-(Jan-2024)", "uri": "**********", "user": "db_admin", "password": "**********" @@ -553,7 +553,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "2cu-cap-202401", + "db_label": "2cu-cap-(Jan-2024)", "uri": "**********", "user": "db_admin", "password": "**********" @@ -580,7 +580,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "2cu-cap-202401", + "db_label": "2cu-cap-(Jan-2024)", "uri": "**********", "user": "db_admin", "password": "**********" @@ -607,7 +607,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "2cu-cap-202401", + "db_label": "2cu-cap-(Jan-2024)", "uri": "**********", "user": "db_admin", "password": "**********" @@ -634,7 +634,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "2cu-cap-202401", + "db_label": "2cu-cap-(Jan-2024)", "uri": "**********", "user": "db_admin", "password": "**********" @@ -661,7 +661,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "2cu-cap-202401", + "db_label": "2cu-cap-(Jan-2024)", "uri": "**********", "user": "db_admin", "password": "**********" @@ -688,7 +688,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "2cu-cap-202401", + "db_label": "2cu-cap-(Jan-2024)", "uri": "**********", "user": "db_admin", "password": "**********" @@ -715,7 +715,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "2cu-cap-202401", + "db_label": "2cu-cap-(Jan-2024)", "uri": "**********", "user": "db_admin", "password": "**********" @@ -742,7 +742,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "2cu-cap-202401", + "db_label": "2cu-cap-(Jan-2024)", "uri": "**********", "user": "db_admin", "password": "**********" @@ -769,7 +769,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "2cu-cap-202401", + "db_label": "2cu-cap-(Jan-2024)", "uri": "**********", "user": "db_admin", "password": "**********" @@ -796,7 +796,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "2cu-cap-202401", + "db_label": "2cu-cap-(Jan-2024)", "uri": "**********", "user": "db_admin", "password": "**********" @@ -823,7 +823,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-perf-202401", + "db_label": "1cu-perf-(Jan-2024)", "uri": "**********", "user": "db_admin", "password": "**********" @@ -850,7 +850,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-perf-202401", + "db_label": "1cu-perf-(Jan-2024)", "uri": "**********", "user": "db_admin", "password": "**********" @@ -877,7 +877,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-perf-202401", + "db_label": "1cu-perf-(Jan-2024)", "uri": "**********", "user": "db_admin", "password": "**********" @@ -904,7 +904,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-perf-202401", + "db_label": "1cu-perf-(Jan-2024)", "uri": "**********", "user": "db_admin", "password": "**********" @@ -931,7 +931,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-perf-202401", + "db_label": "1cu-perf-(Jan-2024)", "uri": "**********", "user": "db_admin", "password": "**********" @@ -958,7 +958,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-perf-202401", + "db_label": "1cu-perf-(Jan-2024)", "uri": "**********", "user": "db_admin", "password": "**********" @@ -985,7 +985,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-perf-202401", + "db_label": "1cu-perf-(Jan-2024)", "uri": "**********", "user": "db_admin", "password": "**********" @@ -1012,7 +1012,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-perf-202401", + "db_label": "1cu-perf-(Jan-2024)", "uri": "**********", "user": "db_admin", "password": "**********" @@ -1039,7 +1039,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-perf-202401", + "db_label": "1cu-perf-(Jan-2024)", "uri": "**********", "user": "db_admin", "password": "**********" @@ -1066,7 +1066,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-perf-202401", + "db_label": "1cu-perf-(Jan-2024)", "uri": "**********", "user": "db_admin", "password": "**********" @@ -1093,7 +1093,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-perf-202401", + "db_label": "1cu-perf-(Jan-2024)", "uri": "**********", "user": "db_admin", "password": "**********" @@ -1120,7 +1120,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-perf-202401", + "db_label": "1cu-perf-(Jan-2024)", "uri": "**********", "user": "db_admin", "password": "**********" @@ -1147,7 +1147,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-cap-202401", + "db_label": "1cu-cap-(Jan-2024)", "uri": "**********", "user": "db_admin", "password": "**********" @@ -1174,7 +1174,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-cap-202401", + "db_label": "1cu-cap-(Jan-2024)", "uri": "**********", "user": "db_admin", "password": "**********" @@ -1201,7 +1201,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-cap-202401", + "db_label": "1cu-cap-(Jan-2024)", "uri": "**********", "user": "db_admin", "password": "**********" @@ -1228,7 +1228,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-cap-202401", + "db_label": "1cu-cap-(Jan-2024)", "uri": "**********", "user": "db_admin", "password": "**********" @@ -1255,7 +1255,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-cap-202401", + "db_label": "1cu-cap-(Jan-2024)", "uri": "**********", "user": "db_admin", "password": "**********" @@ -1282,7 +1282,7 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-cap-202401", + "db_label": "1cu-cap-(Jan-2024)", "uri": "**********", "user": "db_admin", "password": "**********" diff --git a/vectordb_bench/results/getLeaderboardData.py b/vectordb_bench/results/getLeaderboardData.py index 9eac371a7..50f458533 100644 --- a/vectordb_bench/results/getLeaderboardData.py +++ b/vectordb_bench/results/getLeaderboardData.py @@ -17,7 +17,7 @@ def main(): allResults: list[TestResult] = benchMarkRunner.get_results() results: list[CaseResult] = [] for result in allResults: - if result.task_label == "standard": + if "standard" in result.task_label: results += result.results if allResults is not None: diff --git a/vectordb_bench/results/leaderboard.json b/vectordb_bench/results/leaderboard.json index 5d8705571..261a82c94 100644 --- a/vectordb_bench/results/leaderboard.json +++ b/vectordb_bench/results/leaderboard.json @@ -1 +1 @@ -[{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":0.8836,"latency":2523.0,"recall":0.8528,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":0.8937,"latency":3720.2000000000003,"recall":0.8525,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":1.2145,"latency":3622.3999999999996,"recall":0.7487,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":10.6271,"latency":730.7,"recall":0.8898,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":10.8507,"latency":733.1999999999999,"recall":0.8897,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":75.7055,"latency":121.2,"recall":0.9999,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":11.2945,"latency":3611.2000000000003,"recall":0.996,"label":1,"qp$":84832.4640100146},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":17.3271,"latency":3774.7999999999997,"recall":0.9961,"label":1,"qp$":130143.04193615691},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":26.26,"latency":556.1,"recall":0.9999,"label":1,"qp$":197237.6382224077},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":15.2269,"latency":861.8,"recall":0.9888,"label":1,"qp$":114368.53745044857},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":15.1749,"latency":774.3,"recall":0.989,"label":1,"qp$":113977.96786981013},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":27.6181,"latency":305.5,"recall":0.9999,"label":1,"qp$":207438.26413519715},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":228.4,"latency":22.2,"recall":0.9348,"label":1,"qp$":5631780.821917809},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":181.5,"latency":26.1,"recall":0.9345,"label":1,"qp$":4475342.465753425},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":205.7,"latency":24.2,"recall":0.9586,"label":1,"qp$":5072054.794520548},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":67.63,"latency":36.0,"recall":0.8064,"label":1,"qp$":2502240.4933196297},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":63.35,"latency":38.4,"recall":0.8065,"label":1,"qp$":2343884.892086331},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":176.7,"latency":27.599999999999998,"recall":1.0,"label":1,"qp$":6537718.396711202},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":15.33,"latency":84.9,"recall":0.8064,"label":1,"qp$":567194.2446043165},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":15.13,"latency":86.7,"recall":0.8065,"label":1,"qp$":559794.4501541625},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":17.41,"latency":74.30000000000001,"recall":1.0,"label":1,"qp$":644152.1068859198},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":16.34,"latency":88.7,"recall":0.879,"label":1,"qp$":301661.53846153844},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":10.45,"latency":126.8,"recall":0.8208,"label":1,"qp$":192923.0769230769},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":16.18,"latency":87.5,"recall":0.8793,"label":1,"qp$":298707.69230769225},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":9.8,"latency":130.89999999999998,"recall":0.8212,"label":1,"qp$":180923.0769230769},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":36.11,"latency":55.1,"recall":1.0,"label":1,"qp$":666646.1538461539},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":14.84,"latency":92.7,"recall":0.96,"label":1,"qp$":273969.23076923075},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":322.7,"latency":26.4,"recall":0.9478,"label":1,"qp$":994623.287671233},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":265.5,"latency":26.9,"recall":0.9332,"label":1,"qp$":818321.9178082192},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":303.8,"latency":27.3,"recall":0.9478,"label":1,"qp$":936369.8630136987},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":180.2,"latency":28.2,"recall":0.9335,"label":1,"qp$":555410.9589041097},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":730.7,"latency":24.6,"recall":0.9586,"label":1,"qp$":2252157.5342465756},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":104.3,"latency":31.7,"recall":0.9563,"label":1,"qp$":321472.602739726},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":147.7,"latency":35.3,"recall":0.9707,"label":1,"qp$":682567.3940949935},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":782.5,"latency":25.9,"recall":1.0,"label":1,"qp$":3616174.5827984596},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":18.7634,"latency":153.70000000000002,"recall":0.8737,"label":1,"qp$":694226.515930113},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":18.3619,"latency":79.8,"recall":0.8741,"label":1,"qp$":679371.4285714285},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":25.2744,"latency":61.199999999999996,"recall":0.9979,"label":1,"qp$":935126.8242548818},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":261.798,"latency":23.099999999999998,"recall":0.9262,"label":1,"qp$":6455293.150684931},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":166.1851,"latency":23.900000000000002,"recall":0.9264,"label":1,"qp$":4097714.7945205485},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":121.7169,"latency":29.0,"recall":0.9693,"label":1,"qp$":3001238.6301369863},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":46.6189,"latency":43.1,"recall":0.8737,"label":1,"qp$":1724851.3874614593},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":42.4856,"latency":44.0,"recall":0.8741,"label":1,"qp$":1571923.5354573485},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":138.9479,"latency":26.200000000000003,"recall":0.9979,"label":1,"qp$":5140929.496402878},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":20.7437,"latency":75.80000000000001,"recall":0.9291,"label":1,"qp$":382960.6153846154},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":20.2993,"latency":76.5,"recall":0.9293,"label":1,"qp$":374756.30769230763},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":26.4719,"latency":67.0,"recall":1.0,"label":1,"qp$":488712.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":8.6675,"latency":180.2,"recall":0.8369,"label":1,"qp$":160015.38461538462},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":7.8121,"latency":167.7,"recall":0.8369,"label":1,"qp$":144223.3846153846},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":16.869,"latency":87.8,"recall":0.9814,"label":1,"qp$":311427.6923076923},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":365.0835,"latency":23.599999999999998,"recall":0.945,"label":1,"qp$":1125257.3630136987},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":325.5271,"latency":25.1,"recall":0.9452,"label":1,"qp$":1003336.9520547946},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":596.7942,"latency":24.2,"recall":0.9693,"label":1,"qp$":1839434.1780821919},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":303.2044,"latency":27.400000000000002,"recall":0.9246,"label":1,"qp$":934534.1095890411},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":136.0345,"latency":31.9,"recall":0.9244,"label":1,"qp$":419284.4178082192},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":66.7221,"latency":42.1,"recall":0.963,"label":1,"qp$":205650.30821917808},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":131.2549,"latency":30.200000000000003,"recall":0.9867,"label":1,"qp$":606569.4993581515},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":127.9337,"latency":30.099999999999998,"recall":0.9869,"label":1,"qp$":591221.2066752247},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":595.8462,"latency":23.400000000000002,"recall":1.0,"label":1,"qp$":2753589.627727856},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":330.0144,"latency":9.0,"recall":0.9507,"label":1,"qp$":7472024.150943397},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":271.6585,"latency":10.1,"recall":0.9678,"label":1,"qp$":6150758.490566038},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":216.5226,"latency":12.9,"recall":1.0,"label":1,"qp$":4902398.490566038},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":123.9553,"latency":23.0,"recall":0.971,"label":1,"qp$":1403267.5471698113},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":59.1479,"latency":44.5,"recall":0.9906,"label":1,"qp$":669598.8679245282},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":40.999,"latency":55.300000000000004,"recall":1.0,"label":1,"qp$":464139.62264150946},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":579.9416,"latency":9.4,"recall":0.9213,"label":1,"qp$":6565376.603773585},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":425.2529,"latency":11.299999999999999,"recall":0.9686,"label":1,"qp$":4814183.773584905},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":397.0539,"latency":13.799999999999999,"recall":1.0,"label":1,"qp$":4494949.811320755},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":516.27,"latency":7.0,"recall":0.9463,"label":1,"qp$":11689132.075471697},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":354.8416,"latency":10.0,"recall":0.9802,"label":1,"qp$":8034149.433962264},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":427.5229,"latency":8.7,"recall":1.0,"label":1,"qp$":9679763.773584906},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":2884.689,"latency":5.3,"recall":0.8801,"label":1,"qp$":8164214.150943396},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":1689.5799,"latency":6.6,"recall":0.9493,"label":1,"qp$":4781829.905660377},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1517.6792,"latency":10.0,"recall":1.0,"label":1,"qp$":4295318.4905660385},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":822.5318,"latency":5.6,"recall":0.9294,"label":1,"qp$":2327920.1886792453},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":378.9146,"latency":10.3,"recall":0.9758,"label":1,"qp$":1072399.8113207547},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":218.6854,"latency":16.2,"recall":1.0,"label":1,"qp$":618920.9433962263},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":297.5,"latency":7.2,"recall":0.974,"label":1,"qp$":6735849.0566037735},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":228.3,"latency":10.6,"recall":0.994,"label":1,"qp$":5169056.603773585},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":584.0,"latency":4.6,"recall":1.0,"label":1,"qp$":13222641.509433962},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":1871.0,"latency":7.0,"recall":0.9602,"label":1,"qp$":5295283.018867925},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":556.7,"latency":6.7,"recall":0.9723,"label":1,"qp$":1575566.0377358492},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1583.0,"latency":6.8,"recall":0.9836,"label":1,"qp$":4480188.679245283},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":294.3,"latency":10.9,"recall":0.9939,"label":1,"qp$":832924.5283018869},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":2345.0,"latency":8.9,"recall":1.0,"label":1,"qp$":6636792.452830189},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":295.6,"latency":12.3,"recall":1.0,"label":1,"qp$":836603.7735849057},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":143.0,"latency":33.5,"recall":0.9818,"label":1,"qp$":3237735.8490566034},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":106.0,"latency":20.7,"recall":0.9887,"label":1,"qp$":2400000.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":189.0,"latency":11.6,"recall":1.0,"label":1,"qp$":4279245.2830188675},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":379.9721,"latency":12.4,"recall":0.982,"label":1,"qp$":4301570.9433962265},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":71.74,"latency":50.8,"recall":0.9883,"label":1,"qp$":812150.9433962264},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":287.0,"latency":14.9,"recall":0.9865,"label":1,"qp$":3249056.603773585},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":34.6654,"latency":64.69999999999999,"recall":0.9961,"label":1,"qp$":392438.4905660377},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":412.0,"latency":10.3,"recall":1.0,"label":1,"qp$":4664150.9433962265},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":42.169,"latency":46.800000000000004,"recall":1.0,"label":1,"qp$":477384.9056603773},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":537.4975,"latency":18.9,"recall":0.8903,"label":1,"qp$":1356936.1851332397},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":372.0466,"latency":17.8,"recall":0.8904,"label":1,"qp$":939248.0785413746},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1156.2898,"latency":14.4,"recall":0.9989,"label":1,"qp$":2919104.684431978},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":110.248,"latency":69.0,"recall":0.898,"label":1,"qp$":278325.94670406735},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":87.2601,"latency":27.799999999999997,"recall":0.898,"label":1,"qp$":220291.97755960727},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":125.7846,"latency":23.099999999999998,"recall":0.975,"label":1,"qp$":317548.77980364655},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":240.7209,"latency":17.4,"recall":0.8887,"label":1,"qp$":3038552.734922861},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":189.4399,"latency":17.5,"recall":0.8889,"label":1,"qp$":2391246.98457223},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":313.5116,"latency":16.1,"recall":0.9999,"label":1,"qp$":3957369.424964937},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":78.7196,"latency":49.4,"recall":0.9203,"label":1,"qp$":1707172.048192771},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":68.3111,"latency":35.5,"recall":0.9202,"label":1,"qp$":1481445.5421686745},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":210.2147,"latency":26.8,"recall":0.9996,"label":1,"qp$":4558873.012048192},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":188.6436,"latency":917.5,"recall":0.9175,"label":1,"qp$":2381195.5119214584},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":155.6991,"latency":917.1,"recall":0.9171,"label":1,"qp$":1965346.283309958},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":445.3289,"latency":14.1,"recall":0.9999,"label":1,"qp$":5621262.412342216},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":633.6033,"latency":24.6,"recall":0.919,"label":1,"qp$":1599559.5231416551},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":95.5682,"latency":58.7,"recall":0.9463,"label":1,"qp$":241266.14305750353},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":434.4062,"latency":17.4,"recall":0.9181,"label":1,"qp$":1096677.643758766},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":80.3192,"latency":26.599999999999998,"recall":0.9462,"label":1,"qp$":202769.3688639551},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":1509.3293,"latency":18.5,"recall":0.9995,"label":1,"qp$":3810368.4992987383},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":166.7252,"latency":19.599999999999998,"recall":0.9988,"label":1,"qp$":420905.1332398317},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":46.8622,"latency":123.0,"recall":0.9957,"label":1,"qp$":16703.358415841583},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1.8388,"latency":1063.5,"recall":0.9957,"label":1,"qp$":655.4138613861386},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":45.0666,"latency":109.2,"recall":1.0,"label":1,"qp$":16063.342574257427},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":43.5017,"latency":228.7,"recall":0.9957,"label":1,"qp$":4803.8687116564415},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1.5668,"latency":1114.4,"recall":0.9957,"label":1,"qp$":173.02085889570552},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":41.5443,"latency":159.0,"recall":1.0,"label":1,"qp$":4587.714110429448},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":67.9121,"latency":179.5,"recall":0.9909,"label":1,"qp$":7499.495705521471},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.7636,"latency":1921.3,"recall":0.9908,"label":1,"qp$":84.32392638036809},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":32.0,"latency":124.5,"recall":1.0,"label":1,"qp$":3533.7423312883434},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":63.1365,"latency":145.7,"recall":0.991,"label":1,"qp$":22504.09900990099},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.7512,"latency":1983.8,"recall":0.9908,"label":1,"qp$":267.7544554455445},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":30.1358,"latency":129.8,"recall":1.0,"label":1,"qp$":10741.473267326734},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":37.432,"latency":75.0,"recall":0.9975,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":37.0696,"latency":73.5,"recall":0.9976,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":81.1915,"latency":53.0,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":321.6048,"latency":13.4,"recall":0.989,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":22.1467,"latency":86.8,"recall":0.9972,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":303.2551,"latency":10.9,"recall":0.9876,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":21.5388,"latency":81.69999999999999,"recall":0.997,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":394.5418,"latency":8.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":37.878,"latency":45.0,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":180.2757,"latency":6.0,"recall":0.9942,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":179.0033,"latency":6.4,"recall":0.9943,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":526.8846,"latency":3.6,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":626.5243,"latency":6.2,"recall":0.9954,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":78.4227,"latency":25.3,"recall":0.9982,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":599.4213,"latency":6.6,"recall":0.9955,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":78.5351,"latency":26.3,"recall":0.9982,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":2098.2113,"latency":3.4,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":275.6292,"latency":10.0,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":100.6667,"latency":21.1,"recall":0.9909,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":101.1399,"latency":19.7,"recall":0.9907,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":52.2606,"latency":18.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":61.0661,"latency":49.8,"recall":0.9911,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":58.9326,"latency":44.6,"recall":0.9911,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":42.5977,"latency":54.9,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":536.0726,"latency":8.200000000000001,"recall":0.9728,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":467.179,"latency":7.0,"recall":0.9697,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":431.7512,"latency":8.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":274.5407,"latency":4.8999999999999995,"recall":0.9807,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":236.5672,"latency":10.3,"recall":0.981,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":309.4833,"latency":4.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":178.6585,"latency":13.700000000000001,"recall":0.9843,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":178.3732,"latency":15.0,"recall":0.9844,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":229.3526,"latency":12.5,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":1258.7043,"latency":4.8999999999999995,"recall":0.9799,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":1075.8776,"latency":5.3,"recall":0.98,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1494.8493,"latency":4.7,"recall":1.0,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-202401","db_name":"ZillizCloud-8cu-perf-202401","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":5115.5303,"latency":8.7,"recall":0.9469,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-202401","db_name":"ZillizCloud-8cu-perf-202401","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":3685.0767,"latency":9.1,"recall":0.9736,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-202401","db_name":"ZillizCloud-8cu-perf-202401","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":4742.1617,"latency":9.1,"recall":0.9936,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-202401","db_name":"ZillizCloud-8cu-perf-202401","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":6054.4428,"latency":8.1,"recall":0.9155,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-202401","db_name":"ZillizCloud-8cu-perf-202401","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":4104.2598,"latency":10.6,"recall":0.9506,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-202401","db_name":"ZillizCloud-8cu-perf-202401","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":4252.1267,"latency":9.2,"recall":0.9964,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-202401","db_name":"ZillizCloud-8cu-perf-202401","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":1685.3091,"latency":13.299999999999999,"recall":0.9718,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-202401","db_name":"ZillizCloud-8cu-perf-202401","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":769.8991,"latency":10.7,"recall":0.9884,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-202401","db_name":"ZillizCloud-8cu-perf-202401","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":945.4061,"latency":10.9,"recall":0.9941,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-202401","db_name":"ZillizCloud-8cu-perf-202401","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":2214.9028,"latency":8.4,"recall":0.9249,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-202401","db_name":"ZillizCloud-8cu-perf-202401","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":827.975,"latency":12.0,"recall":0.9692,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-202401","db_name":"ZillizCloud-8cu-perf-202401","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":776.9454,"latency":11.4,"recall":0.9966,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-202401","db_name":"ZillizCloud-1cu-cap-202401","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":269.5464,"latency":9.799999999999999,"recall":0.9776,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-202401","db_name":"ZillizCloud-1cu-cap-202401","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":240.0363,"latency":10.6,"recall":0.9822,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-202401","db_name":"ZillizCloud-1cu-cap-202401","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":218.0627,"latency":11.4,"recall":0.9936,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-202401","db_name":"ZillizCloud-1cu-cap-202401","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":392.8825,"latency":6.8999999999999995,"recall":0.9581,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-202401","db_name":"ZillizCloud-1cu-cap-202401","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":343.8204,"latency":8.4,"recall":0.968,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-202401","db_name":"ZillizCloud-1cu-cap-202401","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":216.6773,"latency":10.1,"recall":0.9968,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-202401","db_name":"ZillizCloud-2cu-cap-202401","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":503.2284,"latency":9.0,"recall":0.9677,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-202401","db_name":"ZillizCloud-2cu-cap-202401","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":413.3232,"latency":9.6,"recall":0.981,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-202401","db_name":"ZillizCloud-2cu-cap-202401","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":425.5492,"latency":10.200000000000001,"recall":0.9936,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-202401","db_name":"ZillizCloud-2cu-cap-202401","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":789.1229,"latency":5.6,"recall":0.9396,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-202401","db_name":"ZillizCloud-2cu-cap-202401","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":571.4257,"latency":7.7,"recall":0.9668,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-202401","db_name":"ZillizCloud-2cu-cap-202401","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":411.7653,"latency":9.1,"recall":0.9968,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-202401","db_name":"ZillizCloud-2cu-cap-202401","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":98.0448,"latency":16.1,"recall":0.9803,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-202401","db_name":"ZillizCloud-2cu-cap-202401","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":58.3152,"latency":28.0,"recall":0.9891,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-202401","db_name":"ZillizCloud-2cu-cap-202401","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":46.8304,"latency":28.6,"recall":0.9941,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-202401","db_name":"ZillizCloud-2cu-cap-202401","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":170.5693,"latency":8.9,"recall":0.9605,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-202401","db_name":"ZillizCloud-2cu-cap-202401","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":94.7766,"latency":19.599999999999998,"recall":0.9843,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-202401","db_name":"ZillizCloud-2cu-cap-202401","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":44.8695,"latency":30.900000000000002,"recall":0.9966,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-202401","db_name":"ZillizCloud-1cu-perf-202401","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":722.0315,"latency":7.7,"recall":0.976,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-202401","db_name":"ZillizCloud-1cu-perf-202401","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":467.5795,"latency":8.8,"recall":0.9898,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-202401","db_name":"ZillizCloud-1cu-perf-202401","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":975.2503,"latency":8.200000000000001,"recall":0.9936,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-202401","db_name":"ZillizCloud-1cu-perf-202401","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":873.3712,"latency":6.7,"recall":0.9477,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-202401","db_name":"ZillizCloud-1cu-perf-202401","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":544.6203,"latency":8.4,"recall":0.977,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-202401","db_name":"ZillizCloud-1cu-perf-202401","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":930.9164,"latency":9.6,"recall":0.9968,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-202401","db_name":"ZillizCloud-1cu-perf-202401","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-202401","db_name":"ZillizCloud-1cu-perf-202401","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-202401","db_name":"ZillizCloud-1cu-perf-202401","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-202401","db_name":"ZillizCloud-1cu-perf-202401","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-202401","db_name":"ZillizCloud-1cu-perf-202401","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-202401","db_name":"ZillizCloud-1cu-perf-202401","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-202401","db_name":"ZillizCloud-1cu-cap-202401","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-202401","db_name":"ZillizCloud-1cu-cap-202401","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-202401","db_name":"ZillizCloud-1cu-cap-202401","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-202401","db_name":"ZillizCloud-1cu-cap-202401","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-202401","db_name":"ZillizCloud-1cu-cap-202401","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-202401","db_name":"ZillizCloud-1cu-cap-202401","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0}] \ No newline at end of file +[{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":15.2269,"latency":861.8,"recall":0.9888,"label":1,"qp$":114368.53745044857},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":15.1749,"latency":774.3,"recall":0.989,"label":1,"qp$":113977.96786981013},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":27.6181,"latency":305.5,"recall":0.9999,"label":1,"qp$":207438.26413519715},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":11.2945,"latency":3611.2000000000003,"recall":0.996,"label":1,"qp$":84832.4640100146},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":17.3271,"latency":3774.7999999999997,"recall":0.9961,"label":1,"qp$":130143.04193615691},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":26.26,"latency":556.1,"recall":0.9999,"label":1,"qp$":197237.6382224077},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":100.6667,"latency":21.1,"recall":0.9909,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":101.1399,"latency":19.7,"recall":0.9907,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":52.2606,"latency":18.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":61.0661,"latency":49.8,"recall":0.9911,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":58.9326,"latency":44.6,"recall":0.9911,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":42.5977,"latency":54.9,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":536.0726,"latency":8.200000000000001,"recall":0.9728,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":467.179,"latency":7.0,"recall":0.9697,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":431.7512,"latency":8.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":274.5407,"latency":4.8999999999999995,"recall":0.9807,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":236.5672,"latency":10.3,"recall":0.981,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":309.4833,"latency":4.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":178.6585,"latency":13.700000000000001,"recall":0.9843,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":178.3732,"latency":15.0,"recall":0.9844,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":229.3526,"latency":12.5,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":1258.7043,"latency":4.8999999999999995,"recall":0.9799,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":1075.8776,"latency":5.3,"recall":0.98,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1494.8493,"latency":4.7,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":37.432,"latency":75.0,"recall":0.9975,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":37.0696,"latency":73.5,"recall":0.9976,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":81.1915,"latency":53.0,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":321.6048,"latency":13.4,"recall":0.989,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":22.1467,"latency":86.8,"recall":0.9972,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":303.2551,"latency":10.9,"recall":0.9876,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":21.5388,"latency":81.69999999999999,"recall":0.997,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":394.5418,"latency":8.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":37.878,"latency":45.0,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":180.2757,"latency":6.0,"recall":0.9942,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":179.0033,"latency":6.4,"recall":0.9943,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":526.8846,"latency":3.6,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":626.5243,"latency":6.2,"recall":0.9954,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":78.4227,"latency":25.3,"recall":0.9982,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":599.4213,"latency":6.6,"recall":0.9955,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":78.5351,"latency":26.3,"recall":0.9982,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":2098.2113,"latency":3.4,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":275.6292,"latency":10.0,"recall":1.0,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":10.6271,"latency":730.7,"recall":0.8898,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":10.8507,"latency":733.1999999999999,"recall":0.8897,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":75.7055,"latency":121.2,"recall":0.9999,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":0.8836,"latency":2523.0,"recall":0.8528,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":0.8937,"latency":3720.2000000000003,"recall":0.8525,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":1.2145,"latency":3622.3999999999996,"recall":0.7487,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":18.7634,"latency":153.70000000000002,"recall":0.8737,"label":1,"qp$":694226.515930113},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":18.3619,"latency":79.8,"recall":0.8741,"label":1,"qp$":679371.4285714285},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":25.2744,"latency":61.199999999999996,"recall":0.9979,"label":1,"qp$":935126.8242548818},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":261.798,"latency":23.099999999999998,"recall":0.9262,"label":1,"qp$":6455293.150684931},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":166.1851,"latency":23.900000000000002,"recall":0.9264,"label":1,"qp$":4097714.7945205485},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":121.7169,"latency":29.0,"recall":0.9693,"label":1,"qp$":3001238.6301369863},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":46.6189,"latency":43.1,"recall":0.8737,"label":1,"qp$":1724851.3874614593},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":42.4856,"latency":44.0,"recall":0.8741,"label":1,"qp$":1571923.5354573485},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":138.9479,"latency":26.200000000000003,"recall":0.9979,"label":1,"qp$":5140929.496402878},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":20.7437,"latency":75.80000000000001,"recall":0.9291,"label":1,"qp$":382960.6153846154},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":20.2993,"latency":76.5,"recall":0.9293,"label":1,"qp$":374756.30769230763},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":26.4719,"latency":67.0,"recall":1.0,"label":1,"qp$":488712.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":8.6675,"latency":180.2,"recall":0.8369,"label":1,"qp$":160015.38461538462},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":7.8121,"latency":167.7,"recall":0.8369,"label":1,"qp$":144223.3846153846},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":16.869,"latency":87.8,"recall":0.9814,"label":1,"qp$":311427.6923076923},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":365.0835,"latency":23.599999999999998,"recall":0.945,"label":1,"qp$":1125257.3630136987},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":325.5271,"latency":25.1,"recall":0.9452,"label":1,"qp$":1003336.9520547946},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":596.7942,"latency":24.2,"recall":0.9693,"label":1,"qp$":1839434.1780821919},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":303.2044,"latency":27.400000000000002,"recall":0.9246,"label":1,"qp$":934534.1095890411},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":136.0345,"latency":31.9,"recall":0.9244,"label":1,"qp$":419284.4178082192},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":66.7221,"latency":42.1,"recall":0.963,"label":1,"qp$":205650.30821917808},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":131.2549,"latency":30.200000000000003,"recall":0.9867,"label":1,"qp$":606569.4993581515},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":127.9337,"latency":30.099999999999998,"recall":0.9869,"label":1,"qp$":591221.2066752247},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":595.8462,"latency":23.400000000000002,"recall":1.0,"label":1,"qp$":2753589.627727856},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":228.4,"latency":22.2,"recall":0.9348,"label":1,"qp$":5631780.821917809},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":181.5,"latency":26.1,"recall":0.9345,"label":1,"qp$":4475342.465753425},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":205.7,"latency":24.2,"recall":0.9586,"label":1,"qp$":5072054.794520548},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":67.63,"latency":36.0,"recall":0.8064,"label":1,"qp$":2502240.4933196297},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":63.35,"latency":38.4,"recall":0.8065,"label":1,"qp$":2343884.892086331},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":176.7,"latency":27.599999999999998,"recall":1.0,"label":1,"qp$":6537718.396711202},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":15.33,"latency":84.9,"recall":0.8064,"label":1,"qp$":567194.2446043165},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":15.13,"latency":86.7,"recall":0.8065,"label":1,"qp$":559794.4501541625},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":17.41,"latency":74.30000000000001,"recall":1.0,"label":1,"qp$":644152.1068859198},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":16.34,"latency":88.7,"recall":0.879,"label":1,"qp$":301661.53846153844},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":10.45,"latency":126.8,"recall":0.8208,"label":1,"qp$":192923.0769230769},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":16.18,"latency":87.5,"recall":0.8793,"label":1,"qp$":298707.69230769225},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":9.8,"latency":130.89999999999998,"recall":0.8212,"label":1,"qp$":180923.0769230769},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":36.11,"latency":55.1,"recall":1.0,"label":1,"qp$":666646.1538461539},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":14.84,"latency":92.7,"recall":0.96,"label":1,"qp$":273969.23076923075},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":322.7,"latency":26.4,"recall":0.9478,"label":1,"qp$":994623.287671233},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":265.5,"latency":26.9,"recall":0.9332,"label":1,"qp$":818321.9178082192},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":303.8,"latency":27.3,"recall":0.9478,"label":1,"qp$":936369.8630136987},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":180.2,"latency":28.2,"recall":0.9335,"label":1,"qp$":555410.9589041097},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":730.7,"latency":24.6,"recall":0.9586,"label":1,"qp$":2252157.5342465756},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":104.3,"latency":31.7,"recall":0.9563,"label":1,"qp$":321472.602739726},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":147.7,"latency":35.3,"recall":0.9707,"label":1,"qp$":682567.3940949935},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":782.5,"latency":25.9,"recall":1.0,"label":1,"qp$":3616174.5827984596},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":537.4975,"latency":18.9,"recall":0.8903,"label":1,"qp$":1356936.1851332397},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":372.0466,"latency":17.8,"recall":0.8904,"label":1,"qp$":939248.0785413746},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1156.2898,"latency":14.4,"recall":0.9989,"label":1,"qp$":2919104.684431978},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":110.248,"latency":69.0,"recall":0.898,"label":1,"qp$":278325.94670406735},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":87.2601,"latency":27.799999999999997,"recall":0.898,"label":1,"qp$":220291.97755960727},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":125.7846,"latency":23.099999999999998,"recall":0.975,"label":1,"qp$":317548.77980364655},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":240.7209,"latency":17.4,"recall":0.8887,"label":1,"qp$":3038552.734922861},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":189.4399,"latency":17.5,"recall":0.8889,"label":1,"qp$":2391246.98457223},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":313.5116,"latency":16.1,"recall":0.9999,"label":1,"qp$":3957369.424964937},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":78.7196,"latency":49.4,"recall":0.9203,"label":1,"qp$":1707172.048192771},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":68.3111,"latency":35.5,"recall":0.9202,"label":1,"qp$":1481445.5421686745},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":210.2147,"latency":26.8,"recall":0.9996,"label":1,"qp$":4558873.012048192},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":188.6436,"latency":917.5,"recall":0.9175,"label":1,"qp$":2381195.5119214584},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":155.6991,"latency":917.1,"recall":0.9171,"label":1,"qp$":1965346.283309958},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":445.3289,"latency":14.1,"recall":0.9999,"label":1,"qp$":5621262.412342216},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":633.6033,"latency":24.6,"recall":0.919,"label":1,"qp$":1599559.5231416551},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":95.5682,"latency":58.7,"recall":0.9463,"label":1,"qp$":241266.14305750353},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":434.4062,"latency":17.4,"recall":0.9181,"label":1,"qp$":1096677.643758766},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":80.3192,"latency":26.599999999999998,"recall":0.9462,"label":1,"qp$":202769.3688639551},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":1509.3293,"latency":18.5,"recall":0.9995,"label":1,"qp$":3810368.4992987383},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":166.7252,"latency":19.599999999999998,"recall":0.9988,"label":1,"qp$":420905.1332398317},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":67.9121,"latency":179.5,"recall":0.9909,"label":1,"qp$":7499.495705521471},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.7636,"latency":1921.3,"recall":0.9908,"label":1,"qp$":84.32392638036809},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":32.0,"latency":124.5,"recall":1.0,"label":1,"qp$":3533.7423312883434},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":63.1365,"latency":145.7,"recall":0.991,"label":1,"qp$":22504.09900990099},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.7512,"latency":1983.8,"recall":0.9908,"label":1,"qp$":267.7544554455445},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":30.1358,"latency":129.8,"recall":1.0,"label":1,"qp$":10741.473267326734},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":46.8622,"latency":123.0,"recall":0.9957,"label":1,"qp$":16703.358415841583},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1.8388,"latency":1063.5,"recall":0.9957,"label":1,"qp$":655.4138613861386},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":45.0666,"latency":109.2,"recall":1.0,"label":1,"qp$":16063.342574257427},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":43.5017,"latency":228.7,"recall":0.9957,"label":1,"qp$":4803.8687116564415},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1.5668,"latency":1114.4,"recall":0.9957,"label":1,"qp$":173.02085889570552},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":41.5443,"latency":159.0,"recall":1.0,"label":1,"qp$":4587.714110429448},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":297.5,"latency":7.2,"recall":0.974,"label":1,"qp$":6735849.0566037735},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":228.3,"latency":10.6,"recall":0.994,"label":1,"qp$":5169056.603773585},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":584.0,"latency":4.6,"recall":1.0,"label":1,"qp$":13222641.509433962},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":1871.0,"latency":7.0,"recall":0.9602,"label":1,"qp$":5295283.018867925},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":556.7,"latency":6.7,"recall":0.9723,"label":1,"qp$":1575566.0377358492},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1583.0,"latency":6.8,"recall":0.9836,"label":1,"qp$":4480188.679245283},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":294.3,"latency":10.9,"recall":0.9939,"label":1,"qp$":832924.5283018869},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":2345.0,"latency":8.9,"recall":1.0,"label":1,"qp$":6636792.452830189},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":295.6,"latency":12.3,"recall":1.0,"label":1,"qp$":836603.7735849057},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":143.0,"latency":33.5,"recall":0.9818,"label":1,"qp$":3237735.8490566034},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":106.0,"latency":20.7,"recall":0.9887,"label":1,"qp$":2400000.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":189.0,"latency":11.6,"recall":1.0,"label":1,"qp$":4279245.2830188675},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":379.9721,"latency":12.4,"recall":0.982,"label":1,"qp$":4301570.9433962265},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":71.74,"latency":50.8,"recall":0.9883,"label":1,"qp$":812150.9433962264},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":287.0,"latency":14.9,"recall":0.9865,"label":1,"qp$":3249056.603773585},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":34.6654,"latency":64.69999999999999,"recall":0.9961,"label":1,"qp$":392438.4905660377},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":412.0,"latency":10.3,"recall":1.0,"label":1,"qp$":4664150.9433962265},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":42.169,"latency":46.800000000000004,"recall":1.0,"label":1,"qp$":477384.9056603773},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":330.0144,"latency":9.0,"recall":0.9507,"label":1,"qp$":7472024.150943397},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":271.6585,"latency":10.1,"recall":0.9678,"label":1,"qp$":6150758.490566038},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":216.5226,"latency":12.9,"recall":1.0,"label":1,"qp$":4902398.490566038},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":123.9553,"latency":23.0,"recall":0.971,"label":1,"qp$":1403267.5471698113},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":59.1479,"latency":44.5,"recall":0.9906,"label":1,"qp$":669598.8679245282},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":40.999,"latency":55.300000000000004,"recall":1.0,"label":1,"qp$":464139.62264150946},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":579.9416,"latency":9.4,"recall":0.9213,"label":1,"qp$":6565376.603773585},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":425.2529,"latency":11.299999999999999,"recall":0.9686,"label":1,"qp$":4814183.773584905},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":397.0539,"latency":13.799999999999999,"recall":1.0,"label":1,"qp$":4494949.811320755},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":516.27,"latency":7.0,"recall":0.9463,"label":1,"qp$":11689132.075471697},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":354.8416,"latency":10.0,"recall":0.9802,"label":1,"qp$":8034149.433962264},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":427.5229,"latency":8.7,"recall":1.0,"label":1,"qp$":9679763.773584906},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":2884.689,"latency":5.3,"recall":0.8801,"label":1,"qp$":8164214.150943396},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":1689.5799,"latency":6.6,"recall":0.9493,"label":1,"qp$":4781829.905660377},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1517.6792,"latency":10.0,"recall":1.0,"label":1,"qp$":4295318.4905660385},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":822.5318,"latency":5.6,"recall":0.9294,"label":1,"qp$":2327920.1886792453},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":378.9146,"latency":10.3,"recall":0.9758,"label":1,"qp$":1072399.8113207547},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":218.6854,"latency":16.2,"recall":1.0,"label":1,"qp$":618920.9433962263},{"db":"ZillizCloud","db_label":"8cu-perf-(Jan-2024)","db_name":"ZillizCloud-8cu-perf-(Jan-2024)","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":5115.5303,"latency":8.7,"recall":0.9469,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-(Jan-2024)","db_name":"ZillizCloud-8cu-perf-(Jan-2024)","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":3685.0767,"latency":9.1,"recall":0.9736,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-(Jan-2024)","db_name":"ZillizCloud-8cu-perf-(Jan-2024)","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":4742.1617,"latency":9.1,"recall":0.9936,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-(Jan-2024)","db_name":"ZillizCloud-8cu-perf-(Jan-2024)","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":6054.4428,"latency":8.1,"recall":0.9155,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-(Jan-2024)","db_name":"ZillizCloud-8cu-perf-(Jan-2024)","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":4104.2598,"latency":10.6,"recall":0.9506,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-(Jan-2024)","db_name":"ZillizCloud-8cu-perf-(Jan-2024)","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":4252.1267,"latency":9.2,"recall":0.9964,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-(Jan-2024)","db_name":"ZillizCloud-8cu-perf-(Jan-2024)","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":1685.3091,"latency":13.299999999999999,"recall":0.9718,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-(Jan-2024)","db_name":"ZillizCloud-8cu-perf-(Jan-2024)","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":769.8991,"latency":10.7,"recall":0.9884,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-(Jan-2024)","db_name":"ZillizCloud-8cu-perf-(Jan-2024)","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":945.4061,"latency":10.9,"recall":0.9941,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-(Jan-2024)","db_name":"ZillizCloud-8cu-perf-(Jan-2024)","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":2214.9028,"latency":8.4,"recall":0.9249,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-(Jan-2024)","db_name":"ZillizCloud-8cu-perf-(Jan-2024)","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":827.975,"latency":12.0,"recall":0.9692,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-(Jan-2024)","db_name":"ZillizCloud-8cu-perf-(Jan-2024)","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":776.9454,"latency":11.4,"recall":0.9966,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-(Jan-2024)","db_name":"ZillizCloud-1cu-cap-(Jan-2024)","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":269.5464,"latency":9.799999999999999,"recall":0.9776,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-(Jan-2024)","db_name":"ZillizCloud-1cu-cap-(Jan-2024)","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":240.0363,"latency":10.6,"recall":0.9822,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-(Jan-2024)","db_name":"ZillizCloud-1cu-cap-(Jan-2024)","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":218.0627,"latency":11.4,"recall":0.9936,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-(Jan-2024)","db_name":"ZillizCloud-1cu-cap-(Jan-2024)","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":392.8825,"latency":6.8999999999999995,"recall":0.9581,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-(Jan-2024)","db_name":"ZillizCloud-1cu-cap-(Jan-2024)","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":343.8204,"latency":8.4,"recall":0.968,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-(Jan-2024)","db_name":"ZillizCloud-1cu-cap-(Jan-2024)","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":216.6773,"latency":10.1,"recall":0.9968,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-(Jan-2024)","db_name":"ZillizCloud-2cu-cap-(Jan-2024)","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":503.2284,"latency":9.0,"recall":0.9677,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-(Jan-2024)","db_name":"ZillizCloud-2cu-cap-(Jan-2024)","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":413.3232,"latency":9.6,"recall":0.981,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-(Jan-2024)","db_name":"ZillizCloud-2cu-cap-(Jan-2024)","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":425.5492,"latency":10.200000000000001,"recall":0.9936,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-(Jan-2024)","db_name":"ZillizCloud-2cu-cap-(Jan-2024)","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":789.1229,"latency":5.6,"recall":0.9396,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-(Jan-2024)","db_name":"ZillizCloud-2cu-cap-(Jan-2024)","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":571.4257,"latency":7.7,"recall":0.9668,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-(Jan-2024)","db_name":"ZillizCloud-2cu-cap-(Jan-2024)","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":411.7653,"latency":9.1,"recall":0.9968,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-(Jan-2024)","db_name":"ZillizCloud-2cu-cap-(Jan-2024)","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":98.0448,"latency":16.1,"recall":0.9803,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-(Jan-2024)","db_name":"ZillizCloud-2cu-cap-(Jan-2024)","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":58.3152,"latency":28.0,"recall":0.9891,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-(Jan-2024)","db_name":"ZillizCloud-2cu-cap-(Jan-2024)","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":46.8304,"latency":28.6,"recall":0.9941,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-(Jan-2024)","db_name":"ZillizCloud-2cu-cap-(Jan-2024)","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":170.5693,"latency":8.9,"recall":0.9605,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-(Jan-2024)","db_name":"ZillizCloud-2cu-cap-(Jan-2024)","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":94.7766,"latency":19.599999999999998,"recall":0.9843,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-(Jan-2024)","db_name":"ZillizCloud-2cu-cap-(Jan-2024)","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":44.8695,"latency":30.900000000000002,"recall":0.9966,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-(Jan-2024)","db_name":"ZillizCloud-1cu-perf-(Jan-2024)","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":722.0315,"latency":7.7,"recall":0.976,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-(Jan-2024)","db_name":"ZillizCloud-1cu-perf-(Jan-2024)","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":467.5795,"latency":8.8,"recall":0.9898,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-(Jan-2024)","db_name":"ZillizCloud-1cu-perf-(Jan-2024)","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":975.2503,"latency":8.200000000000001,"recall":0.9936,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-(Jan-2024)","db_name":"ZillizCloud-1cu-perf-(Jan-2024)","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":873.3712,"latency":6.7,"recall":0.9477,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-(Jan-2024)","db_name":"ZillizCloud-1cu-perf-(Jan-2024)","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":544.6203,"latency":8.4,"recall":0.977,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-(Jan-2024)","db_name":"ZillizCloud-1cu-perf-(Jan-2024)","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":930.9164,"latency":9.6,"recall":0.9968,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-(Jan-2024)","db_name":"ZillizCloud-1cu-perf-(Jan-2024)","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-(Jan-2024)","db_name":"ZillizCloud-1cu-perf-(Jan-2024)","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-(Jan-2024)","db_name":"ZillizCloud-1cu-perf-(Jan-2024)","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-(Jan-2024)","db_name":"ZillizCloud-1cu-perf-(Jan-2024)","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-(Jan-2024)","db_name":"ZillizCloud-1cu-perf-(Jan-2024)","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-(Jan-2024)","db_name":"ZillizCloud-1cu-perf-(Jan-2024)","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-(Jan-2024)","db_name":"ZillizCloud-1cu-cap-(Jan-2024)","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-(Jan-2024)","db_name":"ZillizCloud-1cu-cap-(Jan-2024)","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-(Jan-2024)","db_name":"ZillizCloud-1cu-cap-(Jan-2024)","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-(Jan-2024)","db_name":"ZillizCloud-1cu-cap-(Jan-2024)","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-(Jan-2024)","db_name":"ZillizCloud-1cu-cap-(Jan-2024)","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-(Jan-2024)","db_name":"ZillizCloud-1cu-cap-(Jan-2024)","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0}] \ No newline at end of file From 6fd9b54cf37944dca51a1a8dd96bc1d7f6ad49db Mon Sep 17 00:00:00 2001 From: cutecutecat Date: Tue, 16 Apr 2024 11:13:51 +0800 Subject: [PATCH 058/327] fix: new interface of pgvecto.rs after v0.2 Signed-off-by: cutecutecat --- pyproject.toml | 2 +- .../backend/clients/pgvecto_rs/config.py | 76 +++++++++++-------- .../backend/clients/pgvecto_rs/pgvecto_rs.py | 32 ++++---- .../frontend/const/dbCaseConfigs.py | 11 ++- 4 files changed, 71 insertions(+), 50 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a7c5c892b..3a72d15c6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,7 +65,7 @@ qdrant = [ "qdrant-client" ] pinecone = [ "pinecone-client" ] weaviate = [ "weaviate-client" ] elastic = [ "elasticsearch" ] -pgvector = [ "pgvector", "sqlalchemy" ] +pgvector = [ "pgvector", "psycopg2" ] pgvecto_rs = [ "psycopg2" ] redis = [ "redis" ] chromadb = [ "chromadb" ] diff --git a/vectordb_bench/backend/clients/pgvecto_rs/config.py b/vectordb_bench/backend/clients/pgvecto_rs/config.py index 7ad5983f1..73c41c239 100644 --- a/vectordb_bench/backend/clients/pgvecto_rs/config.py +++ b/vectordb_bench/backend/clients/pgvecto_rs/config.py @@ -8,42 +8,30 @@ class PgVectoRSConfig(DBConfig): user_name: SecretStr = "postgres" password: SecretStr - url: SecretStr + host: str = "localhost" + port: int = 5432 db_name: str def to_dict(self) -> dict: user_str = self.user_name.get_secret_value() pwd_str = self.password.get_secret_value() - url_str = self.url.get_secret_value() - host, port = url_str.split(":") return { - "host": host, - "port": port, + "host": self.host, + "port": self.port, "dbname": self.db_name, "user": user_str, - "password": pwd_str, + "password": pwd_str } - class PgVectoRSIndexConfig(BaseModel, DBCaseConfig): metric_type: MetricType | None = None - quantizationType: Literal["trivial", "scalar", "product"] - quantizationRatio: None | Literal["x4", "x8", "x16", "x32", "x64"] - - def parse_quantization(self) -> str: - if self.quantizationType == "trivial": - return "quantization = { trivial = { } }" - elif self.quantizationType == "scalar": - return "quantization = { scalar = { } }" - else: - return f'quantization = {{ product = {{ ratio = "{self.quantizationRatio}" }} }}' def parse_metric(self) -> str: if self.metric_type == MetricType.L2: - return "l2_ops" + return "vector_l2_ops" elif self.metric_type == MetricType.IP: - return "dot_ops" - return "cosine_ops" + return "vector_dot_ops" + return "vector_cos_ops" def parse_metric_fun_op(self) -> str: if self.metric_type == MetricType.L2: @@ -52,16 +40,27 @@ def parse_metric_fun_op(self) -> str: return "<#>" return "<=>" +class PgVectoRSQuantConfig(PgVectoRSIndexConfig): + quantizationType: Literal["trivial", "scalar", "product"] + quantizationRatio: None | Literal["x4", "x8", "x16", "x32", "x64"] -class HNSWConfig(PgVectoRSIndexConfig): + def parse_quantization(self) -> str: + if self.quantizationType == "trivial": + return "quantization = { trivial = { } }" + elif self.quantizationType == "scalar": + return "quantization = { scalar = { } }" + else: + return f'quantization = {{ product = {{ ratio = "{self.quantizationRatio}" }} }}' + + +class HNSWConfig(PgVectoRSQuantConfig): M: int efConstruction: int index: IndexType = IndexType.HNSW def index_param(self) -> dict: options = f""" -capacity = 1048576 -[algorithm.hnsw] +[indexing.hnsw] m = {self.M} ef_construction = {self.efConstruction} {self.parse_quantization()} @@ -72,17 +71,16 @@ def search_param(self) -> dict: return {"metrics_op": self.parse_metric_fun_op()} -class IVFFlatConfig(PgVectoRSIndexConfig): +class IVFFlatConfig(PgVectoRSQuantConfig): nlist: int nprobe: int | None = None index: IndexType = IndexType.IVFFlat def index_param(self) -> dict: options = f""" -capacity = 1048576 -[algorithm.ivf] +[indexing.ivf] nlist = {self.nlist} -nprob = {self.nprobe if self.nprobe else 10} +nsample = {self.nprobe if self.nprobe else 10} {self.parse_quantization()} """ return {"options": options, "metric": self.parse_metric()} @@ -90,14 +88,29 @@ def index_param(self) -> dict: def search_param(self) -> dict: return {"metrics_op": self.parse_metric_fun_op()} +class IVFFlatSQ8Config(PgVectoRSIndexConfig): + nlist: int + nprobe: int | None = None + index: IndexType = IndexType.IVFSQ8 + + def index_param(self) -> dict: + options = f""" +[indexing.ivf] +nlist = {self.nlist} +nsample = {self.nprobe if self.nprobe else 10} +quantization = {{ scalar = {{ }} }} +""" + return {"options": options, "metric": self.parse_metric()} + + def search_param(self) -> dict: + return {"metrics_op": self.parse_metric_fun_op()} -class FLATConfig(PgVectoRSIndexConfig): +class FLATConfig(PgVectoRSQuantConfig): index: IndexType = IndexType.Flat def index_param(self) -> dict: options = f""" -capacity = 1048576 -[algorithm.flat] +[indexing.flat] {self.parse_quantization()} """ return {"options": options, "metric": self.parse_metric()} @@ -107,9 +120,8 @@ def search_param(self) -> dict: _pgvecto_rs_case_config = { - IndexType.AUTOINDEX: HNSWConfig, IndexType.HNSW: HNSWConfig, - IndexType.DISKANN: HNSWConfig, IndexType.IVFFlat: IVFFlatConfig, + IndexType.IVFSQ8: IVFFlatSQ8Config, IndexType.Flat: FLATConfig, } diff --git a/vectordb_bench/backend/clients/pgvecto_rs/pgvecto_rs.py b/vectordb_bench/backend/clients/pgvecto_rs/pgvecto_rs.py index 22476522c..22caa43e6 100644 --- a/vectordb_bench/backend/clients/pgvecto_rs/pgvecto_rs.py +++ b/vectordb_bench/backend/clients/pgvecto_rs/pgvecto_rs.py @@ -1,18 +1,17 @@ -"""Wrapper around the Pgvector vector database over VectorDB""" +"""Wrapper around the Pgvecto.rs vector database over VectorDB""" import io import logging from contextlib import contextmanager from typing import Any import pandas as pd - import psycopg2 +import psycopg2.extras from ..api import VectorDB, DBCaseConfig log = logging.getLogger(__name__) - class PgVectoRS(VectorDB): """Use SQLAlchemy instructions""" @@ -66,6 +65,8 @@ def init(self) -> None: self.conn = psycopg2.connect(**self.db_config) self.conn.autocommit = False self.cursor = self.conn.cursor() + self.cursor.execute('SET search_path = "$user", public, vectors') + self.conn.commit() try: yield @@ -113,7 +114,7 @@ def _create_index(self): self.conn.commit() except Exception as e: log.warning( - f"Failed to create pgvector table: {self.table_name} error: {e}" + f"Failed to create pgvecto.rs table: {self.table_name} error: {e}" ) raise e from None @@ -127,13 +128,10 @@ def _create_table(self, dim: int): f'CREATE TABLE IF NOT EXISTS public."{self.table_name}" \ (id Integer PRIMARY KEY, embedding vector({dim}));' ) - self.cursor.execute( - f'ALTER TABLE public."{self.table_name}" ALTER COLUMN embedding SET STORAGE PLAIN;' - ) self.conn.commit() except Exception as e: log.warning( - f"Failed to create pgvector table: {self.table_name} error: {e}" + f"Failed to create pgvecto.rs table: {self.table_name} error: {e}" ) raise e from None @@ -146,22 +144,24 @@ def insert_embeddings( assert self.conn is not None, "Connection is not initialized" assert self.cursor is not None, "Cursor is not initialized" + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + try: - items = {"id": metadata, "embedding": embeddings} + items = { + "id": metadata, + "embedding": embeddings + } df = pd.DataFrame(items) csv_buffer = io.StringIO() df.to_csv(csv_buffer, index=False, header=False) csv_buffer.seek(0) - self.cursor.copy_expert( - f'COPY public."{self.table_name}" FROM STDIN WITH (FORMAT CSV)', - csv_buffer, - ) + self.cursor.copy_expert(f"COPY public.\"{self.table_name}\" FROM STDIN WITH (FORMAT CSV)", csv_buffer) self.conn.commit() return len(metadata), None except Exception as e: - log.warning( - f"Failed to insert data into pgvector table ({self.table_name}), error: {e}" - ) + log.warning(f"Failed to insert data into pgvecto.rs table ({self.table_name}), error: {e}") + return 0, e def search_embedding( self, diff --git a/vectordb_bench/frontend/const/dbCaseConfigs.py b/vectordb_bench/frontend/const/dbCaseConfigs.py index fad5f362d..24de64725 100644 --- a/vectordb_bench/frontend/const/dbCaseConfigs.py +++ b/vectordb_bench/frontend/const/dbCaseConfigs.py @@ -397,6 +397,11 @@ class CaseConfigInput(BaseModel): inputConfig={ "options": ["trivial", "scalar", "product"], }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) + in [ + IndexType.HNSW.value, + IndexType.IVFFlat.value, + ], ) CaseConfigParamInput_QuantizationRatio_PgVectoRS = CaseConfigInput( @@ -406,7 +411,11 @@ class CaseConfigInput(BaseModel): "options": ["x4", "x8", "x16", "x32", "x64"], }, isDisplayed=lambda config: config.get(CaseConfigParamType.quantizationType, None) - == "product", + == "product" and config.get(CaseConfigParamType.IndexType, None) + in [ + IndexType.HNSW.value, + IndexType.IVFFlat.value, + ], ) CaseConfigParamInput_ZillizLevel = CaseConfigInput( From 81bc5f21719d66c423265d0a7bf281510e748aaa Mon Sep 17 00:00:00 2001 From: "min.tian" Date: Mon, 22 Apr 2024 06:32:05 +0000 Subject: [PATCH 059/327] fix qdrant client: compatibe with open-source qdrant Signed-off-by: min.tian --- .../backend/clients/qdrant_cloud/config.py | 25 ++++++++++++++----- .../clients/qdrant_cloud/qdrant_cloud.py | 18 +++++++------ 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/vectordb_bench/backend/clients/qdrant_cloud/config.py b/vectordb_bench/backend/clients/qdrant_cloud/config.py index a2c8d1a86..5b1dd7f18 100644 --- a/vectordb_bench/backend/clients/qdrant_cloud/config.py +++ b/vectordb_bench/backend/clients/qdrant_cloud/config.py @@ -1,18 +1,31 @@ from pydantic import BaseModel, SecretStr from ..api import DBConfig, DBCaseConfig, MetricType +from pydantic import validator - +# Allowing `api_key` to be left empty, to ensure compatibility with the open-source Qdrant. class QdrantConfig(DBConfig): url: SecretStr api_key: SecretStr def to_dict(self) -> dict: - return { - "url": self.url.get_secret_value(), - "api_key": self.api_key.get_secret_value(), - "prefer_grpc": True, - } + api_key = self.api_key.get_secret_value() + if len(api_key) > 0: + return { + "url": self.url.get_secret_value(), + "api_key": self.api_key.get_secret_value(), + "prefer_grpc": True, + } + else: + return {"url": self.url.get_secret_value(),} + + @validator("*") + def not_empty_field(cls, v, field): + if field.name in ["api_key", "db_label"]: + return v + if isinstance(v, (str, SecretStr)) and len(v) == 0: + raise ValueError("Empty string!") + return v class QdrantIndexConfig(BaseModel, DBCaseConfig): metric_type: MetricType | None = None diff --git a/vectordb_bench/backend/clients/qdrant_cloud/qdrant_cloud.py b/vectordb_bench/backend/clients/qdrant_cloud/qdrant_cloud.py index c4619e657..a51632bc6 100644 --- a/vectordb_bench/backend/clients/qdrant_cloud/qdrant_cloud.py +++ b/vectordb_bench/backend/clients/qdrant_cloud/qdrant_cloud.py @@ -43,8 +43,7 @@ def __init__( if drop_old: log.info(f"QdrantCloud client drop_old collection: {self.collection_name}") tmp_client.delete_collection(self.collection_name) - - self._create_collection(dim, tmp_client) + self._create_collection(dim, tmp_client) tmp_client = None @contextmanager @@ -110,13 +109,18 @@ def insert_embeddings( ) -> (int, Exception): """Insert embeddings into Milvus. should call self.init() first""" assert self.qdrant_client is not None + QDRANT_BATCH_SIZE = 500 try: # TODO: counts - _ = self.qdrant_client.upsert( - collection_name=self.collection_name, - wait=True, - points=Batch(ids=metadata, payloads=[{self._primary_field: v} for v in metadata], vectors=embeddings) - ) + for offset in range(0, len(embeddings), QDRANT_BATCH_SIZE): + vectors = embeddings[offset: offset + QDRANT_BATCH_SIZE] + ids = metadata[offset: offset + QDRANT_BATCH_SIZE] + payloads=[{self._primary_field: v} for v in ids] + _ = self.qdrant_client.upsert( + collection_name=self.collection_name, + wait=True, + points=Batch(ids=ids, payloads=payloads, vectors=vectors), + ) except Exception as e: log.info(f"Failed to insert data, {e}") return 0, e From d9eb301edcdb9e06ce91106872c6c25df6a9deac Mon Sep 17 00:00:00 2001 From: "min.tian" Date: Fri, 26 Apr 2024 13:38:38 +0800 Subject: [PATCH 060/327] fix: pgvector-hnsw-support Signed-off-by: min.tian --- .../backend/clients/pgvector/config.py | 9 +-- .../frontend/const/dbCaseConfigs.py | 59 +++++++++++++++++-- 2 files changed, 56 insertions(+), 12 deletions(-) diff --git a/vectordb_bench/backend/clients/pgvector/config.py b/vectordb_bench/backend/clients/pgvector/config.py index ec9b1eed0..5de49ae76 100644 --- a/vectordb_bench/backend/clients/pgvector/config.py +++ b/vectordb_bench/backend/clients/pgvector/config.py @@ -54,17 +54,10 @@ class HNSWConfig(PgVectorIndexConfig): ef: int | None = None index: IndexType = IndexType.HNSW - def index_param(self) -> dict: - return { - "metric_type": self.parse_metric(), - "index_type": self.index.value, - "params": {"M": self.M, "efConstruction": self.efConstruction}, - } - def index_param(self) -> dict: return { "m" : self.M, - "efConstruction" : self.efConstruction, + "ef_construction" : self.efConstruction, "metric" : self.parse_metric() } diff --git a/vectordb_bench/frontend/const/dbCaseConfigs.py b/vectordb_bench/frontend/const/dbCaseConfigs.py index 24de64725..9b122910a 100644 --- a/vectordb_bench/frontend/const/dbCaseConfigs.py +++ b/vectordb_bench/frontend/const/dbCaseConfigs.py @@ -371,14 +371,27 @@ class CaseConfigInput(BaseModel): ], ) +CaseConfigParamInput_IndexType_PG = CaseConfigInput( + label=CaseConfigParamType.IndexType, + inputType=InputType.Option, + inputConfig={ + "options": [ + IndexType.HNSW.value, + IndexType.IVFFlat.value, + ], + }, +) + CaseConfigParamInput_Lists = CaseConfigInput( label=CaseConfigParamType.lists, inputType=InputType.Number, inputConfig={ "min": 1, "max": 65536, - "value": 10, + "value": 1000, }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) + in [IndexType.IVFFlat.value], ) CaseConfigParamInput_Probes = CaseConfigInput( @@ -387,8 +400,34 @@ class CaseConfigInput(BaseModel): inputConfig={ "min": 1, "max": 65536, - "value": 1, + "value": 10, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) + in [IndexType.IVFFlat.value], +) + +CaseConfigParamInput_EF_PG = CaseConfigInput( + label=CaseConfigParamType.EF, + inputType=InputType.Number, + inputConfig={ + "min": 1, + "max": 65536, + "value": 128, }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) + in [IndexType.HNSW.value], +) + +CaseConfigParamInput_EFC_PG = CaseConfigInput( + label=CaseConfigParamType.EFConstruction, + inputType=InputType.Number, + inputConfig={ + "min": 1, + "max": 65536, + "value": 300, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) + in [IndexType.HNSW.value], ) CaseConfigParamInput_QuantizationType_PgVectoRS = CaseConfigInput( @@ -479,8 +518,20 @@ class CaseConfigInput(BaseModel): CaseConfigParamInput_NumCandidates_ES, ] -PgVectorLoadingConfig = [CaseConfigParamInput_Lists] -PgVectorPerformanceConfig = [CaseConfigParamInput_Lists, CaseConfigParamInput_Probes] +PgVectorLoadingConfig = [ + CaseConfigParamInput_IndexType_PG, + CaseConfigParamInput_Lists, + CaseConfigParamInput_M, + CaseConfigParamInput_EFC_PG +] +PgVectorPerformanceConfig = [ + CaseConfigParamInput_IndexType_PG, + CaseConfigParamInput_Lists, + CaseConfigParamInput_Probes, + CaseConfigParamInput_M, + CaseConfigParamInput_EF_PG, + CaseConfigParamInput_EFC_PG +] PgVectoRSLoadingConfig = [ CaseConfigParamInput_IndexType, From 971787b666772eeef66c8270b2d0fef6a10cab08 Mon Sep 17 00:00:00 2001 From: yangxuan Date: Mon, 29 Apr 2024 15:29:00 +0800 Subject: [PATCH 061/327] enhance: Remove last batch in insert_embeddings Signed-off-by: yangxuan --- .../backend/clients/milvus/milvus.py | 5 +-- .../backend/clients/pgvector/pgvector.py | 45 ++++++++----------- .../backend/runner/serial_runner.py | 2 - vectordb_bench/backend/task_runner.py | 2 +- 4 files changed, 22 insertions(+), 32 deletions(-) diff --git a/vectordb_bench/backend/clients/milvus/milvus.py b/vectordb_bench/backend/clients/milvus/milvus.py index 4e8a35275..58334efe9 100644 --- a/vectordb_bench/backend/clients/milvus/milvus.py +++ b/vectordb_bench/backend/clients/milvus/milvus.py @@ -89,6 +89,7 @@ def init(self) -> None: connections.disconnect("default") def _optimize(self): + self._post_insert() log.info(f"{self.name} optimizing before search") try: self.col.load() @@ -116,7 +117,7 @@ def wait_index(): time.sleep(5) wait_index() - + # Skip compaction if use GPU indexType if self.case_config.index in [IndexType.GPU_CAGRA, IndexType.GPU_IVF_FLAT, IndexType.GPU_IVF_PQ]: log.debug("skip compaction for gpu index type.") @@ -179,8 +180,6 @@ def insert_embeddings( ] res = self.col.insert(insert_data) insert_count += len(res.primary_keys) - if kwargs.get("last_batch"): - self._post_insert() except MilvusException as e: log.info(f"Failed to insert data: {e}") return (insert_count, e) diff --git a/vectordb_bench/backend/clients/pgvector/pgvector.py b/vectordb_bench/backend/clients/pgvector/pgvector.py index a69b7cbde..afe4218c4 100644 --- a/vectordb_bench/backend/clients/pgvector/pgvector.py +++ b/vectordb_bench/backend/clients/pgvector/pgvector.py @@ -10,7 +10,7 @@ from ..api import IndexType, VectorDB, DBCaseConfig -log = logging.getLogger(__name__) +log = logging.getLogger(__name__) class PgVector(VectorDB): """ Use SQLAlchemy instructions""" @@ -36,12 +36,12 @@ def __init__( # construct basic units self.conn = psycopg2.connect(**self.db_config) self.conn.autocommit = False - self.cursor = self.conn.cursor() - + self.cursor = self.conn.cursor() + # create vector extension self.cursor.execute('CREATE EXTENSION IF NOT EXISTS vector') self.conn.commit() - + if drop_old : log.info(f"Pgvector client drop table : {self.table_name}") # self.pg_table.drop(pg_engine, checkfirst=True) @@ -49,7 +49,7 @@ def __init__( self._drop_table() self._create_table(dim) self._create_index() - + self.cursor.close() self.conn.close() self.cursor = None @@ -66,7 +66,7 @@ def init(self) -> None: self.conn = psycopg2.connect(**self.db_config) self.conn.autocommit = False self.cursor = self.conn.cursor() - + try: yield finally: @@ -74,39 +74,36 @@ def init(self) -> None: self.conn.close() self.cursor = None self.conn = None - + def _drop_table(self): assert self.conn is not None, "Connection is not initialized" assert self.cursor is not None, "Cursor is not initialized" - + self.cursor.execute(f'DROP TABLE IF EXISTS public."{self.table_name}"') self.conn.commit() - + def ready_to_load(self): pass def optimize(self): - pass - - def _post_insert(self): - log.info(f"{self.name} post insert before optimize") + log.info(f"{self.name} optimizing") self._drop_index() self._create_index() def ready_to_search(self): pass - + def _drop_index(self): assert self.conn is not None, "Connection is not initialized" assert self.cursor is not None, "Cursor is not initialized" - + self.cursor.execute(f'DROP INDEX IF EXISTS "{self._index_name}"') self.conn.commit() - + def _create_index(self): assert self.conn is not None, "Connection is not initialized" assert self.cursor is not None, "Cursor is not initialized" - + index_param = self.case_config.index_param() if self.case_config.index == IndexType.HNSW: log.debug(f'Creating HNSW index. m={index_param["m"]}, ef_construction={index_param["ef_construction"]}') @@ -117,11 +114,11 @@ def _create_index(self): else: assert "Invalid index type {self.case_config.index}" self.conn.commit() - + def _create_table(self, dim : int): assert self.conn is not None, "Connection is not initialized" assert self.cursor is not None, "Cursor is not initialized" - + try: # create table self.cursor.execute(f'CREATE TABLE IF NOT EXISTS public."{self.table_name}" (id BIGINT PRIMARY KEY, embedding vector({dim}));') @@ -151,16 +148,13 @@ def insert_embeddings( csv_buffer.seek(0) self.cursor.copy_expert(f"COPY public.\"{self.table_name}\" FROM STDIN WITH (FORMAT CSV)", csv_buffer) self.conn.commit() - - if kwargs.get("last_batch"): - self._post_insert() - + return len(metadata), None except Exception as e: - log.warning(f"Failed to insert data into pgvector table ({self.table_name}), error: {e}") + log.warning(f"Failed to insert data into pgvector table ({self.table_name}), error: {e}") return 0, e - def search_embedding( + def search_embedding( self, query: list[float], k: int = 100, @@ -184,4 +178,3 @@ def search_embedding( result = self.cursor.fetchall() return [int(i[0]) for i in result] - diff --git a/vectordb_bench/backend/runner/serial_runner.py b/vectordb_bench/backend/runner/serial_runner.py index aeed0ec74..e4861abd1 100644 --- a/vectordb_bench/backend/runner/serial_runner.py +++ b/vectordb_bench/backend/runner/serial_runner.py @@ -46,11 +46,9 @@ def task(self) -> int: del(emb_np) log.debug(f"batch dataset size: {len(all_embeddings)}, {len(all_metadata)}") - last_batch = self.dataset.data.size - count == len(all_metadata) insert_count, error = self.db.insert_embeddings( embeddings=all_embeddings, metadata=all_metadata, - last_batch=last_batch, ) if error is not None: raise error diff --git a/vectordb_bench/backend/task_runner.py b/vectordb_bench/backend/task_runner.py index a5f4e317f..c5c368d02 100644 --- a/vectordb_bench/backend/task_runner.py +++ b/vectordb_bench/backend/task_runner.py @@ -140,8 +140,8 @@ def _run_perf_case(self, drop_old: bool = True) -> Metric: ) self._init_search_runner() - m.recall, m.serial_latency_p99 = self._serial_search() m.qps = self._conc_search() + m.recall, m.serial_latency_p99 = self._serial_search() except Exception as e: log.warning(f"Failed to run performance case, reason = {e}") traceback.print_exc() From f60816dcf7faf2d210c0f7a78f537c782e30265d Mon Sep 17 00:00:00 2001 From: Davide Mauri Date: Fri, 3 May 2024 19:10:53 -0700 Subject: [PATCH 062/327] Pinning Streamlit to version 1.31.1 to avoid error #315 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3a72d15c6..f3ae99eec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ classifiers = [ dependencies = [ "pytz", "streamlit-autorefresh", - "streamlit>=1.23.0", + "streamlit==1.31.1", "streamlit_extras", "tqdm", "s3fs", From 00406c8c2b7a3f8081ef433e7246a25b699b96a8 Mon Sep 17 00:00:00 2001 From: "Jonathan S. Katz" Date: Fri, 3 May 2024 20:35:36 +0000 Subject: [PATCH 063/327] Optimize pgvector test for semi-recent enhancements This commit adds several changes to the pgvector test to create a more representative test environment based on recent and older changes to pgvector. Notable changes include allowing for testing of parallel index buiding parameters, using loading with the recommended binary loading method, and other changes to better emulate what a typical user of pgvector would do. This commit also has some general cleanups as well. Co-authored-by: Mark Greenhalgh Co-authored-by: Tyler House --- pyproject.toml | 3 +- vectordb_bench/backend/clients/api.py | 1 + .../backend/clients/pgvector/config.py | 198 +++++++++--- .../backend/clients/pgvector/pgvector.py | 295 ++++++++++++++---- .../components/run_test/caseSelector.py | 9 +- .../frontend/const/dbCaseConfigs.py | 137 +++++--- vectordb_bench/models.py | 9 +- 7 files changed, 507 insertions(+), 145 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f3ae99eec..e048a19cb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,13 +59,14 @@ all = [ "redis", "chromadb", "psycopg2", + "psycopg", ] qdrant = [ "qdrant-client" ] pinecone = [ "pinecone-client" ] weaviate = [ "weaviate-client" ] elastic = [ "elasticsearch" ] -pgvector = [ "pgvector", "psycopg2" ] +pgvector = [ "pgvector", "psycopg" ] pgvecto_rs = [ "psycopg2" ] redis = [ "redis" ] chromadb = [ "chromadb" ] diff --git a/vectordb_bench/backend/clients/api.py b/vectordb_bench/backend/clients/api.py index 7d4e4f684..d1325322a 100644 --- a/vectordb_bench/backend/clients/api.py +++ b/vectordb_bench/backend/clients/api.py @@ -20,6 +20,7 @@ class IndexType(str, Enum): Flat = "FLAT" AUTOINDEX = "AUTOINDEX" ES_HNSW = "hnsw" + ES_IVFFlat = "ivfflat" GPU_IVF_FLAT = "GPU_IVF_FLAT" GPU_IVF_PQ = "GPU_IVF_PQ" GPU_CAGRA = "GPU_CAGRA" diff --git a/vectordb_bench/backend/clients/pgvector/config.py b/vectordb_bench/backend/clients/pgvector/config.py index 5de49ae76..a4cc584e8 100644 --- a/vectordb_bench/backend/clients/pgvector/config.py +++ b/vectordb_bench/backend/clients/pgvector/config.py @@ -1,29 +1,62 @@ +from abc import abstractmethod +from typing import Any, Mapping, Optional, Sequence, TypedDict from pydantic import BaseModel, SecretStr -from ..api import DBConfig, DBCaseConfig, IndexType, MetricType +from typing_extensions import LiteralString +from ..api import DBCaseConfig, DBConfig, IndexType, MetricType POSTGRE_URL_PLACEHOLDER = "postgresql://%s:%s@%s/%s" + +class PgVectorConfigDict(TypedDict): + """These keys will be directly used as kwargs in psycopg connection string, + so the names must match exactly psycopg API""" + + user: str + password: str + host: str + port: int + dbname: str + + class PgVectorConfig(DBConfig): - user_name: SecretStr = "postgres" + user_name: SecretStr = SecretStr("postgres") password: SecretStr host: str = "localhost" port: int = 5432 db_name: str - def to_dict(self) -> dict: + def to_dict(self) -> PgVectorConfigDict: user_str = self.user_name.get_secret_value() pwd_str = self.password.get_secret_value() return { - "host" : self.host, - "port" : self.port, - "dbname" : self.db_name, - "user" : user_str, - "password" : pwd_str + "host": self.host, + "port": self.port, + "dbname": self.db_name, + "user": user_str, + "password": pwd_str, } + +class PgVectorIndexParam(TypedDict): + metric: str + index_type: str + index_creation_with_options: Sequence[dict[str, Any]] + maintenance_work_mem: Optional[str] + max_parallel_workers: Optional[int] + + +class PgVectorSearchParam(TypedDict): + metric_fun_op: LiteralString + + +class PgVectorSessionCommands(TypedDict): + session_options: Sequence[dict[str, Any]] + + class PgVectorIndexConfig(BaseModel, DBCaseConfig): metric_type: MetricType | None = None - index: IndexType + create_index_before_load: bool = False + create_index_after_load: bool = True def parse_metric(self) -> str: if self.metric_type == MetricType.L2: @@ -32,7 +65,7 @@ def parse_metric(self) -> str: return "vector_ip_ops" return "vector_cosine_ops" - def parse_metric_fun_op(self) -> str: + def parse_metric_fun_op(self) -> LiteralString: if self.metric_type == MetricType.L2: return "<->" elif self.metric_type == MetricType.IP: @@ -46,48 +79,137 @@ def parse_metric_fun_str(self) -> str: return "max_inner_product" return "cosine_distance" + @abstractmethod + def index_param(self) -> PgVectorIndexParam: + ... + + @abstractmethod + def search_param(self) -> PgVectorSearchParam: + ... + + @abstractmethod + def session_param(self) -> PgVectorSessionCommands: + ... + + @staticmethod + def _optionally_build_with_options(with_options: Mapping[str, Any]) -> Sequence[dict[str, Any]]: + """Walk through mappings, creating a List of {key1 = value} pairs. That will be used to build a where clause""" + options = [] + for option_name, value in with_options.items(): + if value is not None: + options.append( + { + "option_name": option_name, + "val": str(value), + } + ) + return options + + @staticmethod + def _optionally_build_set_options( + set_mapping: Mapping[str, Any] + ) -> Sequence[dict[str, Any]]: + """Walk through options, creating 'SET 'key1 = "value1";' commands""" + session_options = [] + for setting_name, value in set_mapping.items(): + if value: + session_options.append( + {"parameter": { + "setting_name": setting_name, + "val": str(value), + }, + } + ) + return session_options + + +class PgVectorIVFFlatConfig(PgVectorIndexConfig): + """ + An IVFFlat index divides vectors into lists, and then searches a subset of those lists that are + closest to the query vector. It has faster build times and uses less memory than HNSW, + but has lower query performance (in terms of speed-recall tradeoff). + + Three keys to achieving good recall are: + + Create the index after the table has some data + Choose an appropriate number of lists - a good place to start is rows / 1000 for up to 1M rows and sqrt(rows) for + over 1M rows. + When querying, specify an appropriate number of probes (higher is better for recall, lower is better for speed) - + a good place to start is sqrt(lists) + """ + + lists: int | None + probes: int | None + index: IndexType = IndexType.ES_IVFFlat + maintenance_work_mem: Optional[str] = None + max_parallel_workers: Optional[int] = None + + def index_param(self) -> PgVectorIndexParam: + index_parameters = {"lists": self.lists} + return { + "metric": self.parse_metric(), + "index_type": self.index.value, + "index_creation_with_options": self._optionally_build_with_options( + index_parameters + ), + "maintenance_work_mem": self.maintenance_work_mem, + "max_parallel_workers": self.max_parallel_workers, + } - -class HNSWConfig(PgVectorIndexConfig): - M: int - efConstruction: int - ef: int | None = None - index: IndexType = IndexType.HNSW - - def index_param(self) -> dict: + def search_param(self) -> PgVectorSearchParam: return { - "m" : self.M, - "ef_construction" : self.efConstruction, - "metric" : self.parse_metric() + "metric_fun_op": self.parse_metric_fun_op(), } - def search_param(self) -> dict: + def session_param(self) -> PgVectorSessionCommands: + session_parameters = {"ivfflat.probes": self.probes} return { - "ef" : self.ef, - "metric_fun" : self.parse_metric_fun_str(), - "metric_fun_op" : self.parse_metric_fun_op(), + "session_options": self._optionally_build_set_options(session_parameters) } -class IVFFlatConfig(PgVectorIndexConfig): - lists: int | None = 1000 - probes: int | None = 10 - index: IndexType = IndexType.IVFFlat +class PgVectorHNSWConfig(PgVectorIndexConfig): + """ + An HNSW index creates a multilayer graph. It has better query performance than IVFFlat (in terms of + speed-recall tradeoff), but has slower build times and uses more memory. Also, an index can be + created without any data in the table since there isn't a training step like IVFFlat. + """ + + m: int | None # DETAIL: Valid values are between "2" and "100". + ef_construction: ( + int | None + ) # ef_construction must be greater than or equal to 2 * m + ef_search: int | None + index: IndexType = IndexType.ES_HNSW + maintenance_work_mem: Optional[str] = None + max_parallel_workers: Optional[int] = None + + def index_param(self) -> PgVectorIndexParam: + index_parameters = {"m": self.m, "ef_construction": self.ef_construction} + return { + "metric": self.parse_metric(), + "index_type": self.index.value, + "index_creation_with_options": self._optionally_build_with_options( + index_parameters + ), + "maintenance_work_mem": self.maintenance_work_mem, + "max_parallel_workers": self.max_parallel_workers, + } - def index_param(self) -> dict: + def search_param(self) -> PgVectorSearchParam: return { - "lists" : self.lists, - "metric" : self.parse_metric() + "metric_fun_op": self.parse_metric_fun_op(), } - def search_param(self) -> dict: + def session_param(self) -> PgVectorSessionCommands: + session_parameters = {"hnsw.ef_search": self.ef_search} return { - "probes" : self.probes, - "metric_fun" : self.parse_metric_fun_str(), - "metric_fun_op" : self.parse_metric_fun_op(), + "session_options": self._optionally_build_set_options(session_parameters) } + _pgvector_case_config = { - IndexType.HNSW: HNSWConfig, - IndexType.IVFFlat: IVFFlatConfig, + IndexType.HNSW: PgVectorHNSWConfig, + IndexType.ES_HNSW: PgVectorHNSWConfig, + IndexType.IVFFlat: PgVectorIVFFlatConfig, } diff --git a/vectordb_bench/backend/clients/pgvector/pgvector.py b/vectordb_bench/backend/clients/pgvector/pgvector.py index afe4218c4..8f8244412 100644 --- a/vectordb_bench/backend/clients/pgvector/pgvector.py +++ b/vectordb_bench/backend/clients/pgvector/pgvector.py @@ -1,25 +1,36 @@ """Wrapper around the Pgvector vector database over VectorDB""" -import io import logging +import pprint from contextlib import contextmanager -from typing import Any -import pandas as pd -import psycopg2 -import psycopg2.extras +from typing import Any, Generator, Optional, Tuple, Sequence -from ..api import IndexType, VectorDB, DBCaseConfig +import numpy as np +import psycopg +from pgvector.psycopg import register_vector +from psycopg import Connection, Cursor, sql + +from ..api import VectorDB +from .config import PgVectorConfigDict, PgVectorIndexConfig log = logging.getLogger(__name__) + class PgVector(VectorDB): - """ Use SQLAlchemy instructions""" + """Use psycopg instructions""" + + conn: psycopg.Connection[Any] | None = None + cursor: psycopg.Cursor[Any] | None = None + + # TODO add filters support + _unfiltered_search: sql.Composed + def __init__( self, dim: int, - db_config: dict, - db_case_config: DBCaseConfig, - collection_name: str = "PgVectorCollection", + db_config: PgVectorConfigDict, + db_case_config: PgVectorIndexConfig, + collection_name: str = "pg_vector_collection", drop_old: bool = False, **kwargs, ): @@ -29,43 +40,88 @@ def __init__( self.table_name = collection_name self.dim = dim - self._index_name = "pqvector_index" + self._index_name = "pgvector_index" self._primary_field = "id" self._vector_field = "embedding" # construct basic units - self.conn = psycopg2.connect(**self.db_config) - self.conn.autocommit = False - self.cursor = self.conn.cursor() + self.conn, self.cursor = self._create_connection(**self.db_config) # create vector extension - self.cursor.execute('CREATE EXTENSION IF NOT EXISTS vector') + self.cursor.execute("CREATE EXTENSION IF NOT EXISTS vector") self.conn.commit() - if drop_old : - log.info(f"Pgvector client drop table : {self.table_name}") + log.info(f"{self.name} config values: {self.db_config}\n{self.case_config}") + if not any( + ( + self.case_config.create_index_before_load, + self.case_config.create_index_after_load, + ) + ): + err = f"{self.name} config must create an index using create_index_before_load and/or create_index_after_load" + log.error(err) + raise RuntimeError( + f"{err}\n{pprint.pformat(self.db_config)}\n{pprint.pformat(self.case_config)}" + ) + + if drop_old: # self.pg_table.drop(pg_engine, checkfirst=True) self._drop_index() self._drop_table() self._create_table(dim) - self._create_index() + if self.case_config.create_index_before_load: + self._create_index() self.cursor.close() self.conn.close() self.cursor = None self.conn = None + @staticmethod + def _create_connection(**kwargs) -> Tuple[Connection, Cursor]: + conn = psycopg.connect(**kwargs) + register_vector(conn) + conn.autocommit = False + cursor = conn.cursor() + + assert conn is not None, "Connection is not initialized" + assert cursor is not None, "Cursor is not initialized" + + return conn, cursor + @contextmanager - def init(self) -> None: + def init(self) -> Generator[None, None, None]: """ Examples: >>> with self.init(): >>> self.insert_embeddings() >>> self.search_embedding() """ - self.conn = psycopg2.connect(**self.db_config) - self.conn.autocommit = False - self.cursor = self.conn.cursor() + + self.conn, self.cursor = self._create_connection(**self.db_config) + + # index configuration may have commands defined that we should set during each client session + session_options: Sequence[dict[str, Any]] = self.case_config.session_param()["session_options"] + + if len(session_options) > 0: + for setting in session_options: + command = sql.SQL("SET {setting_name} " + "= {val};").format( + setting_name=sql.Identifier(setting['parameter']['setting_name']), + val=sql.Identifier(str(setting['parameter']['val'])), + ) + log.debug(command.as_string(self.cursor)) + self.cursor.execute(command) + self.conn.commit() + + self._unfiltered_search = sql.Composed( + [ + sql.SQL("SELECT id FROM public.{} ORDER BY embedding ").format( + sql.Identifier(self.table_name) + ), + sql.SQL(self.case_config.search_param()["metric_fun_op"]), + sql.SQL(" %s::vector LIMIT %s::int"), + ] + ) try: yield @@ -78,54 +134,166 @@ def init(self) -> None: def _drop_table(self): assert self.conn is not None, "Connection is not initialized" assert self.cursor is not None, "Cursor is not initialized" + log.info(f"{self.name} client drop table : {self.table_name}") - self.cursor.execute(f'DROP TABLE IF EXISTS public."{self.table_name}"') + self.cursor.execute( + sql.SQL("DROP TABLE IF EXISTS public.{table_name}").format( + table_name=sql.Identifier(self.table_name) + ) + ) self.conn.commit() def ready_to_load(self): pass def optimize(self): - log.info(f"{self.name} optimizing") - self._drop_index() - self._create_index() + self._post_insert() - def ready_to_search(self): - pass + def _post_insert(self): + log.info(f"{self.name} post insert before optimize") + if self.case_config.create_index_after_load: + self._drop_index() + self._create_index() def _drop_index(self): assert self.conn is not None, "Connection is not initialized" assert self.cursor is not None, "Cursor is not initialized" + log.info(f"{self.name} client drop index : {self._index_name}") - self.cursor.execute(f'DROP INDEX IF EXISTS "{self._index_name}"') + drop_index_sql = sql.SQL("DROP INDEX IF EXISTS {index_name}").format( + index_name=sql.Identifier(self._index_name) + ) + log.debug(drop_index_sql.as_string(self.cursor)) + self.cursor.execute(drop_index_sql) self.conn.commit() + def _set_parallel_index_build_param(self): + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + index_param = self.case_config.index_param() + + if index_param["maintenance_work_mem"] is not None: + self.cursor.execute( + sql.SQL("SET maintenance_work_mem TO {};").format( + index_param["maintenance_work_mem"] + ) + ) + self.cursor.execute( + sql.SQL("ALTER USER {} SET maintenance_work_mem TO {};").format( + sql.Identifier(self.db_config["user"]), + index_param["maintenance_work_mem"], + ) + ) + self.conn.commit() + + if index_param["max_parallel_workers"] is not None: + self.cursor.execute( + sql.SQL("SET max_parallel_maintenance_workers TO '{}';").format( + index_param["max_parallel_workers"] + ) + ) + self.cursor.execute( + sql.SQL( + "ALTER USER {} SET max_parallel_maintenance_workers TO '{}';" + ).format( + sql.Identifier(self.db_config["user"]), + index_param["max_parallel_workers"], + ) + ) + self.cursor.execute( + sql.SQL("SET max_parallel_workers TO '{}';").format( + index_param["max_parallel_workers"] + ) + ) + self.cursor.execute( + sql.SQL( + "ALTER USER {} SET max_parallel_workers TO '{}';" + ).format( + sql.Identifier(self.db_config["user"]), + index_param["max_parallel_workers"], + ) + ) + self.cursor.execute( + sql.SQL( + "ALTER TABLE {} SET (parallel_workers = {});" + ).format( + sql.Identifier(self.table_name), + index_param["max_parallel_workers"], + ) + ) + self.conn.commit() + + results = self.cursor.execute( + sql.SQL("SHOW max_parallel_maintenance_workers;") + ).fetchall() + results.extend( + self.cursor.execute(sql.SQL("SHOW max_parallel_workers;")).fetchall() + ) + results.extend( + self.cursor.execute(sql.SQL("SHOW maintenance_work_mem;")).fetchall() + ) + log.info(f"{self.name} parallel index creation parameters: {results}") + def _create_index(self): assert self.conn is not None, "Connection is not initialized" assert self.cursor is not None, "Cursor is not initialized" + log.info(f"{self.name} client create index : {self._index_name}") index_param = self.case_config.index_param() - if self.case_config.index == IndexType.HNSW: - log.debug(f'Creating HNSW index. m={index_param["m"]}, ef_construction={index_param["ef_construction"]}') - self.cursor.execute(f'CREATE INDEX IF NOT EXISTS {self._index_name} ON public."{self.table_name}" USING hnsw (embedding {index_param["metric"]}) WITH (m={index_param["m"]}, ef_construction={index_param["ef_construction"]});') - elif self.case_config.index == IndexType.IVFFlat: - log.debug(f'Creating IVFFLAT index. list={index_param["lists"]}') - self.cursor.execute(f'CREATE INDEX IF NOT EXISTS {self._index_name} ON public."{self.table_name}" USING ivfflat (embedding {index_param["metric"]}) WITH (lists={index_param["lists"]});') + self._set_parallel_index_build_param() + options = [] + for option in index_param["index_creation_with_options"]: + if option['val'] is not None: + options.append( + sql.SQL("{option_name} = {val}").format( + option_name=sql.Identifier(option['option_name']), + val=sql.Identifier(str(option['val'])), + ) + ) + if any(options): + with_clause = sql.SQL("WITH ({});").format(sql.SQL(", ").join(options)) else: - assert "Invalid index type {self.case_config.index}" + with_clause = sql.Composed(()) + + index_create_sql = sql.SQL( + "CREATE INDEX IF NOT EXISTS {index_name} ON public.{table_name} USING {index_type} (embedding {embedding_metric})" + ).format( + index_name=sql.Identifier(self._index_name), + table_name=sql.Identifier(self.table_name), + index_type=sql.Identifier(index_param["index_type"]), + embedding_metric=sql.Identifier(index_param["metric"]), + ) + index_create_sql_with_with_clause = ( + index_create_sql + with_clause + ).join(" ") + log.debug(index_create_sql_with_with_clause.as_string(self.cursor)) + self.cursor.execute(index_create_sql_with_with_clause) self.conn.commit() - def _create_table(self, dim : int): + def _create_table(self, dim: int): assert self.conn is not None, "Connection is not initialized" assert self.cursor is not None, "Cursor is not initialized" try: + log.info(f"{self.name} client create table : {self.table_name}") + # create table - self.cursor.execute(f'CREATE TABLE IF NOT EXISTS public."{self.table_name}" (id BIGINT PRIMARY KEY, embedding vector({dim}));') - self.cursor.execute(f'ALTER TABLE public."{self.table_name}" ALTER COLUMN embedding SET STORAGE PLAIN;') + self.cursor.execute( + sql.SQL( + "CREATE TABLE IF NOT EXISTS public.{table_name} (id BIGINT PRIMARY KEY, embedding vector({dim}));" + ).format(table_name=sql.Identifier(self.table_name), dim=dim) + ) + self.cursor.execute( + sql.SQL( + "ALTER TABLE public.{table_name} ALTER COLUMN embedding SET STORAGE PLAIN;" + ).format(table_name=sql.Identifier(self.table_name)) + ) self.conn.commit() except Exception as e: - log.warning(f"Failed to create pgvector table: {self.table_name} error: {e}") + log.warning( + f"Failed to create pgvector table: {self.table_name} error: {e}" + ) raise e from None def insert_embeddings( @@ -133,25 +301,32 @@ def insert_embeddings( embeddings: list[list[float]], metadata: list[int], **kwargs: Any, - ) -> (int, Exception): + ) -> Tuple[int, Optional[Exception]]: assert self.conn is not None, "Connection is not initialized" assert self.cursor is not None, "Cursor is not initialized" try: - items = { - "id": metadata, - "embedding": embeddings - } - df = pd.DataFrame(items) - csv_buffer = io.StringIO() - df.to_csv(csv_buffer, index=False, header=False) - csv_buffer.seek(0) - self.cursor.copy_expert(f"COPY public.\"{self.table_name}\" FROM STDIN WITH (FORMAT CSV)", csv_buffer) + metadata_arr = np.array(metadata) + embeddings_arr = np.array(embeddings) + + with self.cursor.copy( + sql.SQL("COPY public.{table_name} FROM STDIN (FORMAT BINARY)").format( + table_name=sql.Identifier(self.table_name) + ) + ) as copy: + copy.set_types(["bigint", "vector"]) + for i, row in enumerate(metadata_arr): + copy.write_row((row, embeddings_arr[i])) self.conn.commit() + if kwargs.get("last_batch"): + self._post_insert() + return len(metadata), None except Exception as e: - log.warning(f"Failed to insert data into pgvector table ({self.table_name}), error: {e}") + log.warning( + f"Failed to insert data into pgvector table ({self.table_name}), error: {e}" + ) return 0, e def search_embedding( @@ -164,17 +339,9 @@ def search_embedding( assert self.conn is not None, "Connection is not initialized" assert self.cursor is not None, "Cursor is not initialized" - search_param =self.case_config.search_param() - - if self.case_config.index == IndexType.HNSW: - self.cursor.execute(f'SET hnsw.ef_search = {search_param["ef"]}') - self.cursor.execute(f"SELECT id FROM public.\"{self.table_name}\" ORDER BY embedding {search_param['metric_fun_op']} '{query}' LIMIT {k};") - elif self.case_config.index == IndexType.IVFFlat: - self.cursor.execute(f'SET ivfflat.probes = {search_param["probes"]}') - self.cursor.execute(f"SELECT id FROM public.\"{self.table_name}\" ORDER BY embedding {search_param['metric_fun_op']} '{query}' LIMIT {k};") - else: - assert "Invalid index type {self.case_config.index}" - self.conn.commit() - result = self.cursor.fetchall() + # TODO add filters support + result = self.cursor.execute( + self._unfiltered_search, (query, k), prepare=True, binary=True + ) - return [int(i[0]) for i in result] + return [int(i[0]) for i in result.fetchall()] diff --git a/vectordb_bench/frontend/components/run_test/caseSelector.py b/vectordb_bench/frontend/components/run_test/caseSelector.py index 9af023518..49b839163 100644 --- a/vectordb_bench/frontend/components/run_test/caseSelector.py +++ b/vectordb_bench/frontend/components/run_test/caseSelector.py @@ -65,25 +65,28 @@ def caseConfigSetting(st, allCaseConfigs, case, activedDbList): key = "%s-%s-%s" % (db, case, config.label.value) if config.inputType == InputType.Text: caseConfig[config.label] = column.text_input( - config.label.value, + config.displayLabel if config.displayLabel else config.label.value, key=key, + help=config.inputHelp, value=config.inputConfig["value"], ) elif config.inputType == InputType.Option: caseConfig[config.label] = column.selectbox( - config.label.value, + config.displayLabel if config.displayLabel else config.label.value, config.inputConfig["options"], key=key, + help=config.inputHelp, ) elif config.inputType == InputType.Number: caseConfig[config.label] = column.number_input( - config.label.value, + config.displayLabel if config.displayLabel else config.label.value, # format="%d", step=config.inputConfig.get("step", 1), min_value=config.inputConfig["min"], max_value=config.inputConfig["max"], key=key, value=config.inputConfig["value"], + help=config.inputHelp, ) k += 1 if k == 0: diff --git a/vectordb_bench/frontend/const/dbCaseConfigs.py b/vectordb_bench/frontend/const/dbCaseConfigs.py index 9b122910a..1e69c57aa 100644 --- a/vectordb_bench/frontend/const/dbCaseConfigs.py +++ b/vectordb_bench/frontend/const/dbCaseConfigs.py @@ -49,6 +49,8 @@ class CaseConfigInput(BaseModel): label: CaseConfigParamType inputType: InputType = InputType.Text inputConfig: dict = {} + inputHelp: str = "" + displayLabel: str = "" # todo type should be a function isDisplayed: typing.Any = lambda x: True @@ -71,6 +73,18 @@ class CaseConfigInput(BaseModel): }, ) +CaseConfigParamInput_IndexType_PgVector = CaseConfigInput( + label=CaseConfigParamType.IndexType, + inputHelp="Select Index Type", + inputType=InputType.Option, + inputConfig={ + "options": [ + IndexType.HNSW.value, + IndexType.IVFFlat.value, + ], + }, +) + CaseConfigParamInput_M = CaseConfigInput( label=CaseConfigParamType.M, inputType=InputType.Number, @@ -83,6 +97,19 @@ class CaseConfigInput(BaseModel): == IndexType.HNSW.value, ) +CaseConfigParamInput_m = CaseConfigInput( + label=CaseConfigParamType.m, + inputType=InputType.Number, + inputConfig={ + "min": 4, + "max": 64, + "value": 16, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) + == IndexType.HNSW.value, +) + + CaseConfigParamInput_EFConstruction_Milvus = CaseConfigInput( label=CaseConfigParamType.EFConstruction, inputType=InputType.Number, @@ -115,6 +142,30 @@ class CaseConfigInput(BaseModel): }, ) +CaseConfigParamInput_maintenance_work_mem_PgVector = CaseConfigInput( + label=CaseConfigParamType.maintenance_work_mem, + inputHelp="Recommended value: 1.33x the index size, not to exceed the available free memory." + "Specify in gigabytes. e.g. 8GB", + inputType=InputType.Text, + inputConfig={ + "value": "8GB", + }, +) + +CaseConfigParamInput_max_parallel_workers_PgVector = CaseConfigInput( + label=CaseConfigParamType.max_parallel_workers, + displayLabel="Max parallel workers", + inputHelp="Recommended value: (cpu cores - 1). This will set the parameters: max_parallel_maintenance_workers," + " max_parallel_workers & table(parallel_workers)", + inputType=InputType.Number, + inputConfig={ + "min": 0, + "max": 1024, + "value": 16, + }, +) + + CaseConfigParamInput_EFConstruction_PgVectoRS = CaseConfigInput( label=CaseConfigParamType.EFConstruction, inputType=InputType.Number, @@ -127,6 +178,19 @@ class CaseConfigInput(BaseModel): == IndexType.HNSW.value, ) +CaseConfigParamInput_EFConstruction_PgVector = CaseConfigInput( + label=CaseConfigParamType.ef_construction, + inputType=InputType.Number, + inputConfig={ + "min": 8, + "max": 1024, + "value": 256, + }, + isDisplayed=lambda config: config[CaseConfigParamType.IndexType] + == IndexType.HNSW.value, +) + + CaseConfigParamInput_M_ES = CaseConfigInput( label=CaseConfigParamType.M, inputType=InputType.Number, @@ -371,24 +435,13 @@ class CaseConfigInput(BaseModel): ], ) -CaseConfigParamInput_IndexType_PG = CaseConfigInput( - label=CaseConfigParamType.IndexType, - inputType=InputType.Option, - inputConfig={ - "options": [ - IndexType.HNSW.value, - IndexType.IVFFlat.value, - ], - }, -) - CaseConfigParamInput_Lists = CaseConfigInput( label=CaseConfigParamType.lists, inputType=InputType.Number, inputConfig={ "min": 1, "max": 65536, - "value": 1000, + "value": 10, }, isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) in [IndexType.IVFFlat.value], @@ -397,37 +450,47 @@ class CaseConfigInput(BaseModel): CaseConfigParamInput_Probes = CaseConfigInput( label=CaseConfigParamType.probes, inputType=InputType.Number, + inputConfig={ + "min": 1, + "max": 65536, + "value": 1, + }, +) + +CaseConfigParamInput_Lists_PgVector = CaseConfigInput( + label=CaseConfigParamType.lists, + inputType=InputType.Number, inputConfig={ "min": 1, "max": 65536, "value": 10, }, isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) - in [IndexType.IVFFlat.value], + == IndexType.IVFFlat.value, ) -CaseConfigParamInput_EF_PG = CaseConfigInput( - label=CaseConfigParamType.EF, +CaseConfigParamInput_Probes_PgVector = CaseConfigInput( + label=CaseConfigParamType.probes, inputType=InputType.Number, inputConfig={ "min": 1, "max": 65536, - "value": 128, + "value": 1, }, isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) - in [IndexType.HNSW.value], + == IndexType.IVFFlat.value, ) -CaseConfigParamInput_EFC_PG = CaseConfigInput( - label=CaseConfigParamType.EFConstruction, +CaseConfigParamInput_EFSearch_PgVector = CaseConfigInput( + label=CaseConfigParamType.ef_search, inputType=InputType.Number, inputConfig={ "min": 1, - "max": 65536, - "value": 300, + "max": 2048, + "value": 256, }, isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) - in [IndexType.HNSW.value], + == IndexType.HNSW.value, ) CaseConfigParamInput_QuantizationType_PgVectoRS = CaseConfigInput( @@ -518,20 +581,22 @@ class CaseConfigInput(BaseModel): CaseConfigParamInput_NumCandidates_ES, ] -PgVectorLoadingConfig = [ - CaseConfigParamInput_IndexType_PG, - CaseConfigParamInput_Lists, - CaseConfigParamInput_M, - CaseConfigParamInput_EFC_PG -] -PgVectorPerformanceConfig = [ - CaseConfigParamInput_IndexType_PG, - CaseConfigParamInput_Lists, - CaseConfigParamInput_Probes, - CaseConfigParamInput_M, - CaseConfigParamInput_EF_PG, - CaseConfigParamInput_EFC_PG -] +PgVectorLoadingConfig = [CaseConfigParamInput_IndexType_PgVector, + CaseConfigParamInput_Lists_PgVector, + CaseConfigParamInput_m, + CaseConfigParamInput_EFConstruction_PgVector, + CaseConfigParamInput_maintenance_work_mem_PgVector, + CaseConfigParamInput_max_parallel_workers_PgVector, + ] +PgVectorPerformanceConfig = [CaseConfigParamInput_IndexType_PgVector, + CaseConfigParamInput_m, + CaseConfigParamInput_EFConstruction_PgVector, + CaseConfigParamInput_EFSearch_PgVector, + CaseConfigParamInput_Lists_PgVector, + CaseConfigParamInput_Probes_PgVector, + CaseConfigParamInput_maintenance_work_mem_PgVector, + CaseConfigParamInput_max_parallel_workers_PgVector, + ] PgVectoRSLoadingConfig = [ CaseConfigParamInput_IndexType, diff --git a/vectordb_bench/models.py b/vectordb_bench/models.py index 3c2a5b9aa..ec1b610e1 100644 --- a/vectordb_bench/models.py +++ b/vectordb_bench/models.py @@ -1,8 +1,8 @@ import logging import pathlib from datetime import date -from typing import Self -from enum import Enum +from enum import Enum, StrEnum, auto +from typing import List, Self, Sequence, Set import ujson @@ -37,8 +37,10 @@ class CaseConfigParamType(Enum): IndexType = "IndexType" M = "M" EFConstruction = "efConstruction" + ef_construction = "ef_construction" EF = "ef" SearchList = "search_list" + ef_search = "ef_search" Nlist = "nlist" Nprobe = "nprobe" MaxConnections = "maxConnections" @@ -60,7 +62,8 @@ class CaseConfigParamType(Enum): cache_dataset_on_device = "cache_dataset_on_device" refine_ratio = "refine_ratio" level = "level" - + maintenance_work_mem = "maintenance_work_mem" + max_parallel_workers = "max_parallel_workers" class CustomizedCase(BaseModel): pass From bb09b519ea04a0e8d2491eede8f5374e97c74fc6 Mon Sep 17 00:00:00 2001 From: yangxuan Date: Wed, 15 May 2024 19:07:17 +0800 Subject: [PATCH 064/327] fix: Skip streamlit 1.34.0 See also: #315, streamlit/streamlit#8606 Signed-off-by: yangxuan --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e048a19cb..8267f8522 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ classifiers = [ dependencies = [ "pytz", "streamlit-autorefresh", - "streamlit==1.31.1", + "streamlit!=1.34.0", "streamlit_extras", "tqdm", "s3fs", From e3f55d2cc22757a2d998bfd2ebc8ea818fc69713 Mon Sep 17 00:00:00 2001 From: "min.tian" Date: Fri, 24 May 2024 10:07:23 +0800 Subject: [PATCH 065/327] fix bugs: should normalize cosine dataset when test with milvus gpu_index Signed-off-by: min.tian --- vectordb_bench/backend/clients/milvus/config.py | 8 ++++++-- vectordb_bench/backend/clients/milvus/milvus.py | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/vectordb_bench/backend/clients/milvus/config.py b/vectordb_bench/backend/clients/milvus/config.py index 0e93e82b8..eea7b11f5 100644 --- a/vectordb_bench/backend/clients/milvus/config.py +++ b/vectordb_bench/backend/clients/milvus/config.py @@ -14,13 +14,17 @@ class MilvusIndexConfig(BaseModel): index: IndexType metric_type: MetricType | None = None + + @property + def is_gpu_index(self) -> bool: + return self.index in [IndexType.GPU_CAGRA, IndexType.GPU_IVF_FLAT, IndexType.GPU_IVF_PQ] def parse_metric(self) -> str: if not self.metric_type: return "" - # if self.metric_type == MetricType.COSINE: - # return MetricType.L2.value + if self.is_gpu_index and self.metric_type == MetricType.COSINE: + return MetricType.L2.value return self.metric_type.value diff --git a/vectordb_bench/backend/clients/milvus/milvus.py b/vectordb_bench/backend/clients/milvus/milvus.py index 58334efe9..2436e2680 100644 --- a/vectordb_bench/backend/clients/milvus/milvus.py +++ b/vectordb_bench/backend/clients/milvus/milvus.py @@ -8,7 +8,7 @@ from pymilvus import Collection, utility from pymilvus import CollectionSchema, DataType, FieldSchema, MilvusException -from ..api import VectorDB, IndexType +from ..api import VectorDB from .config import MilvusIndexConfig @@ -119,7 +119,7 @@ def wait_index(): wait_index() # Skip compaction if use GPU indexType - if self.case_config.index in [IndexType.GPU_CAGRA, IndexType.GPU_IVF_FLAT, IndexType.GPU_IVF_PQ]: + if self.case_config.is_gpu_index: log.debug("skip compaction for gpu index type.") else : self.col.compact() @@ -157,6 +157,10 @@ def optimize(self): def need_normalize_cosine(self) -> bool: """Wheather this database need to normalize dataset to support COSINE""" + if self.case_config.is_gpu_index: + log.info(f"current gpu_index only supports IP / L2, cosine dataset need normalize.") + return True + return False def insert_embeddings( From e2679faff4fdd9e8ff5d812e1e796b497c0ee923 Mon Sep 17 00:00:00 2001 From: "min.tian" Date: Mon, 3 Jun 2024 15:01:09 +0800 Subject: [PATCH 066/327] Increase the optimization time limit to ensure that most databases pass the test Signed-off-by: min.tian --- vectordb_bench/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/vectordb_bench/__init__.py b/vectordb_bench/__init__.py index eca190832..f3cf3d103 100644 --- a/vectordb_bench/__init__.py +++ b/vectordb_bench/__init__.py @@ -32,10 +32,10 @@ class config: LOAD_TIMEOUT_1536D_500K = 2.5 * 3600 # 2.5h LOAD_TIMEOUT_1536D_5M = 25 * 3600 # 25h - OPTIMIZE_TIMEOUT_DEFAULT = 15 * 60 # 15min - OPTIMIZE_TIMEOUT_768D_1M = 15 * 60 # 15min - OPTIMIZE_TIMEOUT_768D_10M = 2.5 * 3600 # 2.5h - OPTIMIZE_TIMEOUT_768D_100M = 25 * 3600 # 1.04d + OPTIMIZE_TIMEOUT_DEFAULT = 30 * 60 # 30min + OPTIMIZE_TIMEOUT_768D_1M = 30 * 60 # 30min + OPTIMIZE_TIMEOUT_768D_10M = 5 * 3600 # 5h + OPTIMIZE_TIMEOUT_768D_100M = 50 * 3600 # 50h OPTIMIZE_TIMEOUT_1536D_500K = 15 * 60 # 15min From fbb11497b2d5bbca4fbf5574b57840158b376515 Mon Sep 17 00:00:00 2001 From: "min.tian" Date: Fri, 31 May 2024 09:20:26 +0800 Subject: [PATCH 067/327] the dataset_local_dir should be kept consistent Signed-off-by: min.tian --- .env.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.example b/.env.example index 0fb4f0d62..618cac556 100644 --- a/.env.example +++ b/.env.example @@ -6,6 +6,6 @@ # NUM_PER_BATCH= # DEFAULT_DATASET_URL= -DATASET_LOCAL_DIR="/tmp/vector_db_bench/dataset" +DATASET_LOCAL_DIR="/tmp/vectordb_bench/dataset" # DROP_OLD = True From 2759cadb442b6404b6dd2ae021f5b25d3b444550 Mon Sep 17 00:00:00 2001 From: Mark Greenhalgh Date: Thu, 30 May 2024 20:03:19 +0000 Subject: [PATCH 068/327] Introduce a command line interface. --- README.md | 187 +++++++++ pyproject.toml | 9 +- vectordb_bench/__init__.py | 23 +- vectordb_bench/backend/cases.py | 44 ++- vectordb_bench/backend/clients/__init__.py | 1 + vectordb_bench/backend/clients/api.py | 2 +- vectordb_bench/backend/clients/milvus/cli.py | 291 ++++++++++++++ .../backend/clients/milvus/milvus.py | 19 +- .../backend/clients/pgvector/cli.py | 116 ++++++ .../backend/clients/pgvector/config.py | 2 +- .../backend/clients/pgvector/pgvector.py | 8 +- vectordb_bench/backend/clients/redis/cli.py | 74 ++++ vectordb_bench/backend/clients/test/cli.py | 25 ++ vectordb_bench/backend/clients/test/config.py | 18 + vectordb_bench/backend/clients/test/test.py | 62 +++ .../backend/clients/weaviate_cloud/cli.py | 41 ++ .../backend/clients/zilliz_cloud/cli.py | 55 +++ vectordb_bench/backend/task_runner.py | 98 +++-- vectordb_bench/cli/__init__.py | 0 vectordb_bench/cli/cli.py | 362 ++++++++++++++++++ vectordb_bench/cli/vectordbbench.py | 20 + vectordb_bench/config-files/sample_config.yml | 17 + .../components/run_test/dbSelector.py | 8 +- .../components/run_test/submitTask.py | 16 +- .../frontend/const/dbCaseConfigs.py | 3 +- vectordb_bench/interface.py | 46 +-- vectordb_bench/models.py | 50 ++- 27 files changed, 1505 insertions(+), 92 deletions(-) create mode 100644 vectordb_bench/backend/clients/milvus/cli.py create mode 100644 vectordb_bench/backend/clients/pgvector/cli.py create mode 100644 vectordb_bench/backend/clients/redis/cli.py create mode 100644 vectordb_bench/backend/clients/test/cli.py create mode 100644 vectordb_bench/backend/clients/test/config.py create mode 100644 vectordb_bench/backend/clients/test/test.py create mode 100644 vectordb_bench/backend/clients/weaviate_cloud/cli.py create mode 100644 vectordb_bench/backend/clients/zilliz_cloud/cli.py create mode 100644 vectordb_bench/cli/__init__.py create mode 100644 vectordb_bench/cli/cli.py create mode 100644 vectordb_bench/cli/vectordbbench.py create mode 100644 vectordb_bench/config-files/sample_config.yml diff --git a/README.md b/README.md index 4ff2f9121..eeda249ae 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,115 @@ All the database client supported ``` shell init_bench ``` + +OR: + +### Run from the command line. + +``` shell +vectordbbench [OPTIONS] COMMAND [ARGS]... +``` +To list the clients that are runnable via the commandline option, execute: `vectordbbench --help` +``` text +$ vectordbbench --help +Usage: vectordbbench [OPTIONS] COMMAND [ARGS]... + +Options: + --help Show this message and exit. + +Commands: + pgvectorhnsw + pgvectorivfflat + test + weaviate +``` +To list the options for each command, execute `vectordbbench [command] --help` + +```text +$ vectordbbench pgvectorhnsw --help +Usage: vectordbbench pgvectorhnsw [OPTIONS] + +Options: + --config-file PATH Read configuration from yaml file + --drop-old / --skip-drop-old Drop old or skip [default: drop-old] + --load / --skip-load Load or skip [default: load] + --search-serial / --skip-search-serial + Search serial or skip [default: search- + serial] + --search-concurrent / --skip-search-concurrent + Search concurrent or skip [default: search- + concurrent] + --case-type [CapacityDim128|CapacityDim960|Performance768D100M|Performance768D10M|Performance768D1M|Performance768D10M1P|Performance768D1M1P|Performance768D10M99P|Performance768D1M99P|Performance1536D500K|Performance1536D5M|Performance1536D500K1P|Performance1536D5M1P|Performance1536D500K99P|Performance1536D5M99P|Performance1536D50K] + Case type + --db-label TEXT Db label, default: date in ISO format + [default: 2024-05-20T20:26:31.113290] + --dry-run Print just the configuration and exit + without running the tasks + --k INTEGER K value for number of nearest neighbors to + search [default: 100] + --concurrency-duration INTEGER Adjusts the duration in seconds of each + concurrency search [default: 30] + --num-concurrency TEXT Comma-separated list of concurrency values + to test during concurrent search [default: + 1,10,20] + --user-name TEXT Db username [required] + --password TEXT Db password [required] + --host TEXT Db host [required] + --db-name TEXT Db name [required] + --maintenance-work-mem TEXT Sets the maximum memory to be used for + maintenance operations (index creation). Can + be entered as string with unit like '64GB' + or as an integer number of KB.This will set + the parameters: + max_parallel_maintenance_workers, + max_parallel_workers & + table(parallel_workers) + --max-parallel-workers INTEGER Sets the maximum number of parallel + processes per maintenance operation (index + creation) + --m INTEGER hnsw m + --ef-construction INTEGER hnsw ef-construction + --ef-search INTEGER hnsw ef-search + --help Show this message and exit. +``` +#### Using a configuration file. + +The vectordbbench command can optionally read some or all the options from a yaml formatted configuration file. + +By default, configuration files are expected to be in vectordb_bench/config-files/, this can be overridden by setting +the environment variable CONFIG_LOCAL_DIR or by passing the full path to the file. + +The required format is: +```yaml +commandname: + parameter_name: parameter_value + parameter_name: parameter_value +``` +Example: +```yaml +pgvectorhnsw: + db_label: pgConfigTest + user_name: vectordbbench + password: vectordbbench + db_name: vectordbbench + host: localhost + m: 16 + ef_construction: 128 + ef_search: 128 +milvushnsw: + skip_search_serial: True + case_type: Performance1536D50K + uri: http://localhost:19530 + m: 16 + ef_construction: 128 + ef_search: 128 + drop_old: False + load: False +``` +> Notes: +> - Options passed on the command line will override the configuration file* +> - Parameter names use an _ not - + ## What is VectorDBBench VectorDBBench is not just an offering of benchmark results for mainstream vector databases and cloud services, it's your go-to tool for the ultimate performance and cost-effectiveness comparison. Designed with ease-of-use in mind, VectorDBBench is devised to help users, even non-professionals, reproduce results or test new systems, making the hunt for the optimal choice amongst a plethora of cloud services and open-source vector databases a breeze. @@ -220,6 +329,7 @@ class NewDBCaseConfig(DBCaseConfig): # Implement optional case-specific configuration fields # ... ``` + **Step 3: Importing the DB Client and Updating Initialization** In this final step, you will import your DB client into clients/__init__.py and update the initialization process. @@ -258,6 +368,83 @@ class DB(Enum): return NewClientCaseConfig ``` +**Step 4: Implement new_client/cli.py and vectordb_bench/cli/vectordbbench.py** + +In this (optional, but encouraged) step you will enable the test to be run from the command line. +1. Navigate to the vectordb_bench/backend/clients/"client" directory. +2. Inside the "client" folder, create a cli.py file. +Using zilliz as an example cli.py: +```python +from typing import Annotated, Unpack + +import click +import os +from pydantic import SecretStr + +from vectordb_bench.cli.cli import ( + CommonTypedDict, + cli, + click_parameter_decorators_from_typed_dict, + run, +) +from vectordb_bench.backend.clients import DB + + +class ZillizTypedDict(CommonTypedDict): + uri: Annotated[ + str, click.option("--uri", type=str, help="uri connection string", required=True) + ] + user_name: Annotated[ + str, click.option("--user-name", type=str, help="Db username", required=True) + ] + password: Annotated[ + str, + click.option("--password", + type=str, + help="Zilliz password", + default=lambda: os.environ.get("ZILLIZ_PASSWORD", ""), + show_default="$ZILLIZ_PASSWORD", + ), + ] + level: Annotated[ + str, + click.option("--level", type=str, help="Zilliz index level", required=False), + ] + + +@cli.command() +@click_parameter_decorators_from_typed_dict(ZillizTypedDict) +def ZillizAutoIndex(**parameters: Unpack[ZillizTypedDict]): + from .config import ZillizCloudConfig, AutoIndexConfig + + run( + db=DB.ZillizCloud, + db_config=ZillizCloudConfig( + db_label=parameters["db_label"], + uri=SecretStr(parameters["uri"]), + user=parameters["user_name"], + password=SecretStr(parameters["password"]), + ), + db_case_config=AutoIndexConfig( + params={parameters["level"]}, + ), + **parameters, + ) +``` +3. Update cli by adding: + 1. Add database specific options as an Annotated TypedDict, see ZillizTypedDict above. + 2. Add index configuration specific options as an Annotated TypedDict. (example: vectordb_bench/backend/clients/pgvector/cli.py) + 1. May not be needed if there is only one index config. + 2. Repeat for each index configuration, nesting them if possible. + 2. Add a index config specific function for each index type, see Zilliz above. The function name, in lowercase, will be the command name passed to the vectordbbench command. + 3. Update db_config and db_case_config to match client requirements + 4. Continue to add new functions for each index config. + 5. Import the client cli module and command to vectordb_bench/cli/vectordbbench.py (for databases with multiple commands (index configs), this only needs to be done for one command) + +> cli modules with multiple index configs: +> - pgvector: vectordb_bench/backend/clients/pgvector/cli.py +> - milvus: vectordb_bench/backend/clients/milvus/cli.py + That's it! You have successfully added a new DB client to the vectordb_bench project. ## Rules diff --git a/pyproject.toml b/pyproject.toml index 8267f8522..1311cceef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta" [tool.setuptools.packages.find] where = ["."] -include = ["vectordb_bench"] +include = ["vectordb_bench", "vectordb_bench.cli"] [project] name = "vectordb-bench" @@ -24,6 +24,7 @@ classifiers = [ "Operating System :: OS Independent", ] dependencies = [ + "click", "pytz", "streamlit-autorefresh", "streamlit!=1.34.0", @@ -60,21 +61,23 @@ all = [ "chromadb", "psycopg2", "psycopg", + "psycopg-binary", ] qdrant = [ "qdrant-client" ] pinecone = [ "pinecone-client" ] weaviate = [ "weaviate-client" ] elastic = [ "elasticsearch" ] -pgvector = [ "pgvector", "psycopg" ] +pgvector = [ "psycopg", "psycopg-binary", "pgvector" ] pgvecto_rs = [ "psycopg2" ] redis = [ "redis" ] chromadb = [ "chromadb" ] +zilliz_cloud = [] [project.urls] "repository" = "https://github.com/zilliztech/VectorDBBench" [project.scripts] init_bench = "vectordb_bench.__main__:main" - +vectordbbench = "vectordb_bench.cli.vectordbbench:cli" [tool.setuptools_scm] diff --git a/vectordb_bench/__init__.py b/vectordb_bench/__init__.py index f3cf3d103..018cd7732 100644 --- a/vectordb_bench/__init__.py +++ b/vectordb_bench/__init__.py @@ -1,11 +1,13 @@ -import environs import inspect import pathlib -from . import log_util +import environs + +from . import log_util env = environs.Env() -env.read_env(".env") +env.read_env(".env", False) + class config: ALIYUN_OSS_URL = "assets.zilliz.com.cn/benchmark/" @@ -19,9 +21,20 @@ class config: DROP_OLD = env.bool("DROP_OLD", True) USE_SHUFFLED_DATA = env.bool("USE_SHUFFLED_DATA", True) - NUM_CONCURRENCY = [1, 5, 10, 15, 20, 25, 30, 35] - RESULTS_LOCAL_DIR = pathlib.Path(__file__).parent.joinpath("results") + NUM_CONCURRENCY = env.list("NUM_CONCURRENCY", [1, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75, 80, 85, 90, 95, 100], subcast=int ) + + CONCURRENCY_DURATION = 30 + + RESULTS_LOCAL_DIR = env.path( + "RESULTS_LOCAL_DIR", pathlib.Path(__file__).parent.joinpath("results") + ) + CONFIG_LOCAL_DIR = env.path( + "CONFIG_LOCAL_DIR", pathlib.Path(__file__).parent.joinpath("config-files") + ) + + + K_DEFAULT = 100 # default return top k nearest neighbors during search CAPACITY_TIMEOUT_IN_SECONDS = 24 * 3600 # 24h LOAD_TIMEOUT_DEFAULT = 2.5 * 3600 # 2.5h diff --git a/vectordb_bench/backend/cases.py b/vectordb_bench/backend/cases.py index 7f40ae4f8..6f4b35974 100644 --- a/vectordb_bench/backend/cases.py +++ b/vectordb_bench/backend/cases.py @@ -1,6 +1,7 @@ import typing import logging from enum import Enum, auto +from typing import Type from vectordb_bench import config from vectordb_bench.base import BaseModel @@ -10,8 +11,6 @@ log = logging.getLogger(__name__) -Case = typing.TypeVar("Case") - class CaseType(Enum): """ @@ -42,11 +41,15 @@ class CaseType(Enum): Performance1536D500K99P = 14 Performance1536D5M99P = 15 + Performance1536D50K = 50 + Custom = 100 @property - def case_cls(self, custom_configs: dict | None = None) -> Case: - return type2case.get(self) + def case_cls(self, custom_configs: dict | None = None) -> Type["Case"]: + if self not in type2case: + raise NotImplementedError(f"Case {self} has not implemented. You can add it manually to vectordb_bench.backend.cases.type2case or define a custom_configs['custom_cls']") + return type2case[self] @property def case_name(self) -> str: @@ -69,7 +72,7 @@ class CaseLabel(Enum): class Case(BaseModel): - """Undifined case + """Undefined case Fields: case_id(CaseType): default 9 case type plus one custom cases. @@ -86,9 +89,9 @@ class Case(BaseModel): dataset: DatasetManager load_timeout: float | int - optimize_timeout: float | int | None + optimize_timeout: float | int | None = None - filter_rate: float | None + filter_rate: float | None = None @property def filters(self) -> dict | None: @@ -115,20 +118,23 @@ class PerformanceCase(Case, BaseModel): load_timeout: float | int = config.LOAD_TIMEOUT_DEFAULT optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_DEFAULT + class CapacityDim960(CapacityCase): case_id: CaseType = CaseType.CapacityDim960 dataset: DatasetManager = Dataset.GIST.manager(100_000) name: str = "Capacity Test (960 Dim Repeated)" - description: str = """This case tests the vector database's loading capacity by repeatedly inserting large-dimension vectors (GIST 100K vectors, 960 dimensions) until it is fully loaded. -Number of inserted vectors will be reported.""" + description: str = """This case tests the vector database's loading capacity by repeatedly inserting large-dimension + vectors (GIST 100K vectors, 960 dimensions) until it is fully loaded. Number of inserted vectors will be + reported.""" class CapacityDim128(CapacityCase): case_id: CaseType = CaseType.CapacityDim128 dataset: DatasetManager = Dataset.SIFT.manager(500_000) name: str = "Capacity Test (128 Dim Repeated)" - description: str = """This case tests the vector database's loading capacity by repeatedly inserting small-dimension vectors (SIFT 100K vectors, 128 dimensions) until it is fully loaded. -Number of inserted vectors will be reported.""" + description: str = """This case tests the vector database's loading capacity by repeatedly inserting small-dimension + vectors (SIFT 100K vectors, 128 dimensions) until it is fully loaded. Number of inserted vectors will be + reported.""" class Performance768D10M(PerformanceCase): @@ -238,6 +244,7 @@ class Performance1536D500K1P(PerformanceCase): load_timeout: float | int = config.LOAD_TIMEOUT_1536D_500K optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_1536D_500K + class Performance1536D5M1P(PerformanceCase): case_id: CaseType = CaseType.Performance1536D5M1P filter_rate: float | int | None = 0.01 @@ -248,6 +255,7 @@ class Performance1536D5M1P(PerformanceCase): load_timeout: float | int = config.LOAD_TIMEOUT_1536D_5M optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_1536D_5M + class Performance1536D500K99P(PerformanceCase): case_id: CaseType = CaseType.Performance1536D500K99P filter_rate: float | int | None = 0.99 @@ -258,6 +266,7 @@ class Performance1536D500K99P(PerformanceCase): load_timeout: float | int = config.LOAD_TIMEOUT_1536D_500K optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_1536D_500K + class Performance1536D5M99P(PerformanceCase): case_id: CaseType = CaseType.Performance1536D5M99P filter_rate: float | int | None = 0.99 @@ -269,6 +278,17 @@ class Performance1536D5M99P(PerformanceCase): optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_1536D_5M +class Performance1536D50K(PerformanceCase): + case_id: CaseType = CaseType.Performance1536D50K + filter_rate: float | int | None = None + dataset: DatasetManager = Dataset.OPENAI.manager(50_000) + name: str = "Search Performance Test (50K Dataset, 1536 Dim)" + description: str = """This case tests the search performance of a vector database with a medium 50K dataset (OpenAI 50K vectors, 1536 dimensions), at varying parallel levels. +Results will show index building time, recall, and maximum QPS.""" + load_timeout: float | int = 3600 + optimize_timeout: float | int | None = 15 * 60 + + type2case = { CaseType.CapacityDim960: CapacityDim960, CaseType.CapacityDim128: CapacityDim128, @@ -290,5 +310,5 @@ class Performance1536D5M99P(PerformanceCase): CaseType.Performance1536D500K99P: Performance1536D500K99P, CaseType.Performance1536D5M99P: Performance1536D5M99P, - + CaseType.Performance1536D50K: Performance1536D50K, } diff --git a/vectordb_bench/backend/clients/__init__.py b/vectordb_bench/backend/clients/__init__.py index aa6641a93..6a99661c2 100644 --- a/vectordb_bench/backend/clients/__init__.py +++ b/vectordb_bench/backend/clients/__init__.py @@ -32,6 +32,7 @@ class DB(Enum): PgVectoRS = "PgVectoRS" Redis = "Redis" Chroma = "Chroma" + Test = "test" @property diff --git a/vectordb_bench/backend/clients/api.py b/vectordb_bench/backend/clients/api.py index d1325322a..0024bf600 100644 --- a/vectordb_bench/backend/clients/api.py +++ b/vectordb_bench/backend/clients/api.py @@ -47,7 +47,7 @@ def to_dict(self) -> dict: def not_empty_field(cls, v, field): if field.name == "db_label": return v - if isinstance(v, (str, SecretStr)) and len(v) == 0: + if not v and isinstance(v, (str, SecretStr)): raise ValueError("Empty string!") return v diff --git a/vectordb_bench/backend/clients/milvus/cli.py b/vectordb_bench/backend/clients/milvus/cli.py new file mode 100644 index 000000000..05ff95418 --- /dev/null +++ b/vectordb_bench/backend/clients/milvus/cli.py @@ -0,0 +1,291 @@ +from typing import Annotated, TypedDict, Unpack + +import click +from pydantic import SecretStr + +from vectordb_bench.cli.cli import ( + CommonTypedDict, + HNSWFlavor3, + IVFFlatTypedDictN, + cli, + click_parameter_decorators_from_typed_dict, + run, + +) +from vectordb_bench.backend.clients import DB + +DBTYPE = DB.Milvus + + +class MilvusTypedDict(TypedDict): + uri: Annotated[ + str, click.option("--uri", type=str, help="uri connection string", required=True) + ] + + +class MilvusAutoIndexTypedDict(CommonTypedDict, MilvusTypedDict): + ... + + +@cli.command() +@click_parameter_decorators_from_typed_dict(MilvusAutoIndexTypedDict) +def MilvusAutoIndex(**parameters: Unpack[MilvusAutoIndexTypedDict]): + from .config import MilvusConfig, AutoIndexConfig + + run( + db=DBTYPE, + db_config=MilvusConfig( + db_label=parameters["db_label"], + uri=SecretStr(parameters["uri"]), + ), + db_case_config=AutoIndexConfig(), + **parameters, + ) + + +@cli.command() +@click_parameter_decorators_from_typed_dict(MilvusAutoIndexTypedDict) +def MilvusFlat(**parameters: Unpack[MilvusAutoIndexTypedDict]): + from .config import MilvusConfig, FLATConfig + + run( + db=DBTYPE, + db_config=MilvusConfig( + db_label=parameters["db_label"], + uri=SecretStr(parameters["uri"]), + ), + db_case_config=FLATConfig(), + **parameters, + ) + + +class MilvusHNSWTypedDict(CommonTypedDict, MilvusTypedDict, HNSWFlavor3): + ... + + +@cli.command() +@click_parameter_decorators_from_typed_dict(MilvusHNSWTypedDict) +def MilvusHNSW(**parameters: Unpack[MilvusHNSWTypedDict]): + from .config import MilvusConfig, HNSWConfig + + run( + db=DBTYPE, + db_config=MilvusConfig( + db_label=parameters["db_label"], + uri=SecretStr(parameters["uri"]), + ), + db_case_config=HNSWConfig( + M=parameters["m"], + efConstruction=parameters["ef_construction"], + ef=parameters["ef_search"], + ), + **parameters, + ) + + +class MilvusIVFFlatTypedDict(CommonTypedDict, MilvusTypedDict, IVFFlatTypedDictN): + ... + + +@cli.command() +@click_parameter_decorators_from_typed_dict(MilvusIVFFlatTypedDict) +def MilvusIVFFlat(**parameters: Unpack[MilvusIVFFlatTypedDict]): + from .config import MilvusConfig, IVFFlatConfig + + run( + db=DBTYPE, + db_config=MilvusConfig( + db_label=parameters["db_label"], + uri=SecretStr(parameters["uri"]), + ), + db_case_config=IVFFlatConfig( + nlist=parameters["nlist"], + nprobe=parameters["nprobe"], + ), + **parameters, + ) + + +@cli.command() +@click_parameter_decorators_from_typed_dict(MilvusIVFFlatTypedDict) +def MilvusIVFSQ8(**parameters: Unpack[MilvusIVFFlatTypedDict]): + from .config import MilvusConfig, IVFSQ8Config + + run( + db=DBTYPE, + db_config=MilvusConfig( + db_label=parameters["db_label"], + uri=SecretStr(parameters["uri"]), + ), + db_case_config=IVFSQ8Config( + nlist=parameters["nlist"], + nprobe=parameters["nprobe"], + ), + **parameters, + ) + + +class MilvusDISKANNTypedDict(CommonTypedDict, MilvusTypedDict): + search_list: Annotated[ + str, click.option("--search-list", + type=int, + required=True) + ] + + +@cli.command() +@click_parameter_decorators_from_typed_dict(MilvusDISKANNTypedDict) +def MilvusDISKANN(**parameters: Unpack[MilvusDISKANNTypedDict]): + from .config import MilvusConfig, DISKANNConfig + + run( + db=DBTYPE, + db_config=MilvusConfig( + db_label=parameters["db_label"], + uri=SecretStr(parameters["uri"]), + ), + db_case_config=DISKANNConfig( + search_list=parameters["search_list"], + ), + **parameters, + ) + + +class MilvusGPUIVFTypedDict(CommonTypedDict, MilvusTypedDict, MilvusIVFFlatTypedDict): + cache_dataset_on_device: Annotated[ + str, click.option("--cache-dataset-on-device", + type=str, + required=True) + ] + refine_ratio: Annotated[ + str, click.option("--refine-ratio", + type=float, + required=True) + ] + + +@cli.command() +@click_parameter_decorators_from_typed_dict(MilvusGPUIVFTypedDict) +def MilvusGPUIVFFlat(**parameters: Unpack[MilvusGPUIVFTypedDict]): + from .config import MilvusConfig, GPUIVFFlatConfig + + run( + db=DBTYPE, + db_config=MilvusConfig( + db_label=parameters["db_label"], + uri=SecretStr(parameters["uri"]), + ), + db_case_config=GPUIVFFlatConfig( + nlist=parameters["nlist"], + nprobe=parameters["nprobe"], + cache_dataset_on_device=parameters["cache_dataset_on_device"], + refine_ratio=parameters.get("refine_ratio"), + ), + **parameters, + ) + + +class MilvusGPUIVFPQTypedDict(CommonTypedDict, MilvusTypedDict, MilvusIVFFlatTypedDict, MilvusGPUIVFTypedDict): + m: Annotated[ + str, click.option("--m", + type=int, help="hnsw m", + required=True) + ] + nbits: Annotated[ + str, click.option("--nbits", + type=int, + required=True) + ] + + +@cli.command() +@click_parameter_decorators_from_typed_dict(MilvusGPUIVFPQTypedDict) +def MilvusGPUIVFPQ(**parameters: Unpack[MilvusGPUIVFPQTypedDict]): + from .config import MilvusConfig, GPUIVFPQConfig + + run( + db=DBTYPE, + db_config=MilvusConfig( + db_label=parameters["db_label"], + uri=SecretStr(parameters["uri"]), + ), + db_case_config=GPUIVFPQConfig( + nlist=parameters["nlist"], + nprobe=parameters["nprobe"], + m=parameters["m"], + nbits=parameters["nbits"], + cache_dataset_on_device=parameters["cache_dataset_on_device"], + refine_ratio=parameters["refine_ratio"], + ), + **parameters, + ) + + +class MilvusGPUCAGRATypedDict(CommonTypedDict, MilvusTypedDict, MilvusGPUIVFTypedDict): + intermediate_graph_degree: Annotated[ + str, click.option("--intermediate-graph-degree", + type=int, + required=True) + ] + graph_degree: Annotated[ + str, click.option("--graph-degree", + type=int, + required=True) + ] + build_algo: Annotated[ + str, click.option("--build_algo", + type=str, + required=True) + ] + team_size: Annotated[ + str, click.option("--team-size", + type=int, + required=True) + ] + search_width: Annotated[ + str, click.option("--search-width", + type=int, + required=True) + ] + itopk_size: Annotated[ + str, click.option("--itopk-size", + type=int, + required=True) + ] + min_iterations: Annotated[ + str, click.option("--min-iterations", + type=int, + required=True) + ] + max_iterations: Annotated[ + str, click.option("--max-iterations", + type=int, + required=True) + ] + + +@cli.command() +@click_parameter_decorators_from_typed_dict(MilvusGPUCAGRATypedDict) +def MilvusGPUCAGRA(**parameters: Unpack[MilvusGPUCAGRATypedDict]): + from .config import MilvusConfig, GPUCAGRAConfig + + run( + db=DBTYPE, + db_config=MilvusConfig( + db_label=parameters["db_label"], + uri=SecretStr(parameters["uri"]), + ), + db_case_config=GPUCAGRAConfig( + intermediate_graph_degree=parameters["intermediate_graph_degree"], + graph_degree=parameters["graph_degree"], + itopk_size=parameters["itopk_size"], + team_size=parameters["team_size"], + search_width=parameters["search_width"], + min_iterations=parameters["min_iterations"], + max_iterations=parameters["max_iterations"], + build_algo=parameters["build_algo"], + cache_dataset_on_device=parameters["cache_dataset_on_device"], + refine_ratio=parameters["refine_ratio"], + ), + **parameters, + ) diff --git a/vectordb_bench/backend/clients/milvus/milvus.py b/vectordb_bench/backend/clients/milvus/milvus.py index 2436e2680..4590265ae 100644 --- a/vectordb_bench/backend/clients/milvus/milvus.py +++ b/vectordb_bench/backend/clients/milvus/milvus.py @@ -8,7 +8,7 @@ from pymilvus import Collection, utility from pymilvus import CollectionSchema, DataType, FieldSchema, MilvusException -from ..api import VectorDB +from ..api import VectorDB, IndexType from .config import MilvusIndexConfig @@ -122,10 +122,18 @@ def wait_index(): if self.case_config.is_gpu_index: log.debug("skip compaction for gpu index type.") else : - self.col.compact() - self.col.wait_for_compaction_completed() + try: + self.col.compact() + self.col.wait_for_compaction_completed() + except Exception as e: + log.warning(f"{self.name} compact error: {e}") + if hasattr(e, 'code'): + if e.code().name == 'PERMISSION_DENIED': + log.warning(f"Skip compact due to permission denied.") + pass + else: + raise e wait_index() - except Exception as e: log.warning(f"{self.name} optimize error: {e}") raise e from None @@ -143,7 +151,6 @@ def _pre_load(self, coll: Collection): self.case_config.index_param(), index_name=self._index_name, ) - coll.load() log.info(f"{self.name} load") except Exception as e: @@ -160,7 +167,7 @@ def need_normalize_cosine(self) -> bool: if self.case_config.is_gpu_index: log.info(f"current gpu_index only supports IP / L2, cosine dataset need normalize.") return True - + return False def insert_embeddings( diff --git a/vectordb_bench/backend/clients/pgvector/cli.py b/vectordb_bench/backend/clients/pgvector/cli.py new file mode 100644 index 000000000..31b268231 --- /dev/null +++ b/vectordb_bench/backend/clients/pgvector/cli.py @@ -0,0 +1,116 @@ +from typing import Annotated, Optional, TypedDict, Unpack + +import click +import os +from pydantic import SecretStr + +from ....cli.cli import ( + CommonTypedDict, + HNSWFlavor1, + IVFFlatTypedDict, + cli, + click_parameter_decorators_from_typed_dict, + run, +) +from vectordb_bench.backend.clients import DB + + +class PgVectorTypedDict(CommonTypedDict): + user_name: Annotated[ + str, click.option("--user-name", type=str, help="Db username", required=True) + ] + password: Annotated[ + str, + click.option("--password", + type=str, + help="Postgres database password", + default=lambda: os.environ.get("POSTGRES_PASSWORD", ""), + show_default="$POSTGRES_PASSWORD", + ), + ] + + host: Annotated[ + str, click.option("--host", type=str, help="Db host", required=True) + ] + db_name: Annotated[ + str, click.option("--db-name", type=str, help="Db name", required=True) + ] + maintenance_work_mem: Annotated[ + Optional[str], + click.option( + "--maintenance-work-mem", + type=str, + help="Sets the maximum memory to be used for maintenance operations (index creation). " + "Can be entered as string with unit like '64GB' or as an integer number of KB." + "This will set the parameters: max_parallel_maintenance_workers," + " max_parallel_workers & table(parallel_workers)", + required=False, + ), + ] + max_parallel_workers: Annotated[ + Optional[int], + click.option( + "--max-parallel-workers", + type=int, + help="Sets the maximum number of parallel processes per maintenance operation (index creation)", + required=False, + ), + ] + + +class PgVectorIVFFlatTypedDict(PgVectorTypedDict, IVFFlatTypedDict): + ... + + +@cli.command() +@click_parameter_decorators_from_typed_dict(PgVectorIVFFlatTypedDict) +def PgVectorIVFFlat( + **parameters: Unpack[PgVectorIVFFlatTypedDict], +): + from .config import PgVectorConfig, PgVectorIVFFlatConfig + + run( + db=DB.PgVector, + db_config=PgVectorConfig( + db_label=parameters["db_label"], + user_name=SecretStr(parameters["user_name"]), + password=SecretStr(parameters["password"]), + host=parameters["host"], + db_name=parameters["db_name"], + ), + db_case_config=PgVectorIVFFlatConfig( + metric_type=None, lists=parameters["lists"], probes=parameters["probes"] + ), + **parameters, + ) + + +class PgVectorHNSWTypedDict(PgVectorTypedDict, HNSWFlavor1): + ... + + +@cli.command() +@click_parameter_decorators_from_typed_dict(PgVectorHNSWTypedDict) +def PgVectorHNSW( + **parameters: Unpack[PgVectorHNSWTypedDict], +): + from .config import PgVectorConfig, PgVectorHNSWConfig + + run( + db=DB.PgVector, + db_config=PgVectorConfig( + db_label=parameters["db_label"], + user_name=SecretStr(parameters["user_name"]), + password=SecretStr(parameters["password"]), + host=parameters["host"], + db_name=parameters["db_name"], + ), + db_case_config=PgVectorHNSWConfig( + m=parameters["m"], + ef_construction=parameters["ef_construction"], + ef_search=parameters["ef_search"], + maintenance_work_mem=parameters["maintenance_work_mem"], + max_parallel_workers=parameters["max_parallel_workers"], + ), + **parameters, + ) diff --git a/vectordb_bench/backend/clients/pgvector/config.py b/vectordb_bench/backend/clients/pgvector/config.py index a4cc584e8..496a3b440 100644 --- a/vectordb_bench/backend/clients/pgvector/config.py +++ b/vectordb_bench/backend/clients/pgvector/config.py @@ -109,7 +109,7 @@ def _optionally_build_with_options(with_options: Mapping[str, Any]) -> Sequence[ def _optionally_build_set_options( set_mapping: Mapping[str, Any] ) -> Sequence[dict[str, Any]]: - """Walk through options, creating 'SET 'key1 = "value1";' commands""" + """Walk through options, creating 'SET 'key1 = "value1";' list""" session_options = [] for setting_name, value in set_mapping.items(): if value: diff --git a/vectordb_bench/backend/clients/pgvector/pgvector.py b/vectordb_bench/backend/clients/pgvector/pgvector.py index 8f8244412..5e66fd77e 100644 --- a/vectordb_bench/backend/clients/pgvector/pgvector.py +++ b/vectordb_bench/backend/clients/pgvector/pgvector.py @@ -58,14 +58,13 @@ def __init__( self.case_config.create_index_after_load, ) ): - err = f"{self.name} config must create an index using create_index_before_load and/or create_index_after_load" + err = f"{self.name} config must create an index using create_index_before_load or create_index_after_load" log.error(err) raise RuntimeError( f"{err}\n{pprint.pformat(self.db_config)}\n{pprint.pformat(self.case_config)}" ) if drop_old: - # self.pg_table.drop(pg_engine, checkfirst=True) self._drop_index() self._drop_table() self._create_table(dim) @@ -257,7 +256,10 @@ def _create_index(self): with_clause = sql.Composed(()) index_create_sql = sql.SQL( - "CREATE INDEX IF NOT EXISTS {index_name} ON public.{table_name} USING {index_type} (embedding {embedding_metric})" + """ + CREATE INDEX IF NOT EXISTS {index_name} ON public.{table_name} + USING {index_type} (embedding {embedding_metric}) + """ ).format( index_name=sql.Identifier(self._index_name), table_name=sql.Identifier(self.table_name), diff --git a/vectordb_bench/backend/clients/redis/cli.py b/vectordb_bench/backend/clients/redis/cli.py new file mode 100644 index 000000000..172b2b52a --- /dev/null +++ b/vectordb_bench/backend/clients/redis/cli.py @@ -0,0 +1,74 @@ +from typing import Annotated, TypedDict, Unpack + +import click +from pydantic import SecretStr + +from ....cli.cli import ( + CommonTypedDict, + HNSWFlavor2, + cli, + click_parameter_decorators_from_typed_dict, + run, +) +from .. import DB + + +class RedisTypedDict(TypedDict): + host: Annotated[ + str, click.option("--host", type=str, help="Db host", required=True) + ] + password: Annotated[str, click.option("--password", type=str, help="Db password")] + port: Annotated[int, click.option("--port", type=int, default=6379, help="Db Port")] + ssl: Annotated[ + bool, + click.option( + "--ssl/--no-ssl", + is_flag=True, + show_default=True, + default=True, + help="Enable or disable SSL for Redis", + ), + ] + ssl_ca_certs: Annotated[ + str, + click.option( + "--ssl-ca-certs", + show_default=True, + help="Path to certificate authority file to use for SSL", + ), + ] + cmd: Annotated[ + bool, + click.option( + "--cmd", + is_flag=True, + show_default=True, + default=False, + help="Cluster Mode Disabled (CMD) for Redis doesn't use Cluster conn", + ), + ] + + +class RedisHNSWTypedDict(CommonTypedDict, RedisTypedDict, HNSWFlavor2): + ... + + +@cli.command() +@click_parameter_decorators_from_typed_dict(RedisHNSWTypedDict) +def Redis(**parameters: Unpack[RedisHNSWTypedDict]): + from .config import RedisConfig + run( + db=DB.Redis, + db_config=RedisConfig( + db_label=parameters["db_label"], + password=SecretStr(parameters["password"]) + if parameters["password"] + else None, + host=SecretStr(parameters["host"]), + port=parameters["port"], + ssl=parameters["ssl"], + ssl_ca_certs=parameters["ssl_ca_certs"], + cmd=parameters["cmd"], + ), + **parameters, + ) diff --git a/vectordb_bench/backend/clients/test/cli.py b/vectordb_bench/backend/clients/test/cli.py new file mode 100644 index 000000000..f06f33492 --- /dev/null +++ b/vectordb_bench/backend/clients/test/cli.py @@ -0,0 +1,25 @@ +from typing import Unpack + +from ....cli.cli import ( + CommonTypedDict, + cli, + click_parameter_decorators_from_typed_dict, + run, +) +from .. import DB +from ..test.config import TestConfig, TestIndexConfig + + +class TestTypedDict(CommonTypedDict): + ... + + +@cli.command() +@click_parameter_decorators_from_typed_dict(TestTypedDict) +def Test(**parameters: Unpack[TestTypedDict]): + run( + db=DB.NewClient, + db_config=TestConfig(db_label=parameters["db_label"]), + db_case_config=TestIndexConfig(), + **parameters, + ) diff --git a/vectordb_bench/backend/clients/test/config.py b/vectordb_bench/backend/clients/test/config.py new file mode 100644 index 000000000..01a77e000 --- /dev/null +++ b/vectordb_bench/backend/clients/test/config.py @@ -0,0 +1,18 @@ +from pydantic import BaseModel, SecretStr + +from ..api import DBCaseConfig, DBConfig, IndexType, MetricType + + +class TestConfig(DBConfig): + def to_dict(self) -> dict: + return {"db_label": self.db_label} + + +class TestIndexConfig(BaseModel, DBCaseConfig): + metric_type: MetricType | None = None + + def index_param(self) -> dict: + return {} + + def search_param(self) -> dict: + return {} diff --git a/vectordb_bench/backend/clients/test/test.py b/vectordb_bench/backend/clients/test/test.py new file mode 100644 index 000000000..78732eb1e --- /dev/null +++ b/vectordb_bench/backend/clients/test/test.py @@ -0,0 +1,62 @@ +import logging +from contextlib import contextmanager +from typing import Any, Generator, Optional, Tuple + +from ..api import DBCaseConfig, VectorDB + +log = logging.getLogger(__name__) + + +class Test(VectorDB): + def __init__( + self, + dim: int, + db_config: dict, + db_case_config: DBCaseConfig, + drop_old: bool = False, + **kwargs, + ): + self.db_config = db_config + self.case_config = db_case_config + + log.info("Starting Test DB") + + @contextmanager + def init(self) -> Generator[None, None, None]: + """create and destroy connections to database. + + Examples: + >>> with self.init(): + >>> self.insert_embeddings() + """ + + yield + + def ready_to_load(self) -> bool: + return True + + def optimize(self) -> None: + pass + + def insert_embeddings( + self, + embeddings: list[list[float]], + metadata: list[int], + **kwargs: Any, + ) -> Tuple[int, Optional[Exception]]: + """Insert embeddings into the database. + Should call self.init() first. + """ + raise RuntimeError("Not implemented") + return len(metadata), None + + def search_embedding( + self, + query: list[float], + k: int = 100, + filters: dict | None = None, + timeout: int | None = None, + **kwargs: Any, + ) -> list[int]: + raise NotImplementedError + return [i for i in range(k)] diff --git a/vectordb_bench/backend/clients/weaviate_cloud/cli.py b/vectordb_bench/backend/clients/weaviate_cloud/cli.py new file mode 100644 index 000000000..b6f16011b --- /dev/null +++ b/vectordb_bench/backend/clients/weaviate_cloud/cli.py @@ -0,0 +1,41 @@ +from typing import Annotated, Unpack + +import click +from pydantic import SecretStr + +from ....cli.cli import ( + CommonTypedDict, + cli, + click_parameter_decorators_from_typed_dict, + run, +) +from .. import DB + + +class WeaviateTypedDict(CommonTypedDict): + api_key: Annotated[ + str, click.option("--api-key", type=str, help="Weaviate api key", required=True) + ] + url: Annotated[ + str, + click.option("--url", type=str, help="Weaviate url", required=True), + ] + + +@cli.command() +@click_parameter_decorators_from_typed_dict(WeaviateTypedDict) +def Weaviate(**parameters: Unpack[WeaviateTypedDict]): + from .config import WeaviateConfig, WeaviateIndexConfig + + run( + db=DB.WeaviateCloud, + db_config=WeaviateConfig( + db_label=parameters["db_label"], + api_key=SecretStr(parameters["api_key"]), + url=SecretStr(parameters["url"]), + ), + db_case_config=WeaviateIndexConfig( + ef=256, efConstruction=256, maxConnections=16 + ), + **parameters, + ) diff --git a/vectordb_bench/backend/clients/zilliz_cloud/cli.py b/vectordb_bench/backend/clients/zilliz_cloud/cli.py new file mode 100644 index 000000000..31618f4ec --- /dev/null +++ b/vectordb_bench/backend/clients/zilliz_cloud/cli.py @@ -0,0 +1,55 @@ +from typing import Annotated, Unpack + +import click +import os +from pydantic import SecretStr + +from vectordb_bench.cli.cli import ( + CommonTypedDict, + cli, + click_parameter_decorators_from_typed_dict, + run, +) +from vectordb_bench.backend.clients import DB + + +class ZillizTypedDict(CommonTypedDict): + uri: Annotated[ + str, click.option("--uri", type=str, help="uri connection string", required=True) + ] + user_name: Annotated[ + str, click.option("--user-name", type=str, help="Db username", required=True) + ] + password: Annotated[ + str, + click.option("--password", + type=str, + help="Zilliz password", + default=lambda: os.environ.get("ZILLIZ_PASSWORD", ""), + show_default="$ZILLIZ_PASSWORD", + ), + ] + level: Annotated[ + str, + click.option("--level", type=str, help="Zilliz index level", required=False), + ] + + +@cli.command() +@click_parameter_decorators_from_typed_dict(ZillizTypedDict) +def ZillizAutoIndex(**parameters: Unpack[ZillizTypedDict]): + from .config import ZillizCloudConfig, AutoIndexConfig + + run( + db=DB.ZillizCloud, + db_config=ZillizCloudConfig( + db_label=parameters["db_label"], + uri=SecretStr(parameters["uri"]), + user=parameters["user_name"], + password=SecretStr(parameters["password"]), + ), + db_case_config=AutoIndexConfig( + params={parameters["level"]}, + ), + **parameters, + ) diff --git a/vectordb_bench/backend/task_runner.py b/vectordb_bench/backend/task_runner.py index c5c368d02..0680847aa 100644 --- a/vectordb_bench/backend/task_runner.py +++ b/vectordb_bench/backend/task_runner.py @@ -8,7 +8,7 @@ from . import utils from .cases import Case, CaseLabel from ..base import BaseModel -from ..models import TaskConfig, PerformanceTimeoutError +from ..models import TaskConfig, PerformanceTimeoutError, TaskStage from .clients import ( api, @@ -29,7 +29,7 @@ class RunningStatus(Enum): class CaseRunner(BaseModel): - """ DataSet, filter_rate, db_class with db config + """DataSet, filter_rate, db_class with db config Fields: run_id(str): run_id of this case runner, @@ -49,8 +49,9 @@ class CaseRunner(BaseModel): db: api.VectorDB | None = None test_emb: list[list[float]] | None = None - search_runner: MultiProcessingSearchRunner | None = None serial_search_runner: SerialSearchRunner | None = None + search_runner: MultiProcessingSearchRunner | None = None + final_search_runner: MultiProcessingSearchRunner | None = None def __eq__(self, obj): if isinstance(obj, CaseRunner): @@ -58,7 +59,7 @@ def __eq__(self, obj): self.config.db == obj.config.db and \ self.config.db_case_config == obj.config.db_case_config and \ self.ca.dataset == obj.ca.dataset - return False + return False def display(self) -> dict: c_dict = self.ca.dict(include={'label':True, 'filters': True,'dataset':{'data': {'name': True, 'size': True, 'dim': True, 'metric_type': True, 'label': True}} }) @@ -79,20 +80,25 @@ def init_db(self, drop_old: bool = True) -> None: db_config=self.config.db_config.to_dict(), db_case_config=self.config.db_case_config, drop_old=drop_old, - ) + ) # type:ignore + def _pre_run(self, drop_old: bool = True): try: self.init_db(drop_old) self.ca.dataset.prepare(self.dataset_source, filters=self.ca.filter_rate) except ModuleNotFoundError as e: - log.warning(f"pre run case error: please install client for db: {self.config.db}, error={e}") + log.warning( + f"pre run case error: please install client for db: {self.config.db}, error={e}" + ) raise e from None except Exception as e: log.warning(f"pre run case error: {e}") raise e from None def run(self, drop_old: bool = True) -> Metric: + log.info("Starting run") + self._pre_run(drop_old) if self.ca.label == CaseLabel.Load: @@ -105,31 +111,35 @@ def run(self, drop_old: bool = True) -> Metric: raise ValueError(msg) def _run_capacity_case(self) -> Metric: - """ run capacity cases + """run capacity cases Returns: Metric: the max load count """ + assert self.db is not None log.info("Start capacity case") try: - runner = SerialInsertRunner(self.db, self.ca.dataset, self.normalize, self.ca.load_timeout) + runner = SerialInsertRunner( + self.db, self.ca.dataset, self.normalize, self.ca.load_timeout + ) count = runner.run_endlessness() except Exception as e: log.warning(f"Failed to run capacity case, reason = {e}") raise e from None else: - log.info(f"Capacity case loading dataset reaches VectorDB's limit: max capacity = {count}") + log.info( + f"Capacity case loading dataset reaches VectorDB's limit: max capacity = {count}" + ) return Metric(max_load_count=count) def _run_perf_case(self, drop_old: bool = True) -> Metric: - """ run performance cases + """run performance cases Returns: Metric: load_duration, recall, serial_latency_p99, and, qps """ - try: - m = Metric() - if drop_old: + ''' + if drop_old: _, load_dur = self._load_train_data() build_dur = self._optimize() m.load_duration = round(load_dur+build_dur, 4) @@ -142,6 +152,39 @@ def _run_perf_case(self, drop_old: bool = True) -> Metric: self._init_search_runner() m.qps = self._conc_search() m.recall, m.serial_latency_p99 = self._serial_search() + ''' + + log.info("Start performance case") + try: + m = Metric() + if drop_old: + if TaskStage.LOAD in self.config.stages: + # self._load_train_data() + _, load_dur = self._load_train_data() + build_dur = self._optimize() + m.load_duration = round(load_dur + build_dur, 4) + log.info( + f"Finish loading the entire dataset into VectorDB," + f" insert_duration={load_dur}, optimize_duration={build_dur}" + f" load_duration(insert + optimize) = {m.load_duration}" + ) + else: + log.info("Data loading skipped") + if ( + TaskStage.SEARCH_SERIAL in self.config.stages + or TaskStage.SEARCH_CONCURRENT in self.config.stages + ): + self._init_search_runner() + if TaskStage.SEARCH_SERIAL in self.config.stages: + search_results = self._serial_search() + ''' + m.recall = search_results.recall + m.serial_latencies = search_results.serial_latencies + ''' + m.recall, m.serial_latency_p99 = search_results + if TaskStage.SEARCH_CONCURRENT in self.config.stages: + search_results = self._conc_search() + m.qps = search_results except Exception as e: log.warning(f"Failed to run performance case, reason = {e}") traceback.print_exc() @@ -217,18 +260,23 @@ def _init_search_runner(self): gt_df = self.ca.dataset.gt_data - self.serial_search_runner = SerialSearchRunner( - db=self.db, - test_data=self.test_emb, - ground_truth=gt_df, - filters=self.ca.filters, - ) - - self.search_runner = MultiProcessingSearchRunner( - db=self.db, - test_data=self.test_emb, - filters=self.ca.filters, - ) + if TaskStage.SEARCH_SERIAL in self.config.stages: + self.serial_search_runner = SerialSearchRunner( + db=self.db, + test_data=self.test_emb, + ground_truth=gt_df, + filters=self.ca.filters, + k=self.config.case_config.k, + ) + if TaskStage.SEARCH_CONCURRENT in self.config.stages: + self.search_runner = MultiProcessingSearchRunner( + db=self.db, + test_data=self.test_emb, + filters=self.ca.filters, + concurrencies=self.config.case_config.concurrency_search_config.num_concurrency, + duration=self.config.case_config.concurrency_search_config.concurrency_duration, + k=self.config.case_config.k, + ) def stop(self): if self.search_runner: diff --git a/vectordb_bench/cli/__init__.py b/vectordb_bench/cli/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/vectordb_bench/cli/cli.py b/vectordb_bench/cli/cli.py new file mode 100644 index 000000000..00910261b --- /dev/null +++ b/vectordb_bench/cli/cli.py @@ -0,0 +1,362 @@ +import logging +import time +from concurrent.futures import wait +from datetime import datetime +from pprint import pformat +from typing import ( + Annotated, + Callable, + List, + Optional, + Type, + TypedDict, + Unpack, + get_origin, + get_type_hints, + Dict, + Any, +) +import click +from .. import config +from ..backend.clients import DB +from ..interface import benchMarkRunner, global_result_future +from ..models import ( + CaseConfig, + CaseType, + ConcurrencySearchConfig, + DBCaseConfig, + DBConfig, + TaskConfig, + TaskStage, +) +import os +from yaml import load +try: + from yaml import CLoader as Loader +except ImportError: + from yaml import Loader + + +def click_get_defaults_from_file(ctx, param, value): + if value: + if os.path.exists(value): + input_file = value + else: + input_file = os.path.join(config.CONFIG_LOCAL_DIR, value) + try: + with open(input_file, 'r') as f: + _config: Dict[str, Dict[str, Any]] = load(f.read(), Loader=Loader) + ctx.default_map = _config.get(ctx.command.name, {}) + except Exception as e: + raise click.BadParameter(f"Failed to load config file: {e}") + return value + + +def click_parameter_decorators_from_typed_dict( + typed_dict: Type, +) -> Callable[[click.decorators.FC], click.decorators.FC]: + """A convenience method decorator that will read in a TypedDict with parameters defined by Annotated types. + from .models import CaseConfig, CaseType, DBCaseConfig, DBConfig, TaskConfig, TaskStage + The click.options will be collected and re-composed as a single decorator to apply to the click.command. + + Args: + typed_dict (TypedDict) with Annotated[..., click.option()] keys + + Returns: + a fully decorated method + + + For clarity, the key names of the TypedDict will be used to determine the type hints for the input parameters. + The actual function parameters are controlled by the click.option definitions. You must manually ensure these are aligned in a sensible way! + + Example: + ``` + class CommonTypedDict(TypedDict): + z: Annotated[int, click.option("--z/--no-z", is_flag=True, type=bool, help="help z", default=True, show_default=True)] + name: Annotated[str, click.argument("name", required=False, default="Jeff")] + + class FooTypedDict(CommonTypedDict): + x: Annotated[int, click.option("--x", type=int, help="help x", default=1, show_default=True)] + y: Annotated[str, click.option("--y", type=str, help="help y", default="foo", show_default=True)] + + @cli.command() + @click_parameter_decorators_from_typed_dict(FooTypedDict) + def foo(**parameters: Unpack[FooTypedDict]): + "Foo docstring" + print(f"input parameters: {parameters["x"]}") + ``` + """ + decorators = [] + for _, t in get_type_hints(typed_dict, include_extras=True).items(): + assert get_origin(t) is Annotated + if ( + len(t.__metadata__) == 1 + and t.__metadata__[0].__module__ == "click.decorators" + ): + # happy path -- only accept Annotated[..., Union[click.option,click.argument,...]] with no additional metadata defined (len=1) + decorators.append(t.__metadata__[0]) + else: + raise RuntimeError( + "Click-TypedDict decorator parsing must only contain root type and a click decorator like click.option. See docstring" + ) + + def deco(f): + for dec in reversed(decorators): + f = dec(f) + return f + + return deco + + +def click_arg_split(ctx: click.Context, param: click.core.Option, value): + """Will split a comma-separated list input into an actual list. + + Args: + ctx (...): unused click arg + param (...): unused click arg + value (str): input comma-separated list + + Returns: + value (List[str]): list of original + """ + # split columns by ',' and remove whitespace + if value is None: + return [] + return [c.strip() for c in value.split(",") if c.strip()] + + +def parse_task_stages( + drop_old: bool, + load: bool, + search_serial: bool, + search_concurrent: bool, +) -> List[TaskStage]: + stages = [] + if load and not drop_old: + raise RuntimeError("Dropping old data cannot be skipped if loading data") + elif drop_old and not load: + raise RuntimeError("Load cannot be skipped if dropping old data") + if drop_old: + stages.append(TaskStage.DROP_OLD) + if load: + stages.append(TaskStage.LOAD) + if search_serial: + stages.append(TaskStage.SEARCH_SERIAL) + if search_concurrent: + stages.append(TaskStage.SEARCH_CONCURRENT) + return stages + + +log = logging.getLogger(__name__) + + +class CommonTypedDict(TypedDict): + config_file: Annotated[ + bool, + click.option('--config-file', + type=click.Path(), + callback=click_get_defaults_from_file, + is_eager=True, + expose_value=False, + help='Read configuration from yaml file'), + ] + drop_old: Annotated[ + bool, + click.option( + "--drop-old/--skip-drop-old", + type=bool, + default=True, + help="Drop old or skip", + show_default=True, + ), + ] + load: Annotated[ + bool, + click.option( + "--load/--skip-load", + type=bool, + default=True, + help="Load or skip", + show_default=True, + ), + ] + search_serial: Annotated[ + bool, + click.option( + "--search-serial/--skip-search-serial", + type=bool, + default=True, + help="Search serial or skip", + show_default=True, + ), + ] + search_concurrent: Annotated[ + bool, + click.option( + "--search-concurrent/--skip-search-concurrent", + type=bool, + default=True, + help="Search concurrent or skip", + show_default=True, + ), + ] + case_type: Annotated[ + str, + click.option( + "--case-type", + type=click.Choice([ct.name for ct in CaseType if ct.name != "Custom"]), + default="Performance1536D50K", + help="Case type", + ), + ] + db_label: Annotated[ + str, + click.option( + "--db-label", type=str, help="Db label, default: date in ISO format", + show_default=True, + default=datetime.now().isoformat() + ), + ] + dry_run: Annotated[ + bool, + click.option( + "--dry-run", + type=bool, + default=False, + is_flag=True, + help="Print just the configuration and exit without running the tasks", + ), + ] + k: Annotated[ + int, + click.option( + "--k", + type=int, + default=config.K_DEFAULT, + show_default=True, + help="K value for number of nearest neighbors to search", + ), + ] + concurrency_duration: Annotated[ + int, + click.option( + "--concurrency-duration", + type=int, + default=config.CONCURRENCY_DURATION, + show_default=True, + help="Adjusts the duration in seconds of each concurrency search", + ), + ] + num_concurrency: Annotated[ + List[str], + click.option( + "--num-concurrency", + type=str, + help="Comma-separated list of concurrency values to test during concurrent search", + show_default=True, + default=",".join(map(str, config.NUM_CONCURRENCY)), + callback=lambda *args: list(map(int, click_arg_split(*args))), + ), + ] + + +class HNSWBaseTypedDict(TypedDict): + m: Annotated[Optional[int], click.option("--m", type=int, help="hnsw m")] + ef_construction: Annotated[ + Optional[int], + click.option("--ef-construction", type=int, help="hnsw ef-construction"), + ] + + +class HNSWBaseRequiredTypedDict(TypedDict): + m: Annotated[Optional[int], click.option("--m", type=int, help="hnsw m", required=True)] + ef_construction: Annotated[ + Optional[int], + click.option("--ef-construction", type=int, help="hnsw ef-construction", required=True), + ] + + +class HNSWFlavor1(HNSWBaseTypedDict): + ef_search: Annotated[ + Optional[int], click.option("--ef-search", type=int, help="hnsw ef-search") + ] + + +class HNSWFlavor2(HNSWBaseTypedDict): + ef_runtime: Annotated[ + Optional[int], click.option("--ef-runtime", type=int, help="hnsw ef-runtime") + ] + + +class HNSWFlavor3(HNSWBaseRequiredTypedDict): + ef_search: Annotated[ + Optional[int], click.option("--ef-search", type=int, help="hnsw ef-search", required=True) + ] + + +class IVFFlatTypedDict(TypedDict): + lists: Annotated[ + Optional[int], click.option("--lists", type=int, help="ivfflat lists") + ] + probes: Annotated[ + Optional[int], click.option("--probes", type=int, help="ivfflat probes") + ] + + +class IVFFlatTypedDictN(TypedDict): + nlist: Annotated[ + Optional[int], click.option("--lists", "nlist", type=int, help="ivfflat lists", required=True) + ] + nprobe: Annotated[ + Optional[int], click.option("--probes", "nprobe", type=int, help="ivfflat probes", required=True) + ] + + +@click.group() +def cli(): + ... + + +def run( + db: DB, + db_config: DBConfig, + db_case_config: DBCaseConfig, + **parameters: Unpack[CommonTypedDict], +): + """Builds a single VectorDBBench Task and runs it, awaiting the task until finished. + + Args: + db (DB) + db_config (DBConfig) + db_case_config (DBCaseConfig) + **parameters: expects keys from CommonTypedDict + """ + + task = TaskConfig( + db=db, + db_config=db_config, + db_case_config=db_case_config, + case_config=CaseConfig( + case_id=CaseType[parameters["case_type"]], + k=parameters["k"], + concurrency_search_config=ConcurrencySearchConfig( + concurrency_duration=parameters["concurrency_duration"], + num_concurrency=[int(s) for s in parameters["num_concurrency"]], + ), + ), + stages=parse_task_stages( + ( + False if not parameters["load"] else parameters["drop_old"] + ), # only drop old data if loading new data + parameters["load"], + parameters["search_serial"], + parameters["search_concurrent"], + ), + ) + + log.info(f"Task:\n{pformat(task)}\n") + if not parameters["dry_run"]: + benchMarkRunner.run([task]) + time.sleep(5) + if global_result_future: + wait([global_result_future]) diff --git a/vectordb_bench/cli/vectordbbench.py b/vectordb_bench/cli/vectordbbench.py new file mode 100644 index 000000000..396909cd5 --- /dev/null +++ b/vectordb_bench/cli/vectordbbench.py @@ -0,0 +1,20 @@ +from ..backend.clients.pgvector.cli import PgVectorHNSW +from ..backend.clients.redis.cli import Redis +from ..backend.clients.test.cli import Test +from ..backend.clients.weaviate_cloud.cli import Weaviate +from ..backend.clients.zilliz_cloud.cli import ZillizAutoIndex +from ..backend.clients.milvus.cli import MilvusAutoIndex + + +from .cli import cli + +cli.add_command(PgVectorHNSW) +cli.add_command(Redis) +cli.add_command(Weaviate) +cli.add_command(Test) +cli.add_command(ZillizAutoIndex) +cli.add_command(MilvusAutoIndex) + + +if __name__ == "__main__": + cli() diff --git a/vectordb_bench/config-files/sample_config.yml b/vectordb_bench/config-files/sample_config.yml new file mode 100644 index 000000000..5a7d2cfd5 --- /dev/null +++ b/vectordb_bench/config-files/sample_config.yml @@ -0,0 +1,17 @@ +pgvectorhnsw: + db_label: pgConfigTest + user_name: vectordbbench + db_name: vectordbbench + host: localhost + m: 16 + ef_construction: 128 + ef_search: 128 +milvushnsw: + skip_search_serial: True + case_type: Performance1536D50K + uri: http://localhost:19530 + m: 16 + ef_construction: 128 + ef_search: 128 + drop_old: False + load: False diff --git a/vectordb_bench/frontend/components/run_test/dbSelector.py b/vectordb_bench/frontend/components/run_test/dbSelector.py index 61db843f3..5fcbd8c08 100644 --- a/vectordb_bench/frontend/components/run_test/dbSelector.py +++ b/vectordb_bench/frontend/components/run_test/dbSelector.py @@ -1,3 +1,5 @@ +from streamlit.runtime.media_file_storage import MediaFileStorageError + from vectordb_bench.frontend.const.styles import * from vectordb_bench.frontend.const.dbCaseConfigs import DB_LIST @@ -30,7 +32,11 @@ def dbSelector(st): for i, db in enumerate(DB_LIST): column = dbContainerColumns[i % DB_SELECTOR_COLUMNS] dbIsActived[db] = column.checkbox(db.name) - column.image(DB_TO_ICON.get(db, "")) + try: + column.image(DB_TO_ICON.get(db, "")) + except MediaFileStorageError as e: + column.warning(f"{db.name} image not available") + pass activedDbList = [db for db in DB_LIST if dbIsActived[db]] return activedDbList diff --git a/vectordb_bench/frontend/components/run_test/submitTask.py b/vectordb_bench/frontend/components/run_test/submitTask.py index 22d34b4f5..26cb1ef70 100644 --- a/vectordb_bench/frontend/components/run_test/submitTask.py +++ b/vectordb_bench/frontend/components/run_test/submitTask.py @@ -37,22 +37,30 @@ def taskLabelInput(st): def advancedSettings(st): container = st.columns([1, 2]) index_already_exists = container[0].checkbox("Index already exists", value=False) - container[1].caption("if actived, inserting and building will be skipped.") + container[1].caption("if selected, inserting and building will be skipped.") container = st.columns([1, 2]) use_aliyun = container[0].checkbox("Dataset from Aliyun (Shanghai)", value=False) container[1].caption( - "if actived, the dataset will be downloaded from Aliyun OSS shanghai, default AWS S3 aws-us-west." + "if selected, the dataset will be downloaded from Aliyun OSS shanghai, default AWS S3 aws-us-west." ) - return index_already_exists, use_aliyun + container = st.columns([1, 2]) + k = container[0].number_input("k",min_value=1, value=100, label_visibility="collapsed") + container[1].caption( + "K value for number of nearest neighbors to search" + ) + + return index_already_exists, use_aliyun, k def controlPanel(st, tasks, taskLabel, isAllValid): - index_already_exists, use_aliyun = advancedSettings(st) + index_already_exists, use_aliyun, k = advancedSettings(st) def runHandler(): benchMarkRunner.set_drop_old(not index_already_exists) + for task in tasks: + task.case_config.k = k benchMarkRunner.set_download_address(use_aliyun) benchMarkRunner.run(tasks, taskLabel) diff --git a/vectordb_bench/frontend/const/dbCaseConfigs.py b/vectordb_bench/frontend/const/dbCaseConfigs.py index 1e69c57aa..ed101ac69 100644 --- a/vectordb_bench/frontend/const/dbCaseConfigs.py +++ b/vectordb_bench/frontend/const/dbCaseConfigs.py @@ -9,7 +9,7 @@ MAX_STREAMLIT_INT = (1 << 53) - 1 -DB_LIST = [d for d in DB] +DB_LIST = [d for d in DB if d != DB.Test] DIVIDER = "DIVIDER" CASE_LIST_WITH_DIVIDER = [ @@ -19,6 +19,7 @@ DIVIDER, CaseType.Performance1536D5M, CaseType.Performance1536D500K, + CaseType.Performance1536D50K, DIVIDER, CaseType.Performance768D10M1P, CaseType.Performance768D1M1P, diff --git a/vectordb_bench/interface.py b/vectordb_bench/interface.py index c170c67dc..c765d0d63 100644 --- a/vectordb_bench/interface.py +++ b/vectordb_bench/interface.py @@ -1,38 +1,33 @@ -import traceback +import concurrent.futures +import logging +import multiprocessing as mp import pathlib import signal -import logging +import traceback import uuid -import concurrent -import multiprocessing as mp +from enum import Enum from multiprocessing.connection import Connection import psutil -from enum import Enum from . import config -from .metric import Metric -from .models import ( - TaskConfig, - TestResult, - CaseResult, - LoadTimeoutError, - PerformanceTimeoutError, - ResultLabel, -) -from .backend.result_collector import ResultCollector from .backend.assembler import Assembler -from .backend.task_runner import TaskRunner from .backend.data_source import DatasetSource +from .backend.result_collector import ResultCollector +from .backend.task_runner import TaskRunner +from .metric import Metric +from .models import (CaseResult, LoadTimeoutError, PerformanceTimeoutError, + ResultLabel, TaskConfig, TaskStage, TestResult) log = logging.getLogger(__name__) global_result_future: concurrent.futures.Future | None = None + class SIGNAL(Enum): - SUCCESS=0 - ERROR=1 - WIP=2 + SUCCESS = 0 + ERROR = 1 + WIP = 2 class BenchMarkRunner: @@ -42,9 +37,11 @@ def __init__(self): self.drop_old: bool = True self.dataset_source: DatasetSource = DatasetSource.S3 + def set_drop_old(self, drop_old: bool): self.drop_old = drop_old + def set_download_address(self, use_aliyun: bool): if use_aliyun: self.dataset_source = DatasetSource.AliyunOSS @@ -152,13 +149,13 @@ def _async_task_v2(self, running_task: TaskRunner, send_conn: Connection) -> Non latest_runner, cached_load_duration = None, None for idx, runner in enumerate(running_task.case_runners): case_res = CaseResult( - result_id=idx, metrics=Metric(), task_config=runner.config, ) # drop_old = False if latest_runner and runner == latest_runner else config.DROP_OLD - drop_old = config.DROP_OLD + # drop_old = config.DROP_OLD + drop_old = TaskStage.DROP_OLD in runner.config.stages if latest_runner and runner == latest_runner: drop_old = False elif not self.drop_old: @@ -167,7 +164,7 @@ def _async_task_v2(self, running_task: TaskRunner, send_conn: Connection) -> Non log.info(f"[{idx+1}/{running_task.num_cases()}] start case: {runner.display()}, drop_old={drop_old}") case_res.metrics = runner.run(drop_old) log.info(f"[{idx+1}/{running_task.num_cases()}] finish case: {runner.display()}, " - f"result={case_res.metrics}, label={case_res.label}") + f"result={case_res.metrics}, label={case_res.label}") # cache the latest succeeded runner latest_runner = runner @@ -193,7 +190,6 @@ def _async_task_v2(self, running_task: TaskRunner, send_conn: Connection) -> Non c_results.append(case_res) send_conn.send((SIGNAL.WIP, idx)) - test_result = TestResult( run_id=running_task.run_id, task_label=running_task.task_label, @@ -204,7 +200,7 @@ def _async_task_v2(self, running_task: TaskRunner, send_conn: Connection) -> Non send_conn.send((SIGNAL.SUCCESS, None)) send_conn.close() - log.info(f"Succes to finish task: label={running_task.task_label}, run_id={running_task.run_id}") + log.info(f"Success to finish task: label={running_task.task_label}, run_id={running_task.run_id}") except Exception as e: err_msg = f"An error occurs when running task={running_task.task_label}, run_id={running_task.run_id}, err={e}" @@ -246,7 +242,7 @@ def kill_proc_tree(self, sig=signal.SIGTERM, timeout=None, on_terminate=None): called as soon as a child terminates. """ children = psutil.Process().children(recursive=True) - for p in children: + for p in children: try: log.warning(f"sending SIGTERM to child process: {p}") p.send_signal(sig) diff --git a/vectordb_bench/models.py b/vectordb_bench/models.py index ec1b610e1..aa9c930ea 100644 --- a/vectordb_bench/models.py +++ b/vectordb_bench/models.py @@ -17,7 +17,6 @@ from . import config from .metric import Metric - log = logging.getLogger(__name__) @@ -65,15 +64,55 @@ class CaseConfigParamType(Enum): maintenance_work_mem = "maintenance_work_mem" max_parallel_workers = "max_parallel_workers" + class CustomizedCase(BaseModel): pass +class ConcurrencySearchConfig(BaseModel): + num_concurrency: List[int] = config.NUM_CONCURRENCY + concurrency_duration: int = config.CONCURRENCY_DURATION + + class CaseConfig(BaseModel): """cases, dataset, test cases, filter rate, params""" case_id: CaseType custom_case: dict | None = None + k: int | None = config.K_DEFAULT + concurrency_search_config: ConcurrencySearchConfig = ConcurrencySearchConfig() + + ''' + @property + def k(self): + """K search parameter, default is config.K_DEFAULT""" + return self._k + + # + @k.setter + def k(self, value): + self._k = value + ''' + +class TaskStage(StrEnum): + """Enumerations of various stages of the task""" + + DROP_OLD = auto() + LOAD = auto() + SEARCH_SERIAL = auto() + SEARCH_CONCURRENT = auto() + + def __repr__(self) -> str: + return str.__repr__(self.value) + + +# TODO: Add CapacityCase enums and adjust TaskRunner to utilize +ALL_TASK_STAGES = [ + TaskStage.DROP_OLD, + TaskStage.LOAD, + TaskStage.SEARCH_SERIAL, + TaskStage.SEARCH_CONCURRENT, +] class TaskConfig(BaseModel): @@ -81,6 +120,7 @@ class TaskConfig(BaseModel): db_config: DBConfig db_case_config: DBCaseConfig case_config: CaseConfig + stages: List[TaskStage] = ALL_TASK_STAGES @property def db_name(self): @@ -210,18 +250,18 @@ def append_return(x, y): max_db = max(map(len, [f.task_config.db.name for f in filtered_results])) max_db_labels = ( - max(map(len, [f.task_config.db_config.db_label for f in filtered_results])) - + 3 + max(map(len, [f.task_config.db_config.db_label for f in filtered_results])) + + 3 ) max_case = max( map(len, [f.task_config.case_config.case_id.name for f in filtered_results]) ) max_load_dur = ( - max(map(len, [str(f.metrics.load_duration) for f in filtered_results])) + 3 + max(map(len, [str(f.metrics.load_duration) for f in filtered_results])) + 3 ) max_qps = max(map(len, [str(f.metrics.qps) for f in filtered_results])) + 3 max_recall = ( - max(map(len, [str(f.metrics.recall) for f in filtered_results])) + 3 + max(map(len, [str(f.metrics.recall) for f in filtered_results])) + 3 ) max_db_labels = 8 if max_db_labels < 8 else max_db_labels From 686b781bc3cc11632c70976c2295a62ce2853136 Mon Sep 17 00:00:00 2001 From: "Jonathan S. Katz" Date: Fri, 14 Jun 2024 17:17:34 -0400 Subject: [PATCH 069/327] pgvector: ensure vector is sent in binary representation PostgreSQL supports two methods of passing data from client to server: text and binary. While for many data types the difference may not be noticeable, we can see significant performance impact when converting a vector from binary => text => binary representation. See previous explanation here[1]. While the pgvector loading code accounts for this, the query code did not. This is due to the use of a list[float] type, which the pgvector-python adapter currently doesn't support. However, this adapter does support direct binary transfer if the data is represent as a Numpy array[2]. Testing shows that moving to a direct binary representation does have a significant impact on query results - my tests are showing a 3x impact -- and provides a more accurate representation for how this workload would execute. [1] https://github.com/erikbern/ann-benchmarks/pull/488 [2] https://github.com/pgvector/pgvector-python?tab=readme-ov-file#psycopg-3 --- vectordb_bench/backend/clients/pgvector/pgvector.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vectordb_bench/backend/clients/pgvector/pgvector.py b/vectordb_bench/backend/clients/pgvector/pgvector.py index 5e66fd77e..1ae661041 100644 --- a/vectordb_bench/backend/clients/pgvector/pgvector.py +++ b/vectordb_bench/backend/clients/pgvector/pgvector.py @@ -341,9 +341,10 @@ def search_embedding( assert self.conn is not None, "Connection is not initialized" assert self.cursor is not None, "Cursor is not initialized" + q = np.asarray(query) # TODO add filters support result = self.cursor.execute( - self._unfiltered_search, (query, k), prepare=True, binary=True + self._unfiltered_search, (q, k), prepare=True, binary=True ) return [int(i[0]) for i in result.fetchall()] From d341e9f02f0d5ae8b3481c5fc58f00c86a2dcdbd Mon Sep 17 00:00:00 2001 From: "min.tian" Date: Fri, 28 Jun 2024 14:22:39 +0800 Subject: [PATCH 070/327] new metric support: ndcg@100, performance under conc_test Signed-off-by: min.tian --- vectordb_bench/backend/runner/mp_runner.py | 17 +++- .../backend/runner/serial_runner.py | 10 ++- vectordb_bench/backend/task_runner.py | 8 +- .../frontend/components/check_results/data.py | 19 +++-- .../frontend/components/concurrent/charts.py | 82 +++++++++++++++++++ .../frontend/components/tables/data.py | 44 ++++++++++ vectordb_bench/frontend/pages/concurrent.py | 72 ++++++++++++++++ vectordb_bench/frontend/pages/tables.py | 24 ++++++ vectordb_bench/metric.py | 24 +++++- 9 files changed, 282 insertions(+), 18 deletions(-) create mode 100644 vectordb_bench/frontend/components/concurrent/charts.py create mode 100644 vectordb_bench/frontend/components/tables/data.py create mode 100644 vectordb_bench/frontend/pages/concurrent.py create mode 100644 vectordb_bench/frontend/pages/tables.py diff --git a/vectordb_bench/backend/runner/mp_runner.py b/vectordb_bench/backend/runner/mp_runner.py index 25e383bf7..3b99cfe6e 100644 --- a/vectordb_bench/backend/runner/mp_runner.py +++ b/vectordb_bench/backend/runner/mp_runner.py @@ -4,6 +4,7 @@ import multiprocessing as mp import logging from typing import Iterable +import numpy as np from ..clients import api from ... import config @@ -49,6 +50,7 @@ def search(self, test_data: list[list[float]], q: mp.Queue, cond: mp.Condition) start_time = time.perf_counter() count = 0 + latencies = [] while time.perf_counter() < start_time + self.duration: s = time.perf_counter() try: @@ -61,7 +63,8 @@ def search(self, test_data: list[list[float]], q: mp.Queue, cond: mp.Condition) log.warning(f"VectorDB search_embedding error: {e}") traceback.print_exc(chain=True) raise e from None - + + latencies.append(time.perf_counter() - s) count += 1 # loop through the test data idx = idx + 1 if idx < num - 1 else 0 @@ -75,7 +78,7 @@ def search(self, test_data: list[list[float]], q: mp.Queue, cond: mp.Condition) f"actual_dur={total_dur}s, count={count}, qps in this process: {round(count / total_dur, 4):3}" ) - return (count, total_dur) + return (count, total_dur, latencies) @staticmethod def get_mp_context(): @@ -85,6 +88,9 @@ def get_mp_context(): def _run_all_concurrencies_mem_efficient(self) -> float: max_qps = 0 + conc_num_list = [] + conc_qps_list = [] + conc_latency_p99_list = [] try: for conc in self.concurrencies: with mp.Manager() as m: @@ -103,9 +109,14 @@ def _run_all_concurrencies_mem_efficient(self) -> float: start = time.perf_counter() all_count = sum([r.result()[0] for r in future_iter]) + latencies = sum([r.result()[2] for r in future_iter], start=[]) + latency_p99 = np.percentile(latencies, 0.99) cost = time.perf_counter() - start qps = round(all_count / cost, 4) + conc_num_list.append(conc) + conc_qps_list.append(qps) + conc_latency_p99_list.append(latency_p99) log.info(f"End search in concurrency {conc}: dur={cost}s, total_count={all_count}, qps={qps}") if qps > max_qps: @@ -122,7 +133,7 @@ def _run_all_concurrencies_mem_efficient(self) -> float: finally: self.stop() - return max_qps + return max_qps, conc_num_list, conc_qps_list, conc_latency_p99_list def run(self) -> float: """ diff --git a/vectordb_bench/backend/runner/serial_runner.py b/vectordb_bench/backend/runner/serial_runner.py index e4861abd1..9e6818443 100644 --- a/vectordb_bench/backend/runner/serial_runner.py +++ b/vectordb_bench/backend/runner/serial_runner.py @@ -10,7 +10,7 @@ import pandas as pd from ..clients import api -from ...metric import calc_recall +from ...metric import calc_ndcg, calc_recall, get_ideal_dcg from ...models import LoadTimeoutError, PerformanceTimeoutError from .. import utils from ... import config @@ -171,11 +171,12 @@ def search(self, args: tuple[list, pd.DataFrame]): log.info(f"{mp.current_process().name:14} start search the entire test_data to get recall and latency") with self.db.init(): test_data, ground_truth = args + ideal_dcg = get_ideal_dcg(self.k) log.debug(f"test dataset size: {len(test_data)}") log.debug(f"ground truth size: {ground_truth.columns}, shape: {ground_truth.shape}") - latencies, recalls = [], [] + latencies, recalls, ndcgs = [], [], [] for idx, emb in enumerate(test_data): s = time.perf_counter() try: @@ -194,6 +195,7 @@ def search(self, args: tuple[list, pd.DataFrame]): gt = ground_truth['neighbors_id'][idx] recalls.append(calc_recall(self.k, gt[:self.k], results)) + ndcgs.append(calc_ndcg(gt[:self.k], results, ideal_dcg)) if len(latencies) % 100 == 0: @@ -201,6 +203,7 @@ def search(self, args: tuple[list, pd.DataFrame]): avg_latency = round(np.mean(latencies), 4) avg_recall = round(np.mean(recalls), 4) + avg_ndcg = round(np.mean(ndcgs), 4) cost = round(np.sum(latencies), 4) p99 = round(np.percentile(latencies, 99), 4) log.info( @@ -208,10 +211,11 @@ def search(self, args: tuple[list, pd.DataFrame]): f"cost={cost}s, " f"queries={len(latencies)}, " f"avg_recall={avg_recall}, " + f"avg_ndcg={avg_ndcg}," f"avg_latency={avg_latency}, " f"p99={p99}" ) - return (avg_recall, p99) + return (avg_recall, avg_ndcg, p99) def _run_in_subprocess(self) -> tuple[float, float]: diff --git a/vectordb_bench/backend/task_runner.py b/vectordb_bench/backend/task_runner.py index 0680847aa..a6d94f186 100644 --- a/vectordb_bench/backend/task_runner.py +++ b/vectordb_bench/backend/task_runner.py @@ -150,7 +150,8 @@ def _run_perf_case(self, drop_old: bool = True) -> Metric: ) self._init_search_runner() - m.qps = self._conc_search() + + m.qps, m.conc_num_list, m.conc_qps_list, m.conc_latency_p99_list = self._conc_search() m.recall, m.serial_latency_p99 = self._serial_search() ''' @@ -181,10 +182,11 @@ def _run_perf_case(self, drop_old: bool = True) -> Metric: m.recall = search_results.recall m.serial_latencies = search_results.serial_latencies ''' - m.recall, m.serial_latency_p99 = search_results + m.recall, m.ndcg, m.serial_latency_p99 = search_results if TaskStage.SEARCH_CONCURRENT in self.config.stages: search_results = self._conc_search() - m.qps = search_results + m.qps, m.conc_num_list, m.conc_qps_list, m.conc_latency_p99_list = search_results + except Exception as e: log.warning(f"Failed to run performance case, reason = {e}") traceback.print_exc() diff --git a/vectordb_bench/frontend/components/check_results/data.py b/vectordb_bench/frontend/components/check_results/data.py index 10fa3f459..c092da3a0 100644 --- a/vectordb_bench/frontend/components/check_results/data.py +++ b/vectordb_bench/frontend/components/check_results/data.py @@ -87,15 +87,18 @@ def mergeMetrics(metrics_1: dict, metrics_2: dict) -> dict: def getBetterMetric(metric, value_1, value_2): - if value_1 < 1e-7: - return value_2 - if value_2 < 1e-7: + try: + if value_1 < 1e-7: + return value_2 + if value_2 < 1e-7: + return value_1 + return ( + min(value_1, value_2) + if isLowerIsBetterMetric(metric) + else max(value_1, value_2) + ) + except Exception: return value_1 - return ( - min(value_1, value_2) - if isLowerIsBetterMetric(metric) - else max(value_1, value_2) - ) def getBetterLabel(label_1: ResultLabel, label_2: ResultLabel): diff --git a/vectordb_bench/frontend/components/concurrent/charts.py b/vectordb_bench/frontend/components/concurrent/charts.py new file mode 100644 index 000000000..bf886bd8e --- /dev/null +++ b/vectordb_bench/frontend/components/concurrent/charts.py @@ -0,0 +1,82 @@ + + +from vectordb_bench.backend.cases import Case +from vectordb_bench.frontend.components.check_results.expanderStyle import initMainExpanderStyle +import plotly.express as px + +from vectordb_bench.frontend.const.styles import COLOR_MAP + + +def drawChartsByCase(allData, cases: list[Case], st): + initMainExpanderStyle(st) + for case in cases: + chartContainer = st.expander(case.name, True) + caseDataList = [ + data for data in allData if data["case_name"] == case.name] + data = [{ + "conc_num": caseData["conc_num_list"][i], + "qps": caseData["conc_qps_list"][i], + "latency_p99": caseData["conc_latency_p99_list"][i] * 1000, + "db_name": caseData["db_name"], + "db": caseData["db"] + + } for caseData in caseDataList for i in range(len(caseData["conc_num_list"]))] + drawChart(data, chartContainer) + + +def getRange(metric, data, padding_multipliers): + minV = min([d.get(metric, 0) for d in data]) + maxV = max([d.get(metric, 0) for d in data]) + padding = maxV - minV + rangeV = [ + minV - padding * padding_multipliers[0], + maxV + padding * padding_multipliers[1], + ] + return rangeV + + +def drawChart(data, st): + if len(data) == 0: + return + + x = "latency_p99" + xrange = getRange(x, data, [0.05, 0.1]) + + y = "qps" + yrange = getRange(y, data, [0.2, 0.1]) + + color = "db" + color_discrete_map = COLOR_MAP + color = "db_name" + color_discrete_map = None + line_group = "db_name" + text = "conc_num" + + data.sort(key=lambda a: a["conc_num"]) + + fig = px.line( + data, + x=x, + y=y, + color=color, + color_discrete_map=color_discrete_map, + line_group=line_group, + text=text, + markers=True, + # color_discrete_map=color_discrete_map, + hover_data={ + "conc_num": True, + }, + height=720, + ) + fig.update_xaxes(range=xrange, title_text="Latency P99 (ms)") + fig.update_yaxes(range=yrange, title_text="QPS") + fig.update_traces(textposition="bottom right", + texttemplate="conc-%{text:,.4~r}") + # fig.update_layout( + # margin=dict(l=0, r=0, t=40, b=0, pad=8), + # legend=dict( + # orientation="h", yanchor="bottom", y=1, xanchor="right", x=1, title="" + # ), + # ) + st.plotly_chart(fig, use_container_width=True,) diff --git a/vectordb_bench/frontend/components/tables/data.py b/vectordb_bench/frontend/components/tables/data.py new file mode 100644 index 000000000..96134c7ff --- /dev/null +++ b/vectordb_bench/frontend/components/tables/data.py @@ -0,0 +1,44 @@ +from dataclasses import asdict +from vectordb_bench.backend.cases import CaseType +from vectordb_bench.interface import benchMarkRunner +from vectordb_bench.models import CaseResult, ResultLabel +import pandas as pd + + +def getNewResults(): + allResults = benchMarkRunner.get_results() + newResults: list[CaseResult] = [] + + for res in allResults: + results = res.results + for result in results: + if result.label == ResultLabel.NORMAL: + newResults.append(result) + + + df = pd.DataFrame(formatData(newResults)) + return df + + +def formatData(caseResults: list[CaseResult]): + data = [] + for caseResult in caseResults: + db = caseResult.task_config.db.value + db_label = caseResult.task_config.db_config.db_label + case_config = caseResult.task_config.case_config + db_case_config = caseResult.task_config.db_case_config + case = case_config.case_id.case_cls() + filter_rate = case.filter_rate + dataset = case.dataset.data.name + metrics = asdict(caseResult.metrics) + data.append( + { + "db": db, + "db_label": db_label, + "case_name": case.name, + "dataset": dataset, + "filter_rate": filter_rate, + **metrics, + } + ) + return data \ No newline at end of file diff --git a/vectordb_bench/frontend/pages/concurrent.py b/vectordb_bench/frontend/pages/concurrent.py new file mode 100644 index 000000000..0c1415efc --- /dev/null +++ b/vectordb_bench/frontend/pages/concurrent.py @@ -0,0 +1,72 @@ + + + +import streamlit as st +from vectordb_bench.backend.cases import CaseType +from vectordb_bench.frontend.components.check_results.footer import footer +from vectordb_bench.frontend.components.check_results.expanderStyle import initMainExpanderStyle +from vectordb_bench.frontend.components.check_results.priceTable import priceTable +from vectordb_bench.frontend.components.check_results.headerIcon import drawHeaderIcon +from vectordb_bench.frontend.components.check_results.nav import NavToResults, NavToRunTest +from vectordb_bench.frontend.components.check_results.charts import drawMetricChart +from vectordb_bench.frontend.components.check_results.filters import getshownData +from vectordb_bench.frontend.components.concurrent.charts import drawChartsByCase +from vectordb_bench.frontend.components.get_results.saveAsImage import getResults +from vectordb_bench.frontend.const.styles import * +from vectordb_bench.interface import benchMarkRunner +from vectordb_bench.models import TestResult + + +def main(): + # set page config + st.set_page_config( + page_title="VDBBench Conc Perf", + page_icon=FAVICON, + layout="wide", + # initial_sidebar_state="collapsed", + ) + + # header + drawHeaderIcon(st) + + allResults = benchMarkRunner.get_results() + + def check_conc_data(res: TestResult): + case_results = res.results + count = 0 + for case_result in case_results: + if len(case_result.metrics.conc_num_list) > 0: + count += 1 + + return count > 0 + + checkedResults = [res for res in allResults if check_conc_data(res)] + + + st.title("VectorDB Benchmark (Concurrent Performance)") + + # results selector + resultSelectorContainer = st.sidebar.container() + shownData, _, showCases = getshownData( + checkedResults, resultSelectorContainer) + + + resultSelectorContainer.divider() + + # nav + navContainer = st.sidebar.container() + NavToRunTest(navContainer) + NavToResults(navContainer) + + # save or share + resultesContainer = st.sidebar.container() + getResults(resultesContainer, "vectordb_bench_concurrent") + + drawChartsByCase(shownData, showCases, st.container()) + + # footer + footer(st.container()) + + +if __name__ == "__main__": + main() diff --git a/vectordb_bench/frontend/pages/tables.py b/vectordb_bench/frontend/pages/tables.py new file mode 100644 index 000000000..a4dab68a6 --- /dev/null +++ b/vectordb_bench/frontend/pages/tables.py @@ -0,0 +1,24 @@ +import streamlit as st +from vectordb_bench.frontend.components.check_results.headerIcon import drawHeaderIcon +from vectordb_bench.frontend.components.tables.data import getNewResults +from vectordb_bench.frontend.const.styles import FAVICON + + +def main(): + # set page config + st.set_page_config( + page_title="Table", + page_icon=FAVICON, + layout="wide", + # initial_sidebar_state="collapsed", + ) + + # header + drawHeaderIcon(st) + + df = getNewResults() + st.dataframe(df, height=800) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/vectordb_bench/metric.py b/vectordb_bench/metric.py index a2b6d6ff0..5c23072e3 100644 --- a/vectordb_bench/metric.py +++ b/vectordb_bench/metric.py @@ -1,7 +1,7 @@ import logging import numpy as np -from dataclasses import dataclass +from dataclasses import dataclass, field log = logging.getLogger(__name__) @@ -19,6 +19,10 @@ class Metric: qps: float = 0.0 serial_latency_p99: float = 0.0 recall: float = 0.0 + ndcg: float = 0.0 + conc_num_list: list[int] = field(default_factory=list) + conc_qps_list: list[float] = field(default_factory=list) + conc_latency_p99_list: list[float] = field(default_factory=list) QURIES_PER_DOLLAR_METRIC = "QP$ (Quries per Dollar)" @@ -60,3 +64,21 @@ def calc_recall(count: int, ground_truth: list[int], got: list[int]) -> float: recalls[i] = 1 return np.mean(recalls) + + +def get_ideal_dcg(k: int): + ideal_dcg = 0 + for i in range(k): + ideal_dcg += 1 / np.log2(i+2) + + return ideal_dcg + + +def calc_ndcg(ground_truth: list[int], got: list[int], ideal_dcg: float) -> float: + dcg = 0 + ground_truth = list(ground_truth) + for id in set(got): + if id in ground_truth: + idx = ground_truth.index(id) + dcg += 1 / np.log2(idx+2) + return dcg / ideal_dcg From cc7bc07e5b62ff6a3fff733dd08444d742057334 Mon Sep 17 00:00:00 2001 From: "min.tian" Date: Mon, 15 Jul 2024 15:11:13 +0800 Subject: [PATCH 071/327] support custom_dataset Signed-off-by: min.tian --- README.md | 18 ++ fig/custom_case_run_test.png | Bin 0 -> 111864 bytes fig/custom_dataset.png | Bin 0 -> 76838 bytes pyproject.toml | 1 + vectordb_bench/__init__.py | 2 + vectordb_bench/backend/assembler.py | 2 +- vectordb_bench/backend/cases.py | 82 +++++++-- vectordb_bench/backend/dataset.py | 32 +++- vectordb_bench/custom/custom_case.json | 18 ++ .../components/check_results/charts.py | 12 +- .../frontend/components/check_results/data.py | 24 +-- .../components/check_results/expanderStyle.py | 2 +- .../components/check_results/filters.py | 33 ++-- .../components/check_results/headerIcon.py | 2 +- .../components/check_results/priceTable.py | 2 +- .../components/check_results/stPageConfig.py | 2 +- .../frontend/components/concurrent/charts.py | 55 +++--- .../components/custom/displayCustomCase.py | 31 ++++ .../components/custom/displaypPrams.py | 11 ++ .../components/custom/getCustomConfig.py | 40 +++++ .../frontend/components/custom/initStyle.py | 15 ++ .../components/run_test/autoRefresh.py | 2 +- .../components/run_test/caseSelector.py | 68 ++++--- .../components/run_test/dbConfigSetting.py | 6 +- .../components/run_test/dbSelector.py | 16 +- .../components/run_test/generateTasks.py | 8 +- .../frontend/components/run_test/initStyle.py | 14 ++ .../components/run_test/submitTask.py | 2 +- .../{const => config}/dbCaseConfigs.py | 169 ++++++++++++++---- .../frontend/{const => config}/dbPrices.py | 0 .../frontend/{const => config}/styles.py | 0 vectordb_bench/frontend/pages/concurrent.py | 29 ++- vectordb_bench/frontend/pages/custom.py | 64 +++++++ .../frontend/pages/quries_per_dollar.py | 10 +- vectordb_bench/frontend/pages/run_test.py | 4 + vectordb_bench/frontend/pages/tables.py | 4 +- vectordb_bench/frontend/utils.py | 18 +- vectordb_bench/frontend/vdb_benchmark.py | 6 +- vectordb_bench/models.py | 12 +- vectordb_bench/results/getLeaderboardData.py | 2 +- 40 files changed, 611 insertions(+), 207 deletions(-) create mode 100644 fig/custom_case_run_test.png create mode 100644 fig/custom_dataset.png create mode 100644 vectordb_bench/custom/custom_case.json create mode 100644 vectordb_bench/frontend/components/custom/displayCustomCase.py create mode 100644 vectordb_bench/frontend/components/custom/displaypPrams.py create mode 100644 vectordb_bench/frontend/components/custom/getCustomConfig.py create mode 100644 vectordb_bench/frontend/components/custom/initStyle.py create mode 100644 vectordb_bench/frontend/components/run_test/initStyle.py rename vectordb_bench/frontend/{const => config}/dbCaseConfigs.py (78%) rename vectordb_bench/frontend/{const => config}/dbPrices.py (100%) rename vectordb_bench/frontend/{const => config}/styles.py (100%) create mode 100644 vectordb_bench/frontend/pages/custom.py diff --git a/README.md b/README.md index eeda249ae..120a2431b 100644 --- a/README.md +++ b/README.md @@ -281,6 +281,24 @@ Case No. | Case Type | Dataset Size | Filtering Rate | Results | Each case provides an in-depth examination of a vector database's abilities, providing you a comprehensive view of the database's performance. +#### Custom Dataset for Performance case + +Through the `/custom` page, users can customize their own performance case using local datasets. After saving, the corresponding case can be selected from the `/run_test` page to perform the test. + +![image](fig/custom_dataset.png) +![image](fig/custom_case_run_test.png) + +We have strict requirements for the data set format, please follow them. +- `Folder Path` - The path to the folder containing all the files. Please ensure that all files in the folder are in the `Parquet` format. + - Vectors data files: The file must be named `train.parquet` and should have two columns: `id` as an incrementing `int` and `emb` as an array of `float32`. + - Query test vectors: The file must be named `test.parquet` and should have two columns: `id` as an incrementing `int` and `emb` as an array of `float32`. + - Ground truth file: The file must be named `neighbors.parquet` and should have two columns: `id` corresponding to query vectors and `neighbors_id` as an array of `int`. + +- `Train File Count` - If the vector file is too large, you can consider splitting it into multiple files. The naming format for the split files should be `train-[index]-of-[file_count].parquet`. For example, `train-01-of-10.parquet` represents the second file (0-indexed) among 10 split files. + +- `Use Shuffled Data` - If you check this option, the vector data files need to be modified. VectorDBBench will load the data labeled with `shuffle`. For example, use `shuffle_train.parquet` instead of `train.parquet` and `shuffle_train-04-of-10.parquet` instead of `train-04-of-10.parquet`. The `id` column in the shuffled data can be in any order. + + ## Goals Our goals of this benchmark are: ### Reproducibility & Usability diff --git a/fig/custom_case_run_test.png b/fig/custom_case_run_test.png new file mode 100644 index 0000000000000000000000000000000000000000..8817b3439888606e12e1d236e9f701cdc26ea910 GIT binary patch literal 111864 zcmeFZXH=8j)-~*Hy+OgFh=>TNH0et3*bwQRNEhk7cLG*WX#xULBQBO3Sb7zbx<%m(=zk56@P+noRVW-j0DyXQl3ol$vD zpI*JnDL5MQ_6ZxS@Wkn(*LBY68yDO#dF%hLM~M*=z8d>;Ya331j)R|q1GgXQ>grwu5B=w-$23Qt^Pe~LKSTav2>oC0XPzEc z{$FoBZ$IbyUvFiv-Z}d}Z=Z6!`*`Yq-hN?vIDG7X-u^ntyZ+bzy!AeIe(!(Y9clc3 zkNxjz{9jw-t2Vmy{d&4_3m;QuQtNy!U}N}!TFk}Xkw=5DgwaJl)!uISN7CL_V*ckp zqr0v9?+_Y9Up+FTB4*iG+}POYze3zuG3rg?^Syfg%V#%KC3>c5H!7erhRx{f^AodI z&biB`?n{Upd^RQ znI>T?;^4eYUw?nU=&dZpAnS^0e=W$|=X(}ygss0DzW@35BTWoW!^gbSPqW!+V`oy% z9F^b^QTo<8+R*`hc0_Dv8W(w)f2}2O^Vh_h515;Od3dmsdPd5zB{u3b3CRpR#M%??4lWb`j8Bco6Sd8^F=k45<$BuuF% za6;~eKhMld(@bN2SLL-W<-ouI3D{e|l}1=uSjcr9FYQ4hgRQ$?on~Xbd^t6j%qn86 zulDwWLE^VpE1TK#Cb{1j)Z?!p=T9+XHYUBZ~`}CXXLKJ$k5BqMMx& z`f2i5^(5npf8&uO0(=4jZllZSsDNW+)zww!f^KdcoaRXuwIYs{NaE242{T;(9OmT$ zrOb~BF)_#u`Q1AEUOmQy*HZ2*=Y99PU``Rr>N1%+3AJv8&yIx(-@N(dcgDFYM3r5) zuD#Ze2Xman421yqv?7t~*A1a0rh!!IT8SQNf1j&P+>kCj?9q)yu%{mXb(&2S`Aw~VtjbG|iEBo^^XIw%>JRB(i5cv->!<5ZnTwnrH@KQ9lzS&pk)?uOYnOe4pmUuKPiEi4kmPUe?I zDJwq$TRfQhp}Ie#jsxF&Fm!>HHG9F09&;ofqNhu$h0db3=ww z&`C-XpYY!DFq>pn7Ge@?^mrNSUz{r8V`_;Djtl1!NxwRLFk?*(@mG8(sT2+wPt)!} zsFc&z(T?TTbmHmMVy%VTMVZf>7$BThu8YYxI?pk1;URqqeid4`&9 zC|q?jtuU_&*}Ygq#<`!2pFZL#aEhkveh`!#xM{w&E_Nem#=wS&V?qnk?$9sOZuIA5 zEenPS-9fk@O`%e*t^HeUT<1!15}0l&L0BtUz~7&}fJx6(%1TyRwGs^poJmcvg zf&SzBAV>a1#Ndn?^Yrmkr_f&=&)&Fs^Y)3;!NNic=BJpijIP>5NVgR^#5iK++K_Ru z{`U-qi*@&-p;pb|lfEt6J1cz(t`+$n`ss zxA-6Ut%-hemkb?K3@u)H5v4I93|XvQ?q0b}Lyt+Vtn{ahhvDKL?k3E!sB*3tYlrP` zp?+N(VxKUnFgM#pz|dn*i9W{~bT?m3&B~mqd7W!znsxY8?VKkfPbag%b$gL@swN2G z($chh(Ruhw_ww3WW{F(idxli)^Lr5y?V+ygx9dE1--U;Vzt74lf8FB;z9PFQe7_K_ zOu(srWL2#&Skq0g7b@ELO~ysK4q9Rft-Q)A1&Gl7jlAkY~G8WgAM=R?b zBc#nh;`W#f?C7$Qm>t6MSnH$2HjydwLG#My`%71prz(=c#CSEhxVRX9ix=KGMs}tW zBpqrFtZ2lwIqmLcS2nvRd#lsjX7$^e}^DaM;FPM^dla@C(E3B1UT(&mQjLghQ z{?IJzaNFX3yOR=2Gfl-=%zyPr)8WGAbfg|+Yi+HpIly)0_WiD{HiBozPGP^P(5;be z=KFv4+PkcuMR=h|@J3QI{I}&N+)xKTL>(ZN`s*$7+=v>w?f4MT8_(PT1q|9{U6Fho7 zdE4j=mL?y319_`yYP3*TG(qogX6Dg7aTODNeHZu^q{cB|AXD<*>5F1U1a(EnP+{8h z6O0AIiV=qe_0;Bjxw#9BOs9=}4XaB{E6PFsE!9lXEY)w%>W^nWAb+_7O3}k>*K8RX z%bE_29@?S;aQ;4f(W+-6yZk&L3Et6zhZa1En!9 zaC54xLp@$lNXUF`P#wLq^612gRMV9{R+Ew^^5DN&7iy9Fk&0D) zDrN89>4$`dmfmMjO}Vj92Fm_ASeN$$rZ==GynKA{7EXg;x-1nb02jL1H5N6U>Ud}IcsMiJeGKxNG9J94hRkguKlxG4g20zbt zC^0^t(?*uIU1Xa=(4KbG1N;RI4T8|9%95?8d!8MmVE~wRTPY1-o|bWcqz5JYLu!f$ zP9aKsE<5kp&D%oTzVTe=tcN-k5Pf+3ob(_A8ROYo4f?rQ-0aJj$DzBMN&`7HUw+Gf z3h$&*59fN$VSawDKtg9k1iB{^PqT@+ZZ^l6H##@Pb1K?S1|GtVOj=Bt+O|nrr6&E` z9|U+U$4ketwT~Ah;3_qvcm`wcgv%vwS^E{oRqDXHXUJ@Fz`M+nz}{Eb>M;%emXwRk*`1KL9zo>;-TT4|4>MapO!$>y!Ta!MjPZO;@>9y@GBH5ey zBdLd_RxMGshGxy}qr*aXq?Y<|QXV9fOCLQWai>*|Hq3|Tpn3gW0lFzqhU;cd1Z_^; zsdF3e@l{PX`1JTMK0ZE|tyZ;+n?9vRsEx@M8|5&j2`8g2$KaITiTU05(S9#^?Wn?f zVwz1Ic#YS(*g=2Kaj;vvh-%Og*7iyeX{XFg*M^0awPHwxrU`>mQ@Rq!`U9D=KBTGs zG&|{?*^#Nk{wiG?yM}4}3Z$ksxYxJsa1rT>T)h_kX7>B{`#M=tR-l44We^HA90H)25ayYq`vnpZQi}?_9%QKd@;^Xbz@n>wJi&HJQ zD*s;Mg=^QGq-)GWGlEw{dX+ zqzq{{*;4hPJL~}V)&uy8-;jO;_CG5Zo0l)X-n6D}FV~bZ^&2ujVlM#G06(GF%>;?} zS$d)%4SFS9Dg5hLbgw7k$&{D^>3Nhg*)=MutE9rZ=FU*t&Fl9N9W(a3;?rX#THNsD zUBT5si;rGTlwbbj8KprVA0JZikNa&qQ$1(+){HV0y`GWwIWvkxZrw5f8>~-`Pe3o- zVgcx0Z!_OuN!n*r3O*GfOO6(S>cWQ(OgR+^6S?=)MW*R1-$qtRFL}cU*7;seu(c|! zwbOPQq%wA0u9``Ac7^5+t0lvdq;329`AC{168eYM`A^2|KYq%5VkomgUr&jcH<-iZ zXLR42S4w22c2M;TQ8dyDB(vS3fI`ibn-yf@+MWfmsNQ#6Ax-=}q${-!>x;RQYVN>^$kF5vAZF!2EO2oZ%9q)wL}6MUV;MQB0`# zd1{aE{QO`B{Fa@7qPr{zwRA3IW_m?)JP1CPdp1(D^54u9gSJKsdL^s<6SAJjG^@}4L4_f|G-k#`}YM9n=n+#(&a0Fx3J29v7E1T1qrI={Rcgd{VXDL!ktM_qxp4m za#q>_$~P%^O%&sxhwAi2bx}EN3hIHvjQna(qFvNVovW#ztNLFc9ItDrsHgxSqzU2T zI_E-EKK&;i8(x5%Q)N;XwV-+&Uhjm9sKfo#%738e9b6<<4I9UObf1)<$7Q65>9G0c z*smX9LLy4njcygbe$!;1z6=_A3MfLlnaV-Wl9G}{rnBTi`uKW@W7eG8m5WPDob_b? zZ)^(Sr1g`mYJ<}!DyN;zVeh<6B-?AP(U^bZF1OyHC+duu14W?4b3Ja&< za|juM!D<`uAu0*`v?H(~8N|8A&riL!Eg;%#2D0+$WI;cn3E{(o$4*P?nC0F*f&cMI zpgJ5k<~W#TXQt{kcfCIpeDRd-VVmukx>shUu4(`#Rx4>&unQ{DSxvHGeL zVi#YXd0niXqW3SGXw-iTu!vsP9UK~65&;ceU;50#Sx=wCMN#V_@>Yv_Td2S35-92Z z5#F9o4bydw6(OTT+PkN?^`9L*QGG3M?}?pVo%1#$lJx5;mh)V7Kd6T~EWf^fy;H=X ztFa^YYx6htL@Q^*=SAle7${*jxscqFG2m#_&Fo>!FZYm5tZThto;FOStZH<=dsj0> z?DmvyUeoO$fetP%m;#N425ud`2CRW-BrQn+;O~g(CM_%~Q$YVLw@Rs=fc7{?6~cbX zkzVP*upQ=uW+!j!WX@HYOP2|Paz&-koIBr+r&`z`wpXJ|O=_b0Qkiri4Pgn}A;?iY zN}k@*{Gx_=mFL98D#0n_v(A{>DkSf7^@PxFsr3EbAqx?kv;jw?qSskSe%O}b_G0G+ za1H<;<@V!)MiD~?I0J6`L4zLmB5G9HTKZ+j?O#Z{$$V(XWh!~Y`cfrxe%J*VzmnW) zu6cG@y4h%M?u7s0frhA6{b)7i@RinhrIj0!G$S)uLJ6Ao(%rl|41f>dX5vdt%E}L3 z9;e_@cSOx=_qxc_LqibmR$7#jGIHqj2IY|~l-7~wkLU+Aor#iYhj+N8=_Euy_$ zf#Y(KO(bPt=h9OE@?C#D?KU3#Md0S(aFd_>wUU-&0Jy+q3ThVyd2~I;i8{fXyiN`)-oh)W&8i0?_g#T z6s-56Sxq<4D#7>xjxL*O@UjaLBUSK93df?V$Ri#Mq=atmY9+;1^Eyq>H%pE-P>*(@ z-Ne8Ok*W&oic%R5iTTwz$C?s*ja|{5Y@e<0_YTmfk%OE}-Zh|wTHn4#F)r5s%Hg2L z#91s}1g%KFxD>wg-357^5*3GdRQZ?yda2ryC6mHF+MFsLJ8l4K*w{gP3!Zw)PN&MM z(K($&z`8X0N(l;Tun3q7+~vlO>vY&e@aygzQ}%u10Ykvl2tT@jNvnX}3T`>fu2$NNAN@vx;!HG*WbBSkm#^syN z9~J_zC@&eE;qLF>FQumW`F67h#r6*Xio@*@7mME{I_ytU|59;7&)r9y-Zh8r#-(W| z8#n^EOecZrTxZqj!?(TYl+WYF%%xP27LBM#IoVru0l?I~$2F&BE{z9t$7g@CbeILq zWIwjW-_xl{8qbnI==XOQEFat)(CH-#CU>tlIl&$#XQfr*{I~avUWnS&nVU8Je1bXw z@~szws-)wqQm4x~?ee1=XV%r`JASfbN0s{1gB|shnt&dM^rRr&zNiI6eF?QvW#D0A+J-$kin=@ozzwAivn zR;x+&&z*|A@NiB9MDlHQBksKjwe zkJ{#a`7!DsJYEUagYFX{&His#t=WCn=XpT4bUT9v+I_2-i{eNpSY)A7PM8iIEjgB1 z^1L{S_aP@91E?;w>>+3>H@s6^!6}noK+-PdoS-^4=z^GR`D((>!tRpuaR`G6hkOEI zuNc<6P0||o((a^qy7te{p}GaBvrA73 zyluHVe9L;(|7?l3VvnT74E)n1Bf1~WQs;^f1+B`!LeuPncB*6iL?zh_rpN9U+$B1y z7=o!q*S8Ipn%7kWYG*0$brzxHkXsazr?MR!3E%P~STWVvj|5HCg;mjMp**!1K+$06 z6r|Q-ZHCmM)m68(>6W`qz7EEK8mZFmpE!Ndf9B^aW@^|YSgfonH|ewWtVn|Z!fYf@jh@COub z+XvK{ohO^F;cP~02c2K}d2%_`u+jvhm^nY7!<>u(o}hqE^`gk5QrWpfeXHhuq!pp| zQ`TO<&X^03$l$`_kjnA}UH*TA0xRy#s$4xpxlU$129OWzZ}EF*$)W&9^}jCNQdCqS zeLJz_o7@{E%gy&>%0?z3=3LDFi>Qxch(fqu3Wnaugg#- zpX#qE@Nxt_`i37*?UTt??Fz#IK~W2-SX0j2f4l(wzoqAhyH59b2TP1go~ytlO!-8T zPEB5;=WAD2G>H)fhe@agP0eb)zsU^);ujx3e={Zc>5m(pzH{}weAbe4j`C7J4q9C_ zN-djOkbb!P;@ffa z_9u0&W5q>ON@%G=Oi>^1*RDIhnU2PRIAUQ9`{vn@Ibqdwe$9X9x9P%~8sr=sSsMQkC0?LRo@!$wlDKKxPj^3UP zX14&Qy(|J6R_wQ?Zd_hWtPy-QUQ;}se=O^?j^j5~762V)K@6I!x;9MIlBR&Q8! zIM%q#{3Q4(oPqyzmudTbs80{p%{ME6&}x%96zyj%#Gkw6en$qfTEX8prqxQ_XQ`x$!cfEOPaPXF;2^J47Vx27eAE)Zvg ztnbUppHDG%KniDx{WFx*hHD(}t#XdsPbI9=2YEr-Y|Pn2Z>okGIH z3oqi^i9=#;B$xLM`rLZ5-xe|OHH7fb5ijdcwwTG>2DKhgd~RKPZ|DUHh!|UA0f6fs zf5QT=mTzv&2PhzvJg{I+M4rV^odp^M)__K|H`tOd3h__@az=)f%SKo66=8!>g*&js z*=wHCrYwR-j~+_;SB(}9Oh>;`FCb33p78hfejG(=lP5+<=C{ASAXu)!qFx5XSzSVM z(j)9FK_gqv<#5KgR`RK^z5UL&O(EF?zh(Jd);;z~_HSk&d4@VI-@s!i!S(!Wh1l*H zCE84K=n_6n!he>aDgrdgwmLw&AY~?@;V&j}Narw8tnS}EVA|*1&cnvVg#MYe;?;Xk z@*D>+mnBJ7$|{Xj{Mv`#78u8&WLo`9o$hmrDv%@Ye1ignZ^m~WZl#R-Qrab z8_y5mbX4RFo#`SrgZ`Tyhnw?}U(jxsWy;|~DZ;J`c@F7Y4dnc6<4Q|*oeU+bQ)jt+ zNzH6F5k?(0Tt$F=vv0=zXb$P8UH}FR$-|BA?(VYN&yE7Z-fGF0=v!_Z0{X_REtWqE zDoIgoZg)GPY&kHs0xhL)OC8}2}?OS{i?VSNX zJ}Svf~4GWt&k4a7D~Mb*XSu6V&Qj=sb{yw_4LHd?j~amEDsx?9~+mikVGx&%{=3*0ZeJ5 zOn6Hd+sO7q)3<;B)~B~m;59}U4VJr8QdxEkwKJ)8&HkQgtAtAzjHmX@YJ(TbMYzxY zLc6qL?v#@@-45f#BA4tEq~)fTNy*M88#);}Q$8V?&w??oqM5SDC&28Ien zOxd+Y->?W|Z;^Wvmr^N?_NbG9gqQZ5c?gTAJyaE?-Kj8_Zg7u0_-#Ph9io}4Sybdu zK?(>74({pi7e><3`Pg?e0CqC{tTFmEGMi>;W>&o?fNZ`jUFx!$mRtj3wFby_Zr|O8 z+_2)p(tEwnn5U~OXnq+mEeoZXK-^qf;XFJoEe&08+9tVRf2BW-v;Cx#eSl@`hfdSl z*W330FbR!T%OxVc<+yfKtT@8x?%lhqo{xPV)aKit^sPE^Iu(9XiNUSZL% zbI!qfR-tZ`FVSck2+Saq6paY+adG27n33c#uNQ)d{jBu_E)~Ypr-Lvm95U__%GYlv z^`+{xkj6(CN7P~sa;b{lB`I(}!szMkRc z=4OKWeoPgi>Vm8%;}JyipBmpW9m=y231sXk%~Ol(78LnWmwZZ6QS8CpdvSKcrsdv6 zOMe#Qsl}ZfjbU-Cdc$OXJyb2)rYvY)0IzqFg&Rm0q}`#mB0SLM2V7tG{6()a-F5=z z^&z;~bFMpIpkip;9{OKEK)OxumFb`Ff<+n_8YXFNiCWpn&wjR3?3eLsB|aO>>ix;y z+noS2o{!LT`TcG!Yf55v)?D$?wRZ0j-)I7zT_~U2)xo}a@xr(zFiy&0Zf*`UgFc6J zqvIB!Ln&;8^-p-Du9>QiR;F}>$@v1#2#Y@_o-qN}U1k+|q`x^;C*wh0P>0>Pw&0p- z*C$lQaK2Jk+c6qM6ac(qKugS+z|BYAwrKDiGUUE>D-F1GJRU*735!lXK9hTw=u29E zA*+MEcMqvcb(ySAcbTU1QzWKLtBV${j?+K672N$Lpc4-MAf7hAy|YM2{Kpmv>1AOc z%}3CTxWf++2IYPF1{E zNL|QQFhi*F>t;C0Gc1K0*YfM-*qVC*5iT!;rK4FbCe5XXWk`|IQahDITOA+foKkh4 z+W)jY44@VofVQ3Hv(FFOxjh7K05OP)_ z+CTV^Za{?<#iautdr2jI@sk%1?Qpk>Yx9=A8_L7qmF-9wkCZ3X9Rj>wzFJg*>Lsl4 zjzT`buC9-$UMJ(P)8W9AeeFu?*@L=2W$(-8V!hrlw{oCmhfPTLC(o|5nz0(x;{lsb z_km6n~Hxw1h$eiQ&Lvwh^@ou%XEp8nfUv{R{mo%LPvKo~f2S#eOI3E2`9#B}+m z`Y7IBAB+^<)!hRC)ZTuNK~w-8*Y2^;J8qsRG_1B~fF4jeQau1MG~PhdkE8jQ(zVc^ ztG~Y9Bdx2#*6YsU3d_RwiRX%7=~<(NT4DX9ZdmZ0mLQyDc_tBW8LcQ->*m(_w#|BW zDpk>0#OAZPS1UfZ(ppEAeSKr{CexL@*p+1%^X7E%Nv7wu!l)I@IczHsAm)E%~Z5}OHwH?epm)?QtF>zF~2E!EMUGG8Gq03VsK{U7vYe>79N{Y-k z(7d8inp^B?W0lrzu5u&&fQFsXQGfH_edY%y*LT@sHu2*%L+y?sMSHzQaB0U4`#5$* zvlJl{{d9>tLe}LU=!VyHNheVjC&81<>lN44JmVydHUnES8a6%@h8h{f z3iSs9QEqr9!#68Q8%ne9F8c>^e_=E%0d}-3*lg8b=3W=1@qec-e78Ux--=czrfy6rU^ZE`83Dlh}4tgI}(SjQ%{h~Hz8G8D0_)fR?Spmv8; zN;0I&O2UcNz-o%t?lH?&33#AL`kKR zrWAsxZ&gwbpcnFevRK=}H=`ZA=vK8{^>t9KK>f82-F9iSlW2O_Ol=BXorH`pKO-AA zcdu=}IvL7N5mB&yL2x7l2WmWkRRcqEj>K<_n4E3!tPI7w&IdD3Z$fIl zM-2Oq6P1JJ6Dt7$keWfJTt}iG8F($RH+Ewc+-#{}ucXReLgB&Clv+ZFc^1TpKC zEA~V#*|!yHG5oF*dmz4lF*b!{aRcI4G4L=qrL0_l-GdU)Qcrb-@WTL=Zz-fWfR2n?Hz+un_(*{6n5fJpK-Apz4x04keFyUGvMICCXA zk%V{SNd1FnzHBbnTNf<@x>_8rM<@5}Ex3*N6ATz)M4u+MY>=_Hb)$dD0FjrpHC@s? z{Z+H;N3m5|l%qE6v1L6B-nWjTGj-^I)tmYf4p|L1J zMi&~|BE~QQ4y}~O6kN!(wzz)_tx1seED$e>AKF=K11BPb)}h_Oy72%+=-%Y=XIE5Z zKq@JvU_tyW5~;@|Q7#OrjBTg7zI8>FCay&Ay3myxfAkR~39RvY8GG^=&o0T-#0F?G#EO=kyH{k z=`MrcTso}TZ1F-217zn|T9rx1z!^XP;IsE;6k;=aFHz{WvM3PSwJC zZOcbPTYC&xmYA>rW~?U3{W-i>Wm`5`_5_;)U8=@~*__vtyELj|ol%u_7say*6>&G` zB`!YPr0a%Q@;*AzV{I!mB*f*Qze8$ui2%6dSK9O4Kw1S+qnFGd-|iFT?>WAnqgWTb z(2%bCBnYIF`5#$%K;D)d^fCpGPw3v)%uwj}m~;#Ze9R_NS`GL~-$h49yR@8S!njT} z7~w~vy&UA^v-7~zN$xwB=pAviM-1R$fcjr!>4JL=gH=6f-EWmCx47=*h0~s2b3DO# zVOUmDwSO413mld2XCpctZZ&EG9u;`XbSUmwC(l;T)z9|j&nmdg&4{yJFjmkZ0+q=@ z?#>}#b4e~l!exn{#Ng-8HQn}#gZ2hcFbC4)yXvLyK-3BWBG)1F-lAP}9GnW}ck>~IV9$CZ0pDcXmWZ3B6=3|M)c zCwELpw$udXhF6sa`58dy%riYJ4avQW9(pdj2<#4l9u_5PyZjl5U+NVHE)_;jGoVBR z+nEu)b%ED!=`DauA;4UV&P_e*I*wtYC)<@toFIVcQqEuOT<58_Y5cK9!WJ@*YTAjG z+zIIg;KG2SyA5bcfX0WNu>V-YA_(*yzyPBsowy+Ic#k@FqATTQ zAUgO);QUDfe?Kj~iY?*oWQq{jpn+@CO;*By^y z%B(x+hGoqe*xsgfOF2LGVih(2Ua~Jp4*(E?b-M|(=FR>oz>Bgz7T__hRB-25n3FR+ zN4+Ir_@|H2Xo?qg1%a8bg0IwVT~5=1TwSN_AITV$*If*;*eiOF=2hZD}` z?d#WL#9af0ATiBXjI4&BE7pAf+oU9k@4}@==6KBbpuWWEo4%Mac8{r66XNa)B=~*- zW~`(nRW!qK8WSQ6T+IHYG4@wcYB!{$EP<6*$hi1pQgX6MsXarA2pZlT;07GC;pANl z4QJ;@s zx#!*{B%o4+eT1x<-yhXFReblAFUaF>wMd|~M^##d(J_SITAp#U|I@s@Jk~3g=D?)F zc;<}0{^O4wySdqkz>@^JFzk=?e*wrMBqCx7j->wGyU&4iEqLRGevnkTQxh~I5O{(L zmIQP$-`^N1_$=1GWfLzw&Gy8&x}fTiLNNiPBs@X1hshT!r)3Cak4+lZ>(wjzgtfu7 zf@Xg~(|=#vIv-JL1&;y=HaDD@1v?$QqM60asgk$Ut%q43N$!KG!jzPhE`VAFu4#x{ z#HTd*qpPVib?Q!oF9D}{16Xq5z@VEV=tuEmxpJZSeqT)un>%4Onu4k{xmSb8OwA2*=xziwYDqw*Cz{_D z1RAB))}>xY>`#mQbcT0GfPRh8la0qZQV%QZt_x}R?L52!^R5G05Yr_o5vvf`oU+v6 z%HGPKbm6VoCkbwP9XT?yr4T4JClmpWqh~yC8=orG{?S}6)J<0X4~um~KAL{p@r%de z+=HOrp&?l-Vyw86Apfc%^L_tW&>b)dFpL!&3^;U$|M~7O75}~;`NiW~f4>%mw12Za z6Y=;>WOkMA&rxq3JIf=1v~Olq?-SIzjYxRR=IPR^mph5wA^_S`qb zTQHRwM}{_|v!U}Gi(jv|8wHq&OMzje>3`i{`}ZZy8Ao6@hp**? zDCw9LYO?N42@f>|jE4e;7eM@LtiNo8bcldwSg61BXX007|9cM*OvxNb;FoWDe|URQ zxU?PGjHNGMM+cAk6MB^dqO(-k;8Adc%i0Tp*Np%EXL3j66qcfHa4N8zK%s_z;}Z!hd8CR99T zRzH5ECuUI1YPinrO;SNpL}!XP48pAbUY35H2gU>U$GA5xE}dV7v=+NUsy>cV`Fs_m z{*1EW=J`Ec5_WzmLnbVpx8%Xf&Sk3c(3SYVE9?C?O+6?cFy0c27DrcPtHFPsqe5If zm5ulpkH%%<;*k_4=AMPuJ2{e0JbUQ0(;&qt>x6vHdxh^+?C`zukB9@zlj$E3-rGs_ zIE9a&Uhlq@3%YZnb+;rmWvA4jq`WUpR8&tdkP6C1PYg)r(mtzs{=Fd@r55?%03ZL) zKz{`$!O!opiCK?sbYg39uzWOzs^Wh)>Ak9rin45DnX75VudUzj)J`$Oj6qp>%ZfK> zx@yXyiE3C=_CJi}6>Rj^AhL>Qbe#LEM;CtYTfl;0`A$fPBPmQoh?IBt+mKdLsfkGb z(G#s!>`!t6Tf+5LRtc|#pG=4SDi-UqY~S{SPn<2@o(;Hg`5YhZe5ch#;r|TmX{+E6 z@48+V15yAPp%1ppp`{F5rA4xz9?Qj3Fh4O7vTCOwA8y*U^Oor1--a%1Z^>WL+49D? z7w`{R@@ilvKfmH~^#%KN{Ho%Bb*pZ-UbIkTdw7NdX+?Y14vQn>kx?E=1q5sM@?TGr zD$m@~Yq4sF`|gH)XZ@UghM{G9&G+kHuJfgzd2jXp@yR^ZT7GjJy4>U(mm{GL&bVyf zR7hBc6?!omn8`{`5GB>HfPT;Pukfr_e{5NbNCgl`O-RL7<;>2v; z^pli>J?&$r(P2We2zASI(@n)OvQ*0$&Xhl~)A-W^_EAqawJ4%t?}6_eP*B8`NA5|N zzHb>Rdi?chGa=sGb?j4WYSnb9>OQp$P7WF@d6&q*n-|BP4Lkj7KVb3%EF_0xaw(9M z(>H)S2l)=}n$&Sbigl*$|9OPv2B7=A$aaXqmk;I+h?93_M|szbc7MN9P@cJ;&E@$d zMDJ|EvXrV#d|RF!{H{q}ucXWbPpWR*_;^2xdW$sb&~le2?*h!*WaYVL$)k~P>~Cka z4GpXB@(f{Th%t;e%q*#Vlw~y9c1neRg7PNe;NbYH2M^HSI{O*-2+5Ias(*t>{?G3P z&`=He5dA^7Tv-=>_Z#>~VfCFtdtu|@5-e~5`>SB`z71OGe?2-U#Ae*Lr|c&`f!DYI zX>~G+QHs2idv1+&-R3SI<*k$RnIUx><45+jJ1e`!#E(Y+;e8e9Q*aTCF>x&~fclM( zcu~9@?5#Y$X>gq^Q&%3&T;q17`r9}z*+R!rbzZh5bTcFVifTSkGfUh*B06rx>NRGR zL>RV&%li!(1sx7=|&re9r5dl{+3`hm+Ba72Fd&K!F-#b)x zQsodP$ka6*nqa?=JHr5}QcdQ`e_}k^Q2bmolV&Ac^Vz{ODUI76CMbAJi>7b@bA#{fnh_5R%8IIB0 z>luSmX9!hA1J?W+rI4DW9L_PFGMTO6$e=AF`_c-1z7nm>B3IOobw8O?m9vyjxMBU* zNHQ98d7`9rYco?(d)O+rshqE@!MVHgm`gn_tzgGNv&^EUsMO`NG`9EMO=yF6h3>1D za6!V}o^dvYbsX*{jNCP)pXui-nf?a{XsHzdMvchPmleY)4;B|$Y8{bYB)@qCV@#|j zWPYRlBxaVFD@D7Z8aerZQoqIxLT@zo4n~IW@U6?(Yk@2KwAnoADbloxhG9eCz*tcT zPe=jduI^DXjFq)R(8O=}BPGcwU zSefK%yL7l!S!s=B=pbA=qdd?N^WYw{=~^`RFeM0;933s?y5p{vh4?7!vt6|0OPdWC zqm9V$=OT-5r|HuEGQ^nO4o#Ge#Uc#(^}kQ?Pb4kJ@_AdnKvcPH`gWuTsnzIR%M zW}2I-L|0yl{{Ee}?e~_wRa+>ubDd)dFAY7bIZ9BiM6Gg$@oBsWt2fG5bS*3y>Lc5a zEl(;CX-mKRj1o#*JAe6mRPym-(q#o6CILqrd`Km9WWnt)@T;o_jW*=gl1eBw!k!wf zBa(+1baI*=CKTLmbdX1MKultbl#N{bohmdkW&qC$g9UJ6IO8+xijcItwlI}(ml z$y^PBR6H?biY$dMD5q7<<&9@Xrtt>2AbDpc=f;OhI0ysZ31e7Vdf<{DWuffed(b$X z5SBrMT0J_GZ>(X{c6dK1WlxfOzC7Hia6c0Ka*^;0m^ciCpI>0|1vc$7`@Fqz`SrO$t4$~am@Y^%oS3Dy(i!M~mS8K^y`s9!V_CK`@4z{_3?hgR0 z;?jb9+?*3Zk<^i2Ii9UNDHA(YBd(Lsevms$Fc-P#@v~WrgKM&I+R?hW;6M(!Qz$Hp znH_3xAkV$+(;Ub~Jem&N8W%-r`!(Z#K35Z#S|iW9KX-CNP}_A+A*=lfacP%!nH^St zZVpa_D)RA(awj)U%)LN|V#f70xq`q@O|T8Z4tk|qZQJKxMIC4FulC)+cd5_Il}j29 z9AisRBD8xV_OG|JDJjt=+&yNS<`=@|Gp%-?epQyWDt!@MZEQQPRu#bRmXfNV%H^wt zbvH9>`ix_;2trea*nVzqHRV+s4z+KmZ?#g-w=9%*8deZgVO3xnQ=;l43+I&ytM1v# zF0mchp+waQ&v6S_#M&q3tvOf`nMzLeY$qM%6Bs)_KTy$7ml4Likt-QSV8+mLEKpNes{n=e66X5h#5NIt!;Ez@QQ2Y61z zrlb*6=!R1H%=7D+0a)&isGRPy;^I#s!yR)nn0bihHqI<}a>ysBkLc$K8AMJgchGCy6RvP-R=@xk`vt2-=Q7Y0@XYhbA zaOg1XY!&4*;gojYim&-uPL*X5-qYt6N08{;1L z7;_8~7hi~7N@A~s?u(dl6H}Y)F*Dcx?62JG8ykgI_Z{uhr-)?Bt|oQ}g8On57xcig z2+^g_ep;`R{3C>fd!gP-5>y;GBU_Kns5A6@q&L3uLDk|ay6}356}Ne|mtNzy9&q*yZ-TK=CVQU4mCIP9*yZK7 zgx4Q5uU@@MJXdm)$e_F@wQVJr0RiN+T>jX^)Fc`8Irulq*>7W`N%N;pLK}TZa<)}B z)I^AGQc>;S_9Bc;O;Na}RFYAsgr#R@F*<+DsOEbk8XXP9tN#7RFNv@rd&Q^b1{OtX z&1k~J!W8db%jI-?4KM0F?O=(Z8?bjL6EI7w#5IHc1eK!1x9o{ex{!3s8XU49HoLtk z)kHc_ms?8z8(n8MGOt&odHpsZno=Q2j6p>20JWwu=MI{eg}05#QSj~O328P5LmG{1 z&b%DKQ`}~=)%)c3N`HfwE7mcDl?#hhi6Z+4(z>~jsaPFl_|}o8pTzUSxQ5HUEcb2> zu)no=6b{@gUauO&<`1RI+?T=jMlpDEJ@^klA#Bj8oI~vcf1oQXwZw4>ZAr@ZBl4 zqI+9JTGHjg^MtL6+l;OZaZM^OUX5g}nBRwxCXh3O9l4qF zH{M}c!^a}W@dX;Cp&a*nU-aM#O6L?^hilu;WWDk|+9zxmdPPaW4BwOdW604;vfr3u-sWam%uW!G>I&37jdVAjC(BYQ3Ps7ah*S(YCcr>t@^3#%F zrvCayfO)0vMTKh3%pntaE$(OU?-YNQ_rb&P>r?RX`5cR}t!tXYSg+>d;%i7^++qAp zDaQ0<>#~knI(pmmgEMINvoEb!MrGYJW_i|3Meb|{Sg^CNFf?inu- -Ol%?X9gBp zYJQSb1{xeoRwdvanm(#<-;AZ<&H0&!?HgpE0%m737V1*HGArB8jv2k!`0PBD?|6}emF}~6 zZ&zQt4%-+BC8CEOo$kdJeta8gfrcV9<64Co}H9mszw1WT8leH{dz$`_s()`-4 z))mJ=^1^%4XWYj$v^od6$yPhq^_%z>@4vQElnP3_H2Su`h835VVH^84Tj`J_ofWk5 zjmK=L`b%_FgnpWdeufduWdq=xRIFLkWJ-~Gl7WcgHS>@i@=&X{sZYi#Zz=7!-^H7I z2cGXNIu#5>IL*=Ux-t-W#=@=VK*PqUP%X&`W&`mb084P&$8w&Hr?(BcGDD42h#*$= zr7JiM$Hh(dqa|(h zE+`G2(IRo)kky2Ibhf-l40VjOXnGh{v^kxM&VYJF0j6Ejf9nW5`Ud7POtZ;#9zJN< z_Atie!WNC;Oh?%%u~);*iJU8+2!~e7jt$7b$gY;Afv(y&qZ~U%Ur1BzV6~RwMK-V_ zqV+gh!#cGH>26b8-yDI#wpJ}2r8ak!jUc)k+U`;pkLh|@$&QU|8apQi{cwkbtk+8` zEv5x{LVnMTbqs$0>Fk(=D4~DsIFWfTP;{&TPS2S*Fde4jp3hf0-vV|F9IO#n+T)AE_=9xX-PcYK^y$FQ z!``Z6){zAAlucL}oV)cn1J*d&XouE{N^1Kwnwa8NtrbZdR=aIVy5&4@+Ke`aL22xv z-Li?1b}4fz;u^T24+cW3SWZZRiM&WVbB0&C`=MXKiGypKa!rg@6wDde_Ux(}wCtmv zOD=6>A0gk&rSKl5&zDDBb-J4LYC=!GI7RP{u7q9;{zPR)4Qz~AbU&Geg@TL-AGr8} z%LY`F#YAnjOGnFF{1Br~wo(zf9?Ks|3I#pUfAanH#!Hm^d9K+<)7*{GtVX@?OsuqPR9`5YiK1 zd$DV+r5VfyuqK|6Q3$P}SZG9pOIc}S&WD9Kh^Q!kn_qgY)y&T-tIn^rd>>8Bw8_47 z(iA496jv=@*)1BoxN{C{SD+EGAg?51mTy0~s4$%4x3rql zTWXs*uQg+V84VRxh|Dl7ZS&#ly;VY)vL5TM+}5pfYZ^T_;@=G-t6T%nPqd@vjponI zz~(O&VzI+(I^Q}NrBhHag%$i8&XXG{HVT!V$ClsY8CNUJN-t+h_S+)%ZL7w%SMq5Eb^_(Tv zGdBS>Qn(8ezI|PI%iy&e&Va8sS8?udxD-aK`RIBtz#(cXsCn?nk!V3;8)AC&#c)0I zK`ukMh?G--#agO}TjVl=x0~9V_8eH-oc*qVChpjAeS=d;A(E$WtPg3a zvpZcdoxqPR&m*5I3pKk2;{KW>cNbaI{*B&mSTTs#vA4$N#xBrjg9(aUzgd>H4V6_w zi|i>U@Fnje2SOKWic3m#@CE6TcV29Mrfu4qKAG*n-k>tMW%Gc2dUrZQUbjH+f|D;9 zC^G5s@5A_Z>ZNC)E+e&8;y8)6fvaU^7!;@U_1hJFkTa*xqPTWeB8-rlRttV4v-e3}T)63X1&lqgb`=T>}zB0_}Z*OUvJ#*&D(;+6JtZvU5 zz)anH8;uWPG}x$!Ldsq>qa<6a#Q~<-B|uiJe4wp@j@TNF^bO|s){MULZBbK*8tbq4 z1NXGU%Sw-O0TOe1=?J9L%qN5WRumtlO^%s)sMK zMc7$aMPfY)z}BYwycm^JG&Z}hpo8yL)YRS{EJTNxL$~%S`))s4c^z&D0DbD>nDqGb zL!D!11U0l)Is$Dur;7lTKsC%=3$x?I9Hfm0ujbuIxw-K2a??&KUqZ^q1`%;Z2oPyf z2hF;xTb36W2LPPoRJvxNz~mk9RbioY$k0}NcZXp@f)uLqfr}DQ#|QG=7CRL@SqyRxSI@F6_xFCR{*M+stjizaX>Fgp1Z{ z#jB(^G^JE#phs2FpE$ZdCSZ1n0Syo-WE+ujqX@PuqJnll*;rQ8BHNyp32&YmYbxox ziUa-VY8o&hF*DmMJYj`sDQZ`yfp&A1^#_yYi~F`%-o#IRDK{U+nZic0>z6`|h<&IZ zB-l_9EW{kS)FazxKj{{+f-pDC)W@B~s5!Vb!e94rY5RMC<5NXpnE z!Qaw`KPNSNA|M#zcT>?SU&%@aa4`Dno7u>#vYTf+wO>}t$6staXa|B}=k z&{BIju}pB^{w*E6a3D7apT3R3LvNUP5h0-NuTlRw?P+~@IY#o~*X!eOkM@&@4T2sU z$`%;-960l={beVDB02j>ek}4^wyRfb`EK#t>$;A8n2`7GMJ(0EDw35JYg-8R+z>?_ z$PC<05lUh5q$&By>)rW~%SYu?OiNT&%Q_djI695{UoK2?Rqi~%E(GW|`!!I$nM(DG zmZJ3_q>0(&dNJX|8UPR@NPu0DxufK0Y-Ywq>C^82a~m;-B8`M;91>zmtJFO_d(NCM zaUf#uGK_o{PCToqn73CG{$WPQE1fHlablVz8{8L8^Cxxym!?|`6x3h zSc*gXERR@ULL5T<{eVhE7zLFFG0}ToW@+a5c&Irl#HO;fUGI_Y&L`eh8emd6IV_fS z3KdX*or`pa(xRG%o!~kv+NH-9^NG#4)Q<%aN_|mbsj-LV(nl!muwXy;h+cGj;ETt> zdqjago`;*68<)nLa->whB5r3?5@Q(G9A%9(ENLxnQzg_9rwB^JDA$?{{ zsWpk6apn=!Vzax{U)>B{bM0LC=*ENszh6wA2mhf9=%r(Cy%~0Br=emUpU&jU22{Qk z29-`vXa;x;je}WjTL!Xod)AdIhm?T2Dti38B|S_$GZvmAVh$>yPIS?_Ft2 zDhnC0^$$UHdC070nmst`ucWAWe_Ba`%zeSwOnYpn(fU|0WVT$6TzXhnbAiHu<}@88mn3^ zrcXuyqaL~KP;Oql@=2SIwgd|?u+OKrh$=AiFQWHvkyClOJ&`V=_6WXi0H4llgmstK z0-kj$XeafClBavj>?KY|x>6DY~H9Pw|=nWYd3y4QepXX<7Z%_*_wD(2mYi|p=?x=#zT#^V@h%F za!oW;Q*^UG$Ul6d(y%#P>y_E}yxt7+L~sQ~yE;TBYoAQNQ$01~*A3_o%L<3iK9ZUx zMotZ+%#J+z_H86{#E1S0`SW(nsqyjU&dQo%6v|XhE&m*PFsq~Z>hAW7q8wiSFnz2V zVFtPG`ypMQ9|j7r$v>=2TVOD+Mw1o9+j3UWD@InzcYs^=fO9W2)Lrpi>3% zUa;Fw%tXVj2n8DB+4{OR+0w3LQB8S6OJs(}JEVtwSu}{jd=+eoRW~s)DXug(_Ny}& z4m4b$M|b|Ge|F5<>9U?#3!}U4ktM7r%Lh<=CY=#11oT%dfU2_$y;sBQ9Q2jR`JtEr zrOSrx8V&uU)re^2lq&sXx%CaFc4&MYw}F~{yfLH3ICBHh0Nv1SJG#tO%kO#+ojBX|2f$Sy6lc8+#~v51geKan))jX=7+$yoZy zQ)dd-H5g|nHLNJDchzPl`c&`J{QUwrN!{;qKRh&K$fh2a&r5)e8S401SX~C;oawU< z8h@rW?i*k8o;#5R2(h8kDu))ITq%&Wy@<-qi+rB*NYtYKb{|B3h`u))r+QIt7%sU- z>Cc4xZ&A+x^D%(!QZTVEG6k-b#B?u>G%zVTEHbblJQ(H}{{oPU9w?W>^>EXfTsf8D zm*1NLTd!n0mO5pK%p0*ebub5_o6BwnSEYfe-}SV(=*8gH9~Lcae0`HU`B19zjyA4} zbKK=W?kI7b6>zR%oJcrHQJV^z$@utURPD@{cQ-{h5b3xiX35p7P~JP-%GV9{m+>}w zAYFDTriT8TP~JvBId?9XzcBXV>uwjhKHma}hO73>+a_jaBBJ~n+Gef;n^os8Fn_Ls z$o{qnC*UcOc5O?RXO7M>YJ7Jtf$N;XQ-N+b+iS}4`Y+|OwzS|D6c0kfZTXb^+DroG z%f#b6y{%V)OO#dn=mllvh5R9ECJB<){XN?!5%URf!W}?LnH@{KSL~!FLrcx1qXJhx z8-az~?-1-~UhdYo7}T`ZeIAgG#f7DkH3w>>-6r+`pjJmngTkXDCAW=7eXm0{OR>~i zX<&~RahVk%6@Hec1$Cv{;`3?m`fqk^3!JEY)|QCXGv`vAN}GuBl;A+CsqC2XCc? zMWKN0?^Y51fPcaQW-Va5tS3^;d79N|z2RTqRl8c?a79T%1MIyhH4et^(y1%rsI*$Yf zqk58LNFQHH^9+ZkVc4w@BN;*z;4H8m`O`XvecvCs4qBc@GW5E7fkz$YOW_NK$w`w2O?Y$lFY++d7~a#17~UEg|?=mE%K%?)=LQgCkN|vz+kB(BRk`n z=7RLn?gm2n48Jr8SQ>{35URIBzSDam?N2_uqzhRph~O%I74B@g1rTr3&U%pD5b!Wj z38KYQD>uAhvVn^ily*!hj%8zZx1Z=A)=^p#XxjcxDl-jjt@{yJEDV$$;esRWo>z#3 zH#>)s0t{wiY+5`6WQ9%nr}@B-&lRX6Ms3n1E~4<+^;qB%9Fz?s`_S>VkXj`|HIT^Z zz%>KrwQE>U-3R9+uOKrv8+PX}g~y#5wJA3544iH=nIX;hNcUBY-mz~#-F1(GmkgNb zc6Rx=?qf=c*;tchFG{;%|Km3-yj2n&c^uQw2n=WJ4zZJqyF^AVyJ6SAz#Zkt30$TU z;!4_y=_*5#V;~_l_=g3@&Jt8`^=%EAlSm?I6h!f5)5KNGY;0;f^)jy;`da%|w3P3H zwbO0vgmI=dt$b_hrlyPyJj2Ol#uCux@K!Li6Z6ShRQ%7uvcxnQO++9WRuOW&X<>oX zaQ9_$cRN_=sq=oDV9B)2XFNQeWJ${$G~%Fm#0$^N;Q37@keNk3qqRMsyrbU?ATSWr zMSM?nLsN!FU%=cbO_KgKcHmO)*u4C^lMe%au8Poo_zFhUeGAl!;cc&VG#Oo~r^ zx2%0!UIKxf0LV~(ZfNAi43`vMrIBFfD$&{l)<6r@9^fZR-#W&@*8f$zr+;FBKca$4 z=aF$dK1c49q>#E117vb1lDnOK9s}5H!S+t-hr?)xV5BAgA;IAT2{rv`bsSn!kk+xF z72}3B%(k?(M>pMxdP7rpCfgaA>15=vYuZrHqYpFcw3Hsva8*HcZ>jm>O=HzGRz~G$ zNP|yTNJ3@(m_?+M7LQZ+e^@qJ(I%QSc#<_qquDWccP;IM0;;n9uynpll)0}OV`gpF zQCTK|oXx4aeC2&}69uk*T(>{hx5wD*a1%k+X!>$g5si|on$>(NK#KM3uh`B#ZRet? z(7@dDd^?NB-p0L)EtpH&=N}j-B3XwdQ>P+0hEG|@t!s6DF?eWDap$>0p>v|{QJ>0V z-z?QW3(5m6lQyys@W8@1|1f@+t{}HEN`(q52pFKU)F1hV&WIsJcyJeZs(J69fEVtxNYDHP&)*bq2$UWe|Idf{nfMSRJ(%j@l z`Cw{dQH@~kot&5|QGM_f(E1?sF^mG&j*6px%XR_HbQVAq;E{8AuXVH8Ng(p=?NPd3 zm!Z08l>DoMMUF1jjZLYm?=?WY$tn^lqEK+l03t7Z`D)IGgf7FbKiQm;<%UDbks=g; z5vjc7{66N;_*;3})v%kD@$b^nIW2;3k8qfh#7>IMZperz=!I2!n&_LCik-alHgdlI z0qGk6<)Q$TkIS9`U{HlO$iPtN#7ciI285Y=P*CXn6#ymuu$mBjr2p z9TlI>k}xG_61n?6s15-TYXn5;XZ2Sik^+UAqob1`eC?ixL3%0$io-2`Hcp^GcK<9#z`T2A&Ew030S&jC5>tTrFKP&BdF9$-T)Mdpr3(^%U z*|dM1&)eg^^*gA$1o~rbsAJaA)3}g)BjnRJfw1wBzc0yhY9wPprJq_88-r}F9abq& zI3QFcVjUQD-U~yghs%GJS=h6u?Yw30{`a=l9T~=&H(xFB>|?Ey7eod6a4s;>%@N-k~;9;m81Um*8hy;{|?iy1@(X0YVyOt z_YS83@AvnXFcWL*oc{isCdS5v#l^*C;Rk@7*iEqpAuHRpVO)|CUXA_DHJv``3GVEyX1!ife*2 zPe#VZiFQ~imhI&%abdZf7l)02u`H;fq5_}iz7zuF5ae$;1YZ7F9I$oO)sb4-+LK@a zP>|f{p=)d6Cwmj)*&#ECfwA9zvUlkE? z9`8@$0se9~w+sPLM9h49n@+C#b-HG^Y2p-P%;wUi4>8o6<944txOxG$o}4~^F9r)@F?=r?VsOUV)C9o-2+aqR(m*s)_#5FLDKMC@Pj6NXrba; z9otNJxVC^MSUs-Rzp5XBOP@9n96s0lb%nNer+_A%EU~KitC2l>wg?0NTp)7ws(>aA z=MDRp3=fg2vG8#93A!fhwtoYx8O@l)L^H0$zYgW>HuTNypDX@dfiOz&f5rX(47A&= z|1Y)T^)rcaPspohTt_Q)W@j$~Yugnf@b`??^*W=Q$#YDUWmESn!FA_FH`=~Eq8=nA z4k%-k)w=xWhyC8uDloT}Q!F`z3@qCwXo1cCu|x9I=H@1Cwt$!T^SkPB;p4yCbB|9Z zjqWCn)QUl+`>Vx{ZVuBJ)EW{L@@&TQgW_qvg>G26brchtV7R4hE);6CZf3N0~ja1iBo%^vSET>bf> ztT-VEIocFhUvXD9W5jz2Vu^K83&$*tR7E?@B^iiaIr(cSLLd^3HYgJkN(!0IHe1%> zPjk)oKQcpc!B3T8S84;}Q&Xqi(<8oKYg*lC*jR0N6|j~a2vVt5a%mq4D=y)(wl*}( ziHn0E&#_{>hl>hk5>nAdckdPuuR*FBmqvx8mw-F~Kb(%w$jJWl&jp+~3Wen8MD+*4GGgY;S8Th}1g3P4<}^bP9Oi(f{Rz!mrWc)=oLq zf4UpP-cu{~-d0Ul7uDiE3PxWw?08&rb2DY~-Mco^!f^S%J_9XXUBj`w1D`*)%36!_ zOK5W_uGMjA6YIbUq0|g+^(bz2S&qboRi7@22Ou?$*pRL^pi;pat%2Y8^|UANhW{M3 zWL?j2$F1}}AJxHaQ3F%`bJf{MIx^O|U^?6JApg^wh=}`$wZ7?%neiS0&BXheNW5O^ z46*W~zYN7a^0JQnkfgY%C}gb5D$C3U(Iy!}pu&giJa&&Dj#%6c11iR43RFxeVAFa& zPUsX@nsgufmOZY6KYw4ziaCO?az0FmX!p(0ZdP?CTC%o%EQ zymMSehOoG}I4E?av#_9a_SC7lx$jRB3Jb+jJh~>2g1P69R(7?l4d}ui+P?wH8$dxq zb}em%JfBheK)&|NALD^nFJB)19tfB@P=f2p{oQ=Es$sV@r9^|x&^85eag?i~A|}zi z6iMn6k@c7uerYl|8*dsM{7hh%^}o%dAU^`r>JX!kR+-JtCPPWx8t>kn)Ve=YCI*pL z2-2ej?~-m2v){h`j`xvq87bY`+6SNQx8T=cH!^>6?*+Uxt6V>1tD;rb?cKX{si03M z4i3)7{~>6fSnEPTrKMSa{^V_f&Y+w*h@Q{_405~S zHd32g)32@j%^Yuj&oypFXu(NjTcu({_b!TK(8C zgqLs32s0vSB$Rgj<9F$sH*cM8%rWo1H4;wUCV*zt? z=TlPf!n%+obilo7Vd`+eN)1MbmI0q~3izhtn>^(@Qm4jf+^jgq!hV`xRnT}z8{5ID z=5noZ$4rdjng9#FTjOZ<1&8AhWIFpWqc2Y4vP>P1nXFfblk{Fzw(3P65~a~T)h2k- zm9@Re=aU%C<@?;NX;QwSt+{ZfTvvJ0#SRNIe6H-bV?rOQ((@hL{L|3TCz;snHU&bx zR(z>2^_XRAxf(=2u=luxV#kS z$!B31)qxRhQ5W;W_^_m@XBVBzJkk}JUvoJXRu3RpE6jk!qVscZ%#TjfvtYB1J@H+* zm~0WSN#~H-TtaiByxLUU240=daF!{wE&nOBJDi}3DVoBxk)mUv>MPLB&hv(*zI^@yyaSJDxb^Pt)eZn=C^4<;dbhW{a_JdMz`-J+WI%OfDqH) z_+Z#|v8Lt4RJM5;x;_?FM^z64`j)bj*|tm|Ev2oU5e1-M7wA}>rAjYPSK{Vk3B!A6 z4CQqJ9Nqfyxzl^rp5z2HVB|bgKOod+emJ*NY^Om-mcd|%H?Ick2JfqS`}U~P!o(7G zynB-;bg9NZJ-GdN7ps4LBR$}{js&Wy>vCFR-tHd3IWp&)u+z32WVUHRu$J#KM>fmZ zx$l_(+3=sPx2<6Wn`y<)ZiXf%@m-a~MqN;Fw%{;t&>arN<(Dn=?hThH-k_QGLzpEp ztkZjYpp)zPwBT$}f$hLqv(X1BW(Ee?yQpKg-4=ITEw_ZjXz|EM;~{2Wie*4gY#u0e zGP5Xx_TB9qj&<03Q-y*5>I^dhHJe^cH!2IsZ~U64Z%L0}_XFOz<7!UyHnhKghH#xy zT3HUqGPX6~KOb1eVtl{dDi&*A3@B)++zFfzrVl=xci8#){SjhYt>u{wLT`)dZ$z4i zYFr_Pv12F=dXR4$U``c|o70_253;~1XwKu&&bx(KbrDesfFm+icSvxuDJa*+FLS7f z^{aC1kc#N>ad9)5JB_wL7f9FHfYmUMo4(zU{b*-Gpdj>U`Sh%#O%MLT{ z+_kOtA~*xPc{^;nERZSNu#_3NEyeoTBF^T90mBWHV6Ir>8dV(~4{D5oikkM)OIAzr z{`2dkTBq*QweKcRVA5>8E~4rafQh7BPLwwW=))th4y;el1d*ve)ZB zuq?H8b^8I1u7U0BWi58XkoEveS=K|#)Q|K1SYAz`CaTpc?YpZ6zhT2CANLSrzg#1M z?oGPEZvugojRjkwWJZH}%oE-v=SQm~mzAKgr5_Mhn3qw}jH11)O<^G!v;7-P{Fjre zl8B9i9qrvUzx5twqr#4t^GSo9iK%HZIR8Q)mg7B|V|wQfEb@Z#T{5QN$@*Ipr2Wc% z%A0}+v+3@WDH2SpRQ*kS&lD#mg#_)QeX;~LaccyxaCm-)+k@>c%&}!A4JFyt`SmN2 z?mOb>8@P+9UcR*H0I8WsW7qV-K8@s*vQFHgKe9_#YZmKstZJ5Q>H#p&cO>}5#%006 ztnzyxT_izIWe0j~=X;wP_6hwE8|UMl3?ipByvGXnD85Thg)t(W;@SH6)vH&~>2Au# za#cO3Di#TPv;4DV2iTv9Zdi{{PtT?xrpW~8SU^EW@TeY{jP z;Nv<`^%mpu!|Y1|v!8dUpDNL zxQz-j^8J^Bdj;_Y@-;oZy(gM}j*P4KUeiQ3FW_{1Hv!mqQu8zzpi`d~W;O^?M60Ar zwOzgzwApq3KF6hHT54f=?+K;t$CT6BW5EsOeNS;jRgTeI3Ej-cc<6Ri$Ve7=_0*@^ z1MR21KMYivj5)&^9VHkj1sL%azFPuWpbG#kYuX^yl%Cn0k{dGW5ov2Ma#PcyM7 zR!>@KkMR!Yaez8Y^~OxipS->S`|@osb~EF!P7BA@&+$Jnhn7*;X;%t#G!z&X^IOxR ziwi~mM+7ckwiHU2Gz|}i(=n@OFHWph4WN}|M@ym z<2=(%5;}2RX|<3=MAfX)RkA?pR@0D%9|80sJFYL9{fV~8ZF+jTx;JaymJu3!#Z!sz zLk76G8jK3ABUE6uC>-^a(TutCr=OP=nM|23Z5^e7Z;pj$&vWHYXCl73$$~K9w?JMw zq@3sE%+cUm$LPp?yrb$pl#l16TihykdbKs%221(quPMLNC@f8LaBxsqy1`iuil`tq zDf@Q+bOD`~0mL)}zM_-jPcM`HFbnKFf=m$-?g}DOa7jy%PU|LSHc(91Q?G8h7H{_5 zZm5Rc~HZDPQ|7Vq!XUM@tC*1CMuk7Z#5nd{H5(p6YFgAb^$_rEGG zj`0t1wh>}9EmaBwJ^oPju&|hP>#ku%k%ZXTWLso*)eMc=MN?o$fl`Mw_@scPaB~BFw+JKTOP8OP)Z~F1B===qtD1&C)8&Tn# zztrzj$bn@yY5)EMqHEh;=?LO18wvvqo%E0olN;W|^T>))5SVq21)Y~?Mc^;X9Y=3o zPJRImd}xdS+}|!HQ%+xbl@UB;pPJ>*!0l%rb@BmA{p^VcTn<|ORD!eX6vdrI5angm zJNCFCdG|qI5x6!XF~AD9tQH3e6vENoJ0ZMy){g&ukdC==G;rs1nN@S2LlfA&6ea^Z zdV1IPlS9P{r9bxNQ&y%AsK~WT0&BPGgXiZqrb~5{obyV`iu4lFHV8+7@87r2#qwKX zT-=kuri%CU`t{aio#3SRHm^B(cuqBwSYW+4O8_%%tO(ISmDfc1x;eucypAc# z#*ASKusOPv^}wtMF2C`GHj9|0pl<`<8_6yAbsk&OVmQ$8j(!UZ)YT=04#4q#EogLuI(+6)T}aM><;2KV05vM zHzYL;Z{N=Dl8;tdFU<&9fTymT`RBpE{Q{%jl|Zdi<6aZAL%vjz5=p^xm}GNhp1}K)lx;zWEr0tuLvWaE>r`>ga2EFV<8H5Q8XiR>4Zu`O{%)!(;8$mxkUZ`4wl#hcan*{X-A z5fYc3IxfZn?%UonA>KZaH(e<#hSRlaF2}_thUInk+_^`NU7V}WS6c%_e*W}^To{f! zd^(K+1G_ECPufsAkVE&bnp_CmkOCT2`U3T%BQOI*nvMq34pKIv19$C(RCuD8RW<;% zS1x5^wl^nUymkAy@HchqerPZ2#_#XPZ1Dwp1i9(7>>h_6keh6<_Bf@{zeHE|;oWd6 zcxWiG#;?Zw+D03FhX~US##rpCbC7e<_|YA{GV_WKHw+zRMh`{Q5uFM$a$hu&H5kz&5f1N$yS7N8?Ikm{s|4rw;QPNO&=(?3QO^`vCOGB zxBZ_U2)ufE06@A7Vq0bq3NyAo=+@f|Ha5hsj}WjjSulXkWL^53ZrPLa;_3E7rQ-xrbdQKW{XOT897(m4XIFBbzdp0~= zjy|~5^dZ(bNSQG!VIwg|Zt%?%@Q2to_2`gFoJrmJ@*0m`Y-H?&{QQ2?XyyVRUx9)K zn+ZuRu(Rgs6BzSu=?$}Icr`61b6Awg4)8>Sn+%6*`Y>gHD2;_~R zg~-&H5H}Wa-#Rz*1m4-8vUgapcof55rzncKn*8Z=>y=BFa35<-kh|}GRY2e_iP@;W z`)3Tne322baWLQUalk5Uuw2u^V(2J`cyQ}mkZa$xj{TSPa^EQY6klu*6E}ds)-&`W zA>(^kjZoJ&2%sAtLc=4Kl!pmDJw1}1%em1i5|P`A$$-Fd9c}FD^-;rUH-$!tC6?Sr zLpaaZD)bCUuS{Ba%10Y-)EDPYb)|rX&NXb4s?=Km?mGPDWCyeUT;GN zndY-JW4C(-aBdV>#tsSPDnl2ik8%C#(5*kRFrTtjL!6dp4o`Z^Pr5&Lnkd{{8NiFr zM5{>C1}p3`fc5IiK$-asT8L2wRN@x)$ns&19cv%MoM<`A$Ny{4ztmBJN5ES4|H@(Fy1B% zec;fcVp1gO;;^Bip;YjX$CG{*ryH`eZmFO_$v^$dKT@P!OHDa=h4*y+tSE_J`d``;BY4YnV&9h&EsGiL5=s)W`O zN1>EUJCHfTm7`Z2sL=muoKy4N21MfvlgYrRQQs$N&ICS48oQL*jLLo8CmSIU`a44k9DU%{s| zCYos7Z=i(2-HDM57|^FkIHY}FOGI`LB-$bhXo0Kxtc?XwqPV=)m7FaI3;8{aU;Bky zlNxhG>0^q_8m42d+pFF>V4_cG!drf&uPxJQ^88pKSU9tDRXv9Hz2kz(KO067Ognr% zx!Q@o9>Us~Vr1kqLUINMBq8w0NiFRjW5D5igS{xS`H$*_9@!Ug>^P77tp?_tsL%3J zN)0VsN&1gJtnC1S*9!{BFE7VN7wL_W@8<@Nr8Sffk^&IjRsk~~8kEQod0UsG=TsoI zBh^zrCbPpeRRK(@6Q}>2`{QJ!rnscjqIQb1YvtKVlX@UxmDWG;b9#;Ohx&l7^)z3n z&~CK1TP=fmvgF-U+gx52%E6Uqp@zI=TjhgzLt;q8ZtJ~!S1`Y^6vF^}{!`m$>TJPw zZpHZJl-09l<3@VTS-L#ph7>G(px)~++sv58vC*5MTL4R6UCn+`zjUq$s16ZnqGC#8 zv2m;K=np}L6AR6yxS6I~FeQG)-!v*?hnkIr)BI}f>~abD+J_NI}-@eO%HjwhLP|u{I%;uroC^QFFod8A;dYCwO*<6)mqq>4EdS z*(X~LCvJcvKqw%W^XF|Xu5P6J>O_kW&|^CJ%C&0Y4xxurj#8ApAG~P;-ltL8KW@N1 zr}j44@7uS}oBZeA07iN}c7C`->-gi-Tmmy04H{1ta)zM8)gBeB6lteC%(<`qGYT*> zZ{x=mEUXW`^x%>Mc=uL=+S|9E1-MRyV>gQ3Y@!>MV{}2Gmsgxp{TDhyUmi%^2z~kT zLajEJ?BN3kW^gxhmlp>dy%#>8KSxG`;oW#JyDzfQ?sc|O^%wuQZf2nR^&Q>OS%KXL z2!`95jith_tujDGKti)EAEI`SXNBPyl=XqUF<%2s-(s8qH3pFKKqpHr*FH;;2uuNW zL-k#nN1Fnm3m=EDdD?A{HshwfL>N-0r(l`3eM`n+n-h2`3K};LMd#~Fko(T-{HqPU%+Yu{UV;DkuS4XspHt*)ITNzI% zh+iws;H_imiXwb7z`XdsVGg%&jF!~H?#m!-`WM1pp2MxAfG~o$N;}W?N5$gK|I#|{ z0gED6u3Y&EKWXwEs2S~;Zuye>Xpx2PNlzzQEr^91L63|m@+buuJEon?o|(DV60np% z>C;dH%31)?a&_803Q9O?@aWP0&uwiT4pr&PYt_}>^uY=vxo-Zq(cz%Xw9c6m4_HT! zT)emmvO?SN3^Md@-FkZJ)Q|Y2B>k!D>&C#DTEe5?Bdo-)3aZ#1468h$CgraIN zhz=I(`CY=Rd6#Pj$(B3lo+8rH>XDp+#XfX>+eB0Q;!Ysznz!Xk(-nM0M3ulThTg_; zoYF2?phig|b*;42W!KXLq~6gZNA!&!=bbuz`gnFQT)&T@IJ+j*XixK*Tmozq%BLm7<^;& zPcYw}JrDNo`*(=Mwf_zbICx|?L-Mar`JR6_w&&b#tl;0D_xt|Y_uxzOr%yqK;=ez~ z{wwb8|K5k{+wBKs?JsM3)>|v1{U%Y1jJAREI>)-yl;yv`0g~vl5Yo39@r^=E8VwxT89vgN44!t29 zt@`nCu1>zNVlM(@H5Js9lauS{aZLGDd>XVR3jwt=MTCSj`c?ktSpmRi2dgVqK#to& z8TBg*0J!o&{Kp7cu%EYBC3@Fv5Lb)#ojmzHCJ!uSx%5)Cf5q!ar33H*;Bm}$dqhb% zCr|I;!-qxw_ygq0&B*vZ4$dO|{V^%Z%6cEmrg8T!C`2*XcJ?A+=BZ6gPA)%i0OyR1 zSm)>EZD6_ozI@NEvzj`T0`SADmoKl#%lDHelLpqn9Y20NdR#T+J=1IZ@?YVd`k>tS znJa>VCSmL>L6FzKuI_?u;zYddZ6J)%;pNZ4TLvS; zTM2^ZL^XlAZ(wbGzsAaU`)+~BE!!(fN=EEqEmb_6oW;HR4NyXHK5#38dJ=r5|Jq(x zIwn2AlYn1^+dwzU2+Ujt;^fJb1kbMZ9No^RHuB0Tqu;+6`mElzP5f76+Lf?!AT(Yd zK@)SQ^t^`;Ex#IZ!?5!ym!@$SfG|1l?yeNzoG=p`P2Wu3C-W%lMb$>022(vgvQ5?8v~~u zsP}X=v2&7E`Og~OBLoVLgFsN@|3qHWiy;lQTP(e@8?(FS-a(w$$MYcYWWxQiu)<~* zTKiw?B2LxZTuJTjOA)ZAN%?UdP8WL%3LHDWyZhCus;cfTu`ABn`>G-1=HQ^3uIGB;|9NbK5B*)|3#vJ4h$U(R@qrU-E8*nZ9hs{@!d$v^ z>E+TtZR)=OumwbN-UacUT6#UpaJkD;@>w8NL2S$SKLx|tO& zg8m=&-aD$vv|S&a8E5pfGFU)CKt)tUK%^6qefHjKopsLdkF$U0xRz_iK=QtM-sic?bzS#;8w9Hd zADZL{xukgC%`FCu+JCU&_j=%!eN0T@G_}VDhTu1bsV;o*{LX?A8^rhg`y+qIyC1`B zVZjwaq^>xKGQfe*VYBckzJ2?4-@bi|4x<0~)!d(d`2`ffn%_Lg2AG<&ydvGRqe}ny zEp5Kff%}U`r;{6Z(*U&(OVcZT;Q5c`Q5FBUH`|`{_(4cO2ai;?BYCpAYijAClYwR*#O_J zkDqnzqc;0QtqD4&Jeiq47_<5o!C1zaC@dpD#<0u)l;Kq?akEuevM8vag=wBoX56TQ zmBiwql`S!kd6KKXp@6D`9~1e>B(#YTe)6Sdx+AqZ^rD0_s>-I)$a`HBf_%0CnCzA0 zW0xXO^DaO1;x{GJ51BzAN%CN5KkF2aJur5(5vfsIc(z$%TcsNnA6o(}%OtSV$<7d|+2kOW~0MXL|q` zdKpzWC_Y0EpWslF`BkXN>)=?l&E3Zr8wcex%KZJG8lrlRfi_XJk!NpAx;vB zh;O*kNr*hD)8!q$v%bky6#%8hi7v?>HWBi4B zYoJ~WI!GQJ9X;wkaU$al;e}~RYEF@e=Joscj{r5%Om(zoZ0~9?!jMo5fQ60oOv-i8 zyjpOO4&F#TYxqoE@`HQwanUL9Z@EHFZs#AgKbExoe24wcNiWA#x#IE+@HFRFD&MpK zw-vNn4eB9gk6e`KdH?&1xwwJdhbnZ%muIs#mPgNWF!_|(Gc=H;VSv&jjJ6eV9){VZ zq?e}9GHPkBQc_-Z>DvP`*bo#w-DTcZwc*`fb z8ksJYz3*UB;6Sl>Lw8bSWPb9WKD~>F_8q#PH8i^{3a+WdL8A!x|J0I_yFIx&`hYq~ zWIFS~|CV>paYJbYINGo@$RI#+bP-9x#q@pkN^J_3+U%V=SBo9YicG4tnpxPK=I-2z z2W~$sEbhrbw?&E9dUR8(Oq#Ob)U!@PR0zY+n}G^nnHkDVRagSeCJY&HE4AuR(Znm` z855hGouy7>JNi1=?w4P7=a|&}ENT_u)+rjr#Id0ama>?GMZ0Z@0h%95`9b!`MASVY zp^mD_6c-8ZGlLY$qsOy4uryTV=6kBDA#CDyS!o$6N7>o=Wo2>USU+E1AFLQ45BVzW z;gLyivOSikd9y9ecz8rtyrQ(y;qK!ud*?yaFSF^Do;fu2%!*P*wQ_3hHwrCY^x(g6 z;dPgLJx$N=(D((!^M7h1sVz@?$YPquJMD3{-AT2)O`o4#xRe)SYL{=3iS7VMwZK;y_r4@pEfJ0iicvG9o@Yw=tM~dL(E7p_vpy#(*d-rsf|;yK%`({(Nw4A&_iAI4#d zfTq^!r}xkOK!YHtq}==((vz-p2O}f8<-UG388*s!aIDe~c6+HeeK!PjIe9a;0;Pm+ z78Z6`b!C(SouVw&YX!4!dnWjaQfPNTPv9vcypATG3@Cn7J`rBXL2Y(wVT=qB@L7dT zoVLNCegdbsV@-TJC3Wu@`#;4xGPk2my*DgNXHN*XwwFB|zFTUeTc%!_SVeo?HASn! z0EGP#w9y_`gMC67-VDR-7r@ULOS;U^HYCJXE7emKNVB4McsFG1*sokqovSf4_F1Y; za&rtc(VGYRmT$1_9WZxZmXS3@@6OS3ykju%CM`C0Q`D(9UTtQCT*LNZSk-zFp2e#HuBQrkSrm4pkjoT=<97T!H0uUKDFkrWcKrNIy zXgCTfM6|+myl2V`lZKZ|oeLcwpzYyDYcxa(&SXQT>E0XM$_F@jri$Bia@tyM!Q;be zmZs0B^t@v>y9@GJQDv)oEd`E5`UK@3uHb%~@N1_c_~g`sz8Y^}<8oI+zre3rE5 z{Sw%sL!CYk3g6DUk{DS;JDjpWbaM&j;#|0KFFogNIY|qG6va?FsS73unm5d1CZ&<( z8WLKRQ9&;e;J)1u~z zf*D@rk^}!bLVV%2>u6j1h0kiueD{m@gkGFRse@yD?#pQ|HoiXPK^|gK?h(J{9+VWj z)v35i-&$e|_I#8=K;c^2!M*PFkYuStiL4lbo6=}HyQBKhG%RyQgW zyQ<6!HQI0@&KcTyYdQ4<7l}l+E6+;ou;!DBbCCdEd!XK>!rE@m`jlugO@H=A{bpjd zbwmp}5p4u0!pU03*l7IB6KFuo?nZ9dEQCihw!fEz-K z_TsiDJZtH#dtPq4R2wpZt_Pg%+vk4Uonbt4lpsOe!i;g@qV=-jYJ96?TXW5*51&Nm zV#*3@){W;I8&2fe`@zZ4X+VrxFNCFMtbq2R3rxdQ6t=dT(nz_>O|3<;3r;r4;cjp9 z)#je+Rp%W2%n*bOcGAdTzuCqs___VFPSUE4aaU^fu$$Ri7wRy#BCw!cJkifIgCy z3Eagqs^}1K2FZriD^}aT!&tLKg!qzJ9qc_%-#iLqI88-u2OhxWLeUq=$Znlw7oJX| zr0dj^(LR<|k`9`+m;yqT^K_OmF8g%|Hp6+Fqy}twbI~DTv`7u_m-0lce+{WUiIx=#92(_ioqOoVTBh2?X?Bm*+R(rud!|xK0~wtY=EQVj z&^YWUB6`fYD?G#Qj<*dAl<1;3U}`pAnn(gftIX0Sl=tGV{L*d4t*)B_^thqv7xxiL zq$_$!PRjlRCSe$RWY-md8sp+)R-`5z%@oSe$evfl^Kq zn+~*#j+#c1I?JPHvDOt(rl_(R=xr4CY>FpQ#ANHitu;dRDz3z)f8X*Pp<0tHHCN+A zn-Ly8wy;W`bsWm5p7s8Cba0j!oso7VgNV5V1FAi7`e1z1(i)R&jg2r&gRYsN z+IhHd5>vJ4IMiL8`IhR-tsKO{ZgKJ*WPI)e!V={zvL6*6cLcceB8R*$94_2T7`)m%)vqc+qD(>1tTd)y_Gl#@SVA^5@ph2gw z=Ph@iZ`tAxkwFi!(?SikOt94zta}Rtw@Go!Sb>gH?n~NTciQ)c%G&%(k_U?6)o#6{ ziI*hW7*5^9w8q>oc#3TY@>%5RpDjEqy2UQFvz4xSQ2vr_nu%q6xi#w14MBZ(vcA30 zvUP7>wx!5?P7m&QnWSsggE_%O45yB<$-g-EJ68{$%*)7_0$LSmVfP%nN$1?tXD=aB*&!IZBOzKqxQX_`e&`l^G|&GM6t!o#+q${U=7g6~ zjOS{~&PfcZ4c~&=i6-dSu{41-{nvZ?*48s8G(`n!KqQSZEgl_5T9xnLy<1X%&53`D zhNri+<<+&I<&3V!<1K+iMn2SI$SptH{`SiSk9_diUN78V`r$b}+mApqE~2&N{KL`J z=1|!Jfv<*I1zTIG9`;lCmt`Ofo0o`hTqYQER}anh50`p|u27X%ZCGn@ot?rF8fVv- z_U*erXC!@)bov{Rk|1#WM(BP0T8~5%qFp&9C1}8%8sA+~Xd(&Gja_+l{SOsjNV`iu zI+%}>$(cgX&Adm)u;5dhDxbp;*}OT5l8M%)d^*o5`YO)3dT3)Ke(oR&Q(5I%SXyq? zKltlpMtZs@)iq^i=J`McudaTXO;?h1b_V_YZX=#bPujd`T`%ZjVh3hVigq7)7)_`o zmn}*pdUzS+W!o}-RCv+Kqo|RDR?qQfEs=&Q3Wo6gRm}>c{rP(4+j({s!zWKE<1GZA zbfmIDqyxjWT8GW&}!!#mCY|B)7ss;F%xBk$*-^9olyo18z8=kw82OR zw1{PAyKrhlK=LinMsE^x@FCo(#+x-3Pn*IXA3X zyEN;!u|niJK*4*4N21=<(>oWK@jNP<2~m$ZYmMS8iOspHLlPNfzyJRGMM+WsqJQ}7 zq^hc=Q$*80-kD!pt2nH{T7Hde({=a&1d4=HRqdq&1Kpexhm}$Z|}kKbdu>tzCG)nElTh19s{7)_xfQqJ~d zn;nuu@p5z5b!&@c@dgm70uDHw@i?ZUeIlW_2mXpqBuSP${93J7G0%aaXWWe~xQ5{} zUagqCFmN*+g%KbK)wuxICoMhoOC1|I@_id_%(AH#W#mvep*|a#eo+P&UYSBnpB|zZ zTM=fxy098k9xBShy=<(k8m5fNCri&yY%*O{Ixxw{8_(D*RbrT)uU6f!)q}dve5}?l z(7WLN39LfMF~+aA%$q-p_>d!4jRJy#I`o$;rVBhdqd?y_y*ca2g(&tkp^SEUvSRb1 zRO)wlK9lVy4=$_j%Qp_z-enkQ0D-*JUukBVviv8?)#@Q5p0-Hq-+0#VSho!k=e>}w z?0eO^YyM2T zP?~jNDX!zYq!lQki`8jYf5>1Eo&ct#A60 zv1EoImX^c&tR{>83)_`6oWK*$g7e2c8FaN0YuEAJp~EtmZj@!_;H}d`m`S}6&FzM3 zkp+acwFkCCRYS$K7HUcX^xELhGO2%bKubDAX{$-&3=9b9gdkL!NG`L?XXf4=J>BTkC9LX|HkD(9lP!revgU*V*B;px>>gJ#GnWDMtr^px)xj9yZa% zvc+THl!<@x`ulR8QwZf)9+>4n8w`4rn>W3*qBX4`9?%It7p#=2(RT9K zNv7>-cU@sg=mkR+w9(Jd1J;SwPX>Si3M+cY6hrk7Z+q+?(xz`U^Rl)_6SAMyvIGVP zi~UT4*iWhLqb7)jvN4dD=4jz!2J#O);|oW`s>rts4HEsC1`4~L7t~hFy#I331ods2 z?+^O1xTQc(>Q{)VUcWDY@`=bbxz3uwzT&gqQ+^)0j%=!Y+5>V&$FI7cp~KhB`!Hn&nK@KO=*@BIq0fNC3 zKXrHZK{0JMwUNDGnN5PzNo(7qi`8SjCIc&_tV({PB&*}BIfNk`ah_=(?q0DI(s<6O zvC4;h!t4nJ9)E1_3yi843LTM_vU(3tn*&5JL^4}XbP?HYoVfMH>Y+9L`GJ}B@$G`E zj%NwoEXR*${Yba(OiX=wV!#!$=3VDsg*BA3d zgV-j2JBrn+LkdRgsvLlvlbef8?1pyNxd0>y$^3&EiniR2YWG$=B#SYdTmK~m zO|#Ll14iiL>m={Z=WlJv1w(O{O^UMTg2fH=%}cdJjY;QuvL~fMby<)>NVvS9n{Q;B z^y3YQZD~3@60Mu#=0{+WjKqyBo+v9S_N)bjM~;dJ^4`05FBz?qzr4}jDNfo}pZL0{H{fP%TU&^m~O^kyOyOrv%KkU95P-H z05xe3E0;y0+8gD=DIhmg|7C8m0zl?qEaisAW{_ZQ3ahb)JY>GC22TqN@CH!*^E|=m zr%dARRq%A@doz|kyPN%HAP~(aYLy%jA%dcHTmj5Md(Hk{f~3Ns9y((?^evL1bXT`M zH0;RFwV*BuVz+viT&FxNGp(*2^m5n4Q+l8)3FQf-K5^G5Dz`RkSA)BHbeuInI7cP< zTyK`qH>vV&v+dGjEyexWWmiLQ&3oKX=j!d+a++D)a;a&9j$XA(?QW*&%*NmlH45%OtBa*c ziC{1gDK7+6Ks@+yd1vHeus8GdM0M`nd#w#!9U#IGO(=&X7eN3ms`LKwpsoHjX`Kbw z;~mm*B7jyJrPh9+A7?C3od(nPWvQixgoN2 zfvTbITw@gEwAua}vOv9ghW1Yct7J9y&;6g^cXSse3juP{yOh210WfFJ_!&o@Wo8rddYWww^r4dCkk0h> zcn5V+dHIt<&@+|LS#ya3SY?lu*oA$Uiq86;ZD?qiY!T1!pH_Q)h`<~!s#+A8N<@G-WJiz z-+={RjlB@7_2eA8KVb$TF5b;E5%$ewhXUv2Qv3d@mTni9+~Sh?d;;h`bY|R{PCj+p zwlD4%FXuM9vKmn~@6Qm_OLg6C^seAV>=oL*mmp#BVNo9Nc$Np9<%|YEd{8fz6eatVT-JAW^?PEF`y|`C{(RQrL0w>lSi4rEA)tE=5Tp4pqP&*JHo8^tdIi|paK za5B;1w%iMfK#mB(hER}L`e=bN?YR`(;EjNN^v&fCi{@cq0~t~k?a=67C4S>o25u`4 za!XvF(^;`Q?aG2O`GriO3sty(9$}>|p!4>58Omv`jfS>_r4_eYfuh8V4|KO>whRrw zLmc-(up7rC%I|`5C@1Cvp;I#2W(cY~Bs4^t{8$ZC)MrN(LjxKxFHm&Jck0x2^*3*n z?%us1v{r`b|A^&`Z2F!<7oY|8_O=)TDFBfte>+Kcg5U%!W1(4Qr|^|Xr>hEZh%X2g z{x&jdrY3Q$6sWSKfxh`pVWE)KK-!z1ezKaFKHD?(_pt_vx{^wN z=FD@yF|w=FJJmb>s~e~M1)-&$6+XleLa2KH41cKm(EZLN1ycD7Jr*M%e1@+`uQZD` zu#0Y$8a)7Jj_=Ci-L|Ub^O9Jpe;CHBs`18u>S27@vf-7x4*$?^{*TkM{ZCUKgX9-1 z$XP|8KJby>a1Hoxo_5SJ*2nms8^H%BckSB6qX>PO{(I5z;lq~{C7{h&DBm~T4Sd9d z{L6d-pF?t=v$41Py&OOkzp7k1-{NEVg?D=YR#9OfxH!uT+TZogpTDks(*msDb^6C< z$P5CkXMAkz&c^w_#mXQJ#el>#?5rPm^M6u|`GIg4=d&EpkgLX{bqZnxkj6yvXNH&l zLr0pN;Up0X9mVuJWn&$j)l+wEeEITFUQX^#SZ9~8m%+F4c(~W(oMVscK=Zoq&>;|^ z-3A3*bQ(zg5%Dib=AcFkEATx5{o#ju@_-v7qW!Ueo33%+ccI|&YS&?PfUnl8{@FD0m#c{iAZ30F0Un5Udmv4> zE&kGV4sH%6_@+bKCAhh;qnw->0ObMry$)JfZU)`1wl*w12>hTTrpIn* z{YawrkEpja+TCj>54E{Gd6ISW7BxpJHwwUEP&Ru#_>RTEs$2jCV@-`0$2CVec)IXx z+*V-uG@C`qK@$ACJf_y3ctGtCRMDXf_kFuAyzS(N57Qv?tI-RMlkKo%Z)bI^9zsf~ z932xt%G}kFsxa1DVxART$*m!CoSj8xI?-K+g1%Mf6%D3C;8nlFZT$^3^(fPI<0 zqyNyh|B#UKzrKuM?XSApoGa8_^Sj=hz568KW`zH#-_NXk(9|-BB z+OWR1U0z-m@?4pLzCVG}({VA9u6Er$J?7QlP6Es8@bDv`6$krJ1D;F{{Bepc5#1X{ z6I&v;q0!VcMQ<}Q&Rx8ifq!*ExcK{4RDP=*-Kwssc^sMx_6wxzYB%{}YrOO{H8n-b z_V3%5l6;DpN-PhpsTutE>C-1u=basVDX*YFz{<+XtTo}9x~}dqXdD}CCND3qZZIes z)FaxabgZ4w-7TlvACXm15aRFu;e!Bj5zx=-eiP>V@~G?UvmZOwWwt>(!OZ*yjpiDv z#tf(EbYVn(L}~RT#E4{zOixdrffkN|LDlkSI%J6EJGS-Rft;KPIrsH$0KyfH;1GerG2$f&4>JYAB4s;Vk&rax7_Ak99W z`qP!Sc{YLHw86>8_;{x{6D6EqL8t2reDPsa$4;x|A5re>CCJ6V{C?f(mU^8}8$KNb z8PF74qNzF4ygK1%zCT_W-xJY;^X7LU-)Ch;NT$CZu$Fj6^}+G_w1%cMrtuL ztKWP5(eazW@R8mv?ZMX^pkL%s!@+(B<#wxG8?a*j{>VQ0FF$$5Irv7&eQ$pH|JhUhkVoigd#y5R#&gc6 z;v}E$wfr#ZPY>liM}x~p3(qG7D26wgVx(OIZXJ?2CE+|BL*1zaIF+h;ZF$RWj?Q%E z%t*IJh<=LMoaZ*J2yng}4V<;#0j9R4^#iv^{ zFi@$kEyjc0TFgDZ{K-Pdc^cW(0nk-ZfmkMX?E%a9xW*^}pTshXS9>+pWl`_X%q8dT zz1e3Pq*7$KBQ<#R`S-6|n+(IfxN(+vs5yW6ROU>I&0>Tbu;8XR`ig2lOVEPWlY0JFMuQ7xWAK+wiRMN{&aEjn-<&KCu!1c{WI25%=&a?n~tZ zi6{$;8-|942Cj6f!~9#k=!sSZIZe&Gp^|e|f+1HI$JVBE3s2;|Tkdm9-ENr$ zTENSAa4q>aaWT(v>Fl3rC&Zk3^RABOks5E^dMUaYvvA~G<6A%UulCsG)_r+-&eH_@ zp{vk$Tdd4;b@mvZ)5&e1^2oV}+wu*xMuwHOwXoKNl1WpE-R_zR9$AlYb{$;oAn87eNs)OcA9dJqj4W}ND{&x}OHGr~k# z!>0NQZ#|poGkN3Gq;%dQE3QFu;g8gj`XeDqk}mIG;6e#TKDmX4U?!8i9q(i)>04=J z?%N4I^?sLsLEF!eym=KTwRckZ(Zpw6hhF@Re3J zWHMPEazQz{ZWufb;@~4(p&KGrRuhdI^o_tY-7fKe{ zqJC}fk?;wv)`CDdJtAeqiIU*fM2QCQx#1I#GTuJZf&JtG{js>J5FB1lsy*5{ z#iEV#O?Y_K1`d3X5^L3urW1L(igWgI$lLU~bA0}K%)Y%=4yRGOzHTk@B)7eWrheS% z_J8`ebiG{KKLCB5Slowma;`0`9Z|!R{A8HZ)6Ff(XAcKdOiatxuMlm+UABf?6<}W( z_x8ywe!MkOVwY1Vt7Bjp(8_c7VxWK=w^sao$MB~4B`qcwI*guv~ zyD}&$P`OsqP+%A8zee6Kh^I3h2<}M5-Ic_G{b)3gHu|;JgI*C#UnRALB)ZbgJ8Sj} zUgZvvSZw$AuUPBk8);xQR`J9#DmSXu(770Eo9KvVq35}LF0TwR{L@w+J>RB}{I<#H zeKdsjmXHXM-hP#aJ-td&RBv4GJSL)f;6s=+&HD^3_YOUPDcmD>KpC1gw_Y33hrz%+;ltpph6e zHRDHDxtJHWWQ_xFF-gi~@Qb9T5m-qNI)T`?nITyQd z*bF%BVj)+z{!GKF$+B}CId|7;F$Nk9N@g2EB!=3g8KqDJax0z4jiNr-(*m_KIp^xu z{r%t8aR!B}U4xz@&a;D+4!ze{`H`DJD|mu4y!%^>LkrR!N4Xi{UsN=h{{21Q#U46xeE+i#Hg@zDr5@(D(HDay*Kf zUdZsyu4b!QyWh23OY5X4QelVjQ^n@_I-Ln5mQbmUN5vB%oOJW8@qzgJ8*{bxcf97` z<6E$g)z#ID$$ux`uci0<(LSsn!?UcOcx~srrYdPyewY0D)2D!?Hc4rL%9$pvw#u%R z!j6{|{DAf9tH8i1hqf=LdHE>F=NZt5PI0J2w0*UfO=(Y*kk9s(+j5V%hMkDpO66CF zrce0Wp`1LysLh#L&jXxNX$!HlJYb;wIJ`MwPaX+ap_zZ`YU}uqh5U!*i4cX$4i)<5OjVOhTm~^=7O_^L5GuVR`qQeCJA=Db;>O z--vfxCC%{!+ZMO9nSu!v19{7a3Hvsuw2~>DakF~=H^FAmB1lSA>MMbl%q8x9v#FTs zO0va2CX`FntZe89B^JHw2tod4XH?1PKBM%g?rL0fQJkL0pH>$Z?i7@231326}{ zRus-vl8uxk6d({yaGkX*{M5+ah&*jsZW(_#LBeOw0N7Q3{N4EDj&!HY7zyfYfB)lG z#@?ijBA!p*UiwtFpscKYzwzb2y^pfZd&I?!{A*J93vuq^{SB4_CDFYF)w9Rr522(y zR%cH@pztUn;+Yy7$vxA!UbT7#-Mu=YKPSeQtakSB0* zUcnwyy3S~(ue=&^C0(OB?BOc$xqn^O`$PTO){>wpeVL#fw~Fzpjqeti?G+UpeA-Yj zZ`{Y>!=BJBaPi`+w?4S@+WZ%7W{9RsDkEXa7=NIp@(br0qK&M*WG&hX?pAhuL2d)J z+RuH)driu|M4fT!eU4ekI+0lUf#l=kh$YGR#MJnXR5(42r-O6@5~q`e5hpHzg#x z49KWnTaRgcioJnb-e^qnoC(`*$ooNZ%anNM>|sSzov|0c`7j>_8`(V!9w${L{iM~7syd_=j>>7%BcP9Aw!j1cqZ|Mt8 z!d%2*+pl~);xM8}t_Ql2rL@#3+H5DoYnSLBk=Yfb2O~~-cWONfA{S4Fe62IK-CBjFByZ10X$8NiB*uAg z-+?;}XYkv+=lId3-`%fbON}+NRux_xblDs#Y_{7|k6M+j*)rO>vyw9qxIS<$?e^X7 zQj;~FSQ>U^)HzI<>&Qcx9Z|?%-Y4RXqM3m2@Z?`|W?7-nG(})xx=W*vTKtOb)<$2K zhPJ+f+)-Bwn8qC`iZ`)Jjg!Sbh(rwwULlRGLX)Dkj@i@%i5rR}9f`Ej_=0zpr zDFC1-Qt7KizFFMr%G!FFzNq_W6>+9le3%VfNvO-tW{r&HpA}BlC-kJGrFlj~ZC99Y z`f&vZs)KD07MsRtlDDQQ_9GebdopNfxOgE6>nUOJCh@ZyX*0iAFi0%q~#=L4;gh zaf3>%`riMq&1$A|MeA!n`6ww%K({#=sl4n)nz{Ud63iDlP0?@HK+ zn;ffWmm$mA<t&4YV_!heEkGdW1$@Jbqwf-@787#UVOhIn8#6 zv!-@sz_yY=I5||6AJs`f1@}ZZw?N<2f+Nv}9^MHK8+#onRM!TxQ0YXG z?G++7%Zc{3#Ok%xonAM3N5E=+oUNuNbRv$XUivxLtM#S$;VOE1v~duJ%YdNF#?@;_ zxr*DZ7QN}cD(tO-bGD6W23eU`e|oND)FqgablK#7lhB@RD^khVt(u1JvKBE}<*90F zNOipa^kH5==cd?~KTSU7s3u`QwMB8sY(=gSaNFVC<;?=P2OA?%?|pxaPp2KMS;D4E z=s+ad-P@`%HP@N$w6$Hd-wd33_y);?l9=WKe)#_8b0!(19Erbo*_TYE>*&+g`K6Cf zdiVXtOD`XG(b@W{MyPk)|6(}&JfySY!_2IPAT`j^YKXNkj3}|)`tW{pvctF$%s|-2 z_VNmDG>2h1>@u;R{TTtJt{qhqNk{f6W0R84>jZjayhSddhmiToXa0Rwnv1!)xq(N& z7BBIOpiStI2D*Q{k=-7-%A3Uv9OtG=g2RNp>Db3m7WdYSG=0nMUfi(ZSj&-v;MjK@w z#TIQLNLG2vPnsD##%P~eh`&?YwNi&|-0-$5sE*!21~y;ru~3dOlGw&Fg`EUA zqdj(AwNqs;GqWtoB{$7w&es-hmnQ&MQ9H z4I*H?vrp~CRw)>KylWbqCd&x}wA)BWFk{#wmw%+SI;{L6a6(%-GM#ZDzlV^{Ie-)+pN(mv6FS7E@$R6lk8zUhv(Q~ zV+s`11yBlb(&Jt}wX;!;D|_)=i7wUKiLL`OU~Q1h^%b>bbVb>qd9j4PLTv0%;21dm zPW-s&bTRU`1V4!Qkjc|~>e;N1B9qq)ygnUM9Ob_K>h3zbpv~8O29wl7QUtkM2Ts$_ zs#IT}%-9#Y8N_}1GuD6S8NTQUzpgSc*8XnqhP z{n0;nt?JJd`f)m5Zz{$`M-%PnxCm+&1bsgJFM_1YD)b4Dd(dF)TnASO5#)Gr9whKc zA2!&t<%n*ulV>awNNH5)V;O49@y=d}0ZIE2D4jFP@rcTV$0@2RJznv+4I*ZBJ+W(N zyxnYl)|b$ovmxyf)&<+Q@@ie$(y-ffLQtZM9<60x>TT5UjP>6JUzz;?Q=BXx zcC;OITrwVE#r2haau1L@1G#1L@x?F6B{$X@aMI@?^dBq1LvvQ+12v-?6=NeyrlY}P zci61a=<+_cId(74@EUy(G{0?nS}k_QK_;>(us6@#5wU+ta!Bb-qmKEB~fr@}FEpc)tW zt0qc3{?Knfz0ia=C@mWW(#<( ztSo}gqkXo4EY8Did*~MY(VGPf-8t1u!LVeh*5y5>5cQ-saN8Ju(*oG)uU7wk$JdoR zI~~p1lKs+OgIPdr&{|zid^@rJJh;FA_lN&2n9%D$J2^Qzo;i~Ocu;fXx36Y-P}kv> z%i!UX0beXgoq^HP(1;&b+WGOvl9zcErzP$7>e*gLqtUDoB20Y#uW2bkU`yrX%+0l; zz9~0Ob+HwVyqu}vc~bf#3W!v=bWzb2D{I?tN{n|!S+L`qnwtFAg#Nw;Nb-o25TOC_ zuX(r4aR(v%;gQm<+k@%Kfkj+g%+~$Z_s#cM{%LlA4g%q)mvqAOwD!=LnHw4mxq%{u zQCrYu;SzCtX9eE<4zoJt!D)fnQE+e|*hGh?^dP^FoKCQTLe{;jH$DFTWnb+qcVI6`iKV zUn0*X1KIYoT_+{~KOmy{7h{+9>+|l~@aNL*(g7X_xYEbf-zI@X-HaAUOwG!ACy$9UZdYP& z7taEgwZGUx0rr8W4xxUq(kT_T+rJ5CUx@skfU|+sl)rJ~jKs6aUtz9Ni;J%EqxA1> z*-jnt)+;}<^8o%-i}>Dx@qd`&;Qw8+i`+-mzBal4cb?C7NpD(ttSoqD8`d`mMMXu` zfpdTSIVP5Rf9D}R*avY^2snq2&CSg&ye&odchknlyHem!p&`T*zx)#LgoH)GVA4_L(HFlw2lHqJYJ?__SyM17iKy1I@ID^%iw zs)GM;z)JtOhKRoA_q0UMgXG{ga=roLjI)j^>AU2X4l-*iE7PsD0OXB`h9N9$wqIbU z@r>{IKTkOjNBoXFC?aS}_L`e(d~bJ_|3Fc36>;kzFOdiR2G|YlTqfw@p1EU8_J5nG zt5*o0j>tCP=mV0)PaIEs(({)91Jom6QXL%1FX`+o*q?#DuMg_L%aGnjTIGrauC78479u(y)1H(MlfC)YCAO4d{=ZiDLH}Q@ zR#P@vUalVa_C|mR;>H8@&i^PqAMUsSHTwu9OVqc)Dk6+b(Sr`PuHv?Xq5AfB>+kYu;2mE^c@8)rlo%GtXer zz~TojYF!Ac#_3M`RAL60LPw#Q-vOP!1HebkAHI6+I!zBeo!pjEXohb{Ww>43mZ24v zjLOTuaA^N+6Tg7xV1{#lq9A145+d#jsKy)ZC%??(5CkIO(A9_Nt1B^e^nCr-9mcN$ zxI-l>KP>=*!^2}OhXnZ;Bk2@;+>lw_!A(36;<4{9Mt?q!_UZ*1aiPvy&vi)VM4o>y z5>D;5DQ&)jKL6%U{V1bCw#voO_L}A|$lCvzB66%%{MSxFpPx1^W##4uvD>p9XF7PI z?4vt^6SP*(1_5vzz^Gmcob1YUeR99-vb-i3C80%AhY^+WAAfD7_Z`Xk=8Kk=E#du*cx8z4Do z7=oM<;;SU8W0)7M{~>Ul^&REzPZ?p3EO69<6_ud|Jvf-`{MSIpJcu?+#SjPrSN>RW^_cGPmGu?kT#@0;prs+#WAHnWHG}Y{ z3rH4AJmEpA`HQNpEC861gaaY-CINpsM~+lWWQjIjARo{!b1N+^<(0f- zVCQmw+Wp3_aoCd@)uCsJh4Cf4E+iB=4frHp5SIwn|3#27hBvtyv((WIx7nB59tEgC zVa;X)_MrF!!Tpi#Y@o8O|F~+TEGY*0oWod0=GkTX`m}6@SF~OngwcD<2l#9 zZv9SY*6tLv2YMj(nv=Exs^uUr9RQ+738>yrNT!^&4N40KsF-f>Oxoivw3xj`L_+VvCZj3g=0;E?IBULGd@M)9+ z;`!dGAwHx{(?naMr1&|Ni^|jS!WODZ??7_*#4&w#ray9F0gtX9;~f_=YXq9dTAlWw z_^jBl+w%q)CQl&UK~UM#2jzg)c~0=J1vE3a+18q(kgJ{TQC8O=74iPd+Qzk4$E7qf zz1g{C*Djn>OF9g!j67PV)-2K4L*B?}_zZ>=0ib_mr)vvi`!u2;>(+rF0_G^b>hr`H zj-Ul=4wDm5N=*(^chNH3GV5c-F;^`eN3AIxsb|F|Ntk}oUeL=R_h>ix$J#{vKUFbg zQ*@z}g{pIR_tskFbo<*=+PdR@=m^U$Y%nwT?M8SLm^=Jhzi&&b4yLp-$zq_y-i$UW zXto%U4d3kq49^Y3qIv^FdoQT28>JP0AtM@^zrJTvL^hH|!1%6X2xJcjJ#NY6Qt9lF zPH4EAgcyPb`CzazmNWD>Tw;LS|M$8Xk#$ta1DXXEz>{I`O<5t6k+hGWba})kaYi=S z^>YYkpOq6A?pUB(e9^4&RV*`eM}}?u`Gs`RVaImRa zc-isx*84Lg^!gGRkML$hNTd+c?!Wd`7-6GiI%>9L2bL0MPoR2rC6%I{mPeScAL{AtZS>;`2}A^g8?SUboJ&(4 zm<5mYJqiy#BVImqxTi3MKa^CnBOOWs>orjFDFBP|#8;sRikm1Tu7>Z~)%%7xi(fq_ zE+|eV|5b{doUR-8)cgGcSf<6Z|CMT}bH!(IitO+qz~cP!koek4$&D`>YINakFKVzN zN)Wdur}RV@L{(6CZ8hjNN{SoobOq{SbZbd9)SI%bQhlcA22gW-=HHVP^?n1GtE zKD(^)f3WwSQBmg0yQpE*Q9(?IfPe~ufPmx-Dud*lBr6C=7Rj+yMo>{uat;bj&N(zK zBBEr;xs}}H)J3M}rAEVIgsJaJ5CW|L8dmZX1~YTD-M70p{`ml3a(>w?;=cnZBwYGvVoA|oBqAb#t{e;S zlv@4SRptsqCc6hcQ=~6=Cr>}X`fn*8pX1Qr)sWnJ@r$n+`a_+zYz|eC3>Zv1_SMiP zDNH-~g36W{h)CSzq@t*B1ND#k?36f0~ zl7ZZ%*Q0N&kdtlf4v4vW^_29~Q?Z>XOUpiX=FTSXaiUL#hhx%9gz$FLo;^njI5sl47(kMgbO9T zZ7iVHaUzGCPo?^S^=xMyZ8D?=P_q z>6LEw!%bM~Iu4kRO^5~8Cbp0m0&=`Fi1-eXugcrn8V;|?hRp(gEGqusC4j%u-`L<9f|cGh1H6(?@W2Zf$F(@ELL5Zf%8Od!`AsHU75*xUIU ziRg1FheVkX2_)nx&Lb0#BN{Be&~!eQB_)PTR*b9|O}V$SYrL9N7t4ibzd8|ys$oy| zo3FO6du|N1|^O zX1^aXAU*X8aV2?JZxC651j7a7ZbFEJ_)W@9e%`$$mWfE6&JbYy*l{s#C4Q8DvOBj* zsTsWF9IBhF&-~bXT;K3~qw*gO_D%QM$MFicQVxR3MD_U5U7{(+?j}Ad@@zpt%$=Jz`NBY|r&mal;i9A={P>1$qEpk8x+dU7brF zz@dZu7k~h9R;Q)>SwLRI#8TQAcP~_7FCle^i0g{&H^ya@8M(iyWScv@SzuD7D|xT*dP(J`r;W`1jZ z^2xPe1wv)l^28Qjo}(j~(o1@n)UACT;UsgfgUM(=Jei%DV!l`h9R2vz%9j$qU{L-{ zww3Z3J-mlW%}fpmc>inrAG1(T3GhIYb?x;tT)BDSL@nNb&~|A71qQBjghRi+oXY=>3&GMpMv``$KRAXoQ~A_);sB1%`FQ<3}~JxB8D0#bzo z!+q}cLMA-#77}KDI!Iz85azVb0(g$4GmmG8MWiIhn5-OO2KSjc)(LJ#kCZ^!bR zH-k9jbqJqC=1^fS*fft3_e>Zis6MRe&~Vco(Q#0V+>?c-j{Kym^FWl4yi`o$by;m7$!CT#P4;x&Z8k5g z<>B0{TMinr{K+KiR9)XWUfcY$LCyjKo-=EW7xpC=S2yM=%~BN0oLN}MNjI$&0)wGV zf!IuJqmF(MU*4}oYL5}e%(>BQ9l!k@x*uCu(T{axA~vEz)>acl-NZ4QEd;B8Y?+hU z>+vH*Re-pn1gDxGR|{)XLD<3q(YY;hiL9nw!KS_OS_vGSw=otX`KiqJ4QLw(@>MbH zl@H)vgTV2drJlH7TDl~bX4$mAUjC8(;=-Xn{*=Fb z@lC4Yv5apStZI0!Kb5PWU$;2;hx{M-=TBcg*X5gjo_h85=2iK_iZoYlJ=Oeh@J=|H z+`#86C|R07tt4_AS2#Nq+!iNj)3Mk=zrWqF1JU=>yc<0B%{Z0&EO-Bylm5itRKAJ5 z@2>Ci^QUvr(k>yn*L6%{xUx0sq;4ttZMjdokFbNL!BB}+L?MjrgYkOd=e?KfuJk?G z-=o3fpP}c&)G6RkGZ1<4uonCd`bqn(zE{(7&%#!GPqkvu>qV zt$oCOzPs5BN;_ms>;2?koS=vlWou8~c)>Zg4@)k1;}+ItC7`uzDokegCE2-oflu=?Hrh|Rv&h5u z?YsLJnF&+>EwZ)mkSEUBgFaPbd_3~|q~4H4{~k8DF0VDFF!s=6jmV&>` z`z8tvF1Gmj`9ZscV>!%w*!#{2{q21#(c3t0cFr3VhcmVWB$pz>WRF+l*=2k*?B?K`G9@%*3SP0it&sI?iQ@UFz~cYc(y**E zYUiqmh(7dO)%qAj7_F(;d}r2K*BJcPaeI6F(V%@{%Lr-tB-=HWXwAT*?qxff=DJy$ zw@LR@B{bG7KX39U|F7bak-{n8`-p;2Q1&y!V7qBX8m*j#s2XYJR z9>vCu=h<5BY;Rwjr@j8pTm_qKL(`$+Sy(DZLd6Z&x#5*x&$@4V#yYakNSLR}& zXM^i~VS)9!R(nNluGDn2DzgoJqU@8+*Z>+yi|gF5`{OK<-9mzuaHM?-jGNC3o>cU- zIfcT^3Ccldz~OTLuezdKEg1LDrbDG$Py6>h1Cf8DZf4jo|iHD zSi@Z4=Lxe`J~pcg#^8Ks-FpT%*4MS+fKyv><_9J}?0@qrx?!hxmuS3kabf6V=@sY~ z{Nf~CcLHlr?0mpR3Z=$9x$^G5ac7yAgkHlA--R}zKN>J~b(;>&{b--Gv@0+IujH56 z4WqJ`@xqP{S#(VPdM++5=t;pRi8o?p8&AVc4Wj4Sb|GE>Q~KzJ1#VPocKdTey{;BW1$LWBPk^)+Zh? zN0+Z$Y0$N%J>8<}J}>1oDDz~Brra548gikqi@+$_y(IPO)ho1cN^1)iTfFRvW&K8r%R1_eS>w~!0VUc1?Mhl>KLjmWvx ztv(8&dubj_H46*ZxT)+jIM~@4O3o>2hgfTfQ=3T^h&$1~dE&M-?Cv;Lp|c@NQErVc z6j_o*$xMr|;iI1M%{3g$@43G+X1_mXraoPAP#+gy6!KdO&=Sq#n8K4_ayz&jcMZQj z-HUzr>(S0@Jq7|%?5&S)qRjT=*BAVX2cJl<96GYTkpG_2np`yUtDdm7rj>7$7CozI zboTQ1rYw~p-_Od>bw{`3SCwzMeaud!UC;2|=RmhD`S{ngfmBw;l}oC{*y+|7H5b); z#V0;QU)LJd)h}1JYW2;Hl)vyM*PyW4eR-3CSWa1MO^xtaezTM z&&KCAo6?ID)6^rOd?z2KZ~Dzhn=NSRkoz5L@e!OU^NT~S_%7pr02=&{cJTQv=KI2v zdlOArs&lVgHTnyzxN0xDE3v30=7h1U>ybS2d+|deC#Y^Dp*KvhPN#?lG#>RcZr~`g zb#Ey(?#{fQ9Zkrv%YfrAF>@E==k2d%b$o4xj%QS_>hTqfv|TJr{6y1o!)V-fJ@in& zl1$=Fz2nD^&ks*8pFOOP8``Gfux}|Fif)#y@FnCBPF{fOWTOhdsDOR%1Y5F~f_}jw%bi(u^YntI6WO*Yxu#iODE@;LVUG{axF=5ob_f zl8UFCRx(;`vf$aaD7KT;(m}?JTBcpeZkhVM%AYV-Z>=0Wax&ItJy~*gTvbJb)2uG5 z-UzyP*YDPR_*fWE2e>(FQ-e-eXZNnBA6;`JP$v{M}B8_E_Og0ZrMpNqQf&ybhe70eqgb&A|W#=&2L5 z1j7pKN4J%Nggbgdat|2jM($)pPBgtX?5#sdgV`}E^HK(rF&ueeY<_o)Onogwj8!_p zV0WyPj`t$TF3Rd^^;P*c6$X>~vr7WEQj+Cv2v7ELC-~kTd~=DThMKW)Pa9W|BB>om1;RqK@1=TJEE^pMzI&{#wHh2>u?@@s4mh zb?fQ#7i|Z3L5Y-jT-qgHJLANz-LK4eQ3^MYaeYfkM~yq9`;1!^yM_-=;DwiR-k~ay zgoRIG?N@HnZ_Ju!%Q)+Aq=tpnxY-s$i=IdL5@#W+9B=En6)a2tIgMV`jl7OSNA^V* zcDplL=lf=9lO*YEa}2%|7Qy5t?a8H=^%^F9Z8koMcN+jo>Pu}HQ$1*AX4VqH z5e7$$n{u{EM>s;VTsz)0>HVeVR9kHOo}_hcm<_89#%1EQ%~#D|pF7Me6rJ%qZ4t}X z7CUY;&9Ji7&~)zMP6CtodT{bmuW_lhUpkjwUd-j+hvTv|y!!#{=NGyk{&JBkvY|Ya zlW@Dc>_CFQ72pkHb_v zZRRv$#nevDbXHY{{+ri6@bl=QP3E7|(;4~{yBvDKS^5^)gj&rMRpyw9HLxk@pC+e5 z?vDwFxcjcao*p6G!Oa4CwBa$_W$~A?t$9E}v!DQ#g<)NrOQZ^$hL?S}77Z*cK3K)i zXJxgbZ^gl_^5nguB1Y$JwzEE?=3?zmn5Ri-2^TmrReDd51XH7PcJ95^B}@~SMvN#+ zTR|m9spsb!*H*mPw(W%vdk%E?Z<#*bH{02XGgs#{%gcI@w43+gUh&nSpVMIdYrwL~ z@101PXJ==(8z}HcKhZeq)}vz`cFv$%B1ri>pY@+z!6|RQh1(Zq4hVKC7noG{*65uq zcj~)?^x`E=vg0j!DQ0IS-L9(|6q@NE=FM?Zy^cU}{(YRPSS>FM2YuI3V~B65*efyJ zm_gabF-v^kl6BC>9OZFGB7*fp7Z!$tgB4<13QS)00|Kq;Yfcbtx38UTiv#oJ(bY;O zF|=qTOXYKax9*obI?i|3v^zs3Mr{dhb+(e_~@FPn1L0kc*a9^-u$9m6CJ7SDh) zhTp2cZTM8^IF~z(wBrfa!~^!yv-9p1*OpoV2vHRTTW;taPmVjAOZw=TU~?ldbnQ4E zJHBa|78curR!zts98ACm6y#m+o!FI4D!qKpC%j}>qgN=3$FpRysl(LQoqaDaE$?=q zM79x~4EOg%mkDKXedjd(W~cyLZr3*teZVR=BLSzf=~F6|w38$>9_?*cs!fCBuWR2k zSxN>?e(*qHC7b2~n6-jRaMQi`yFxfPx;ekM2O z7?off8{fm?0`fX-F7=X0)nhIH-Fmwd$%Z@l+58;S%xoF+B1IRMnW1NE7X**G&TN|> z3bJT=9Zy^t-^iV=h9-v(xO`VIG-}U^+hxfgsyFI|`uuloH(E~_w!dZMXSjJk*22<~ z77!VEm!=Qd>eXK#Uu#sHhY6KE5Cv>D1na0e>K2$Z?0e=$x2uwZsZq^O-a6oyPo6gp z7;?I}zJjv5!1vb6vaH9W=YvJlmRGU1ynN{C_t!`h-LZUEGJw z0ol~59o%qV5V0GpdCN6naPRi*=}q6kGdh>AKzwre=&1owZaU@gIt)e+0D4;f+*~f4 zK|X0SZrxuttZs9fvBqTN;yH(2;#P_maIetMH!8GlkDvP$%>4~wj#^Jjj(kWAV`%n0 z{%<$!>vcId;<2nahFlnGfG_S+IPWnPYgB1h@WiS$YsOM|^U>S+;Zjd*8jWcUCe_AB zDg(;W7dG*a9S64qv)g4bDV3_I#J)Go8}IR+X7v_P(S=o#CwIW6PA3c)p!Qro3;fir zTs{|6)(1_6bo+9msImCC)l$2moDd(kl@Ab;aKhDl{ZcHGNs>5MyCSL*E)r1txjxwy zU^z6tOrT_(J$qIct@^YhsW!DQcYwZII3- zP1DzL(~;WSWRB8@4BNi^sVHZR#5T3u5QmOTADX2y%RP5yeTy$mma?bQ9G21>osGG5a&Pd1sjqmJp9-WbuH?7H&3;w(~WUxMq z`ZgJ-(mfd$^X^@$s6wcggF~Kvp<&?)YFlY(xI#CID@wQQWy&e$T{Q@w`I98Ys&dHA zDAY!&&9taJExq?`B%VG=$Cs@dcRnI;VN6T_!o`i7Cfra}85ET)lZrYs*9(1_}*E z)}>$^L}llv8)juMks#S&sKjmQb(0BIfB$_?#Y_iDxiJ`XRjDOOGUkq) zoNB2RQ=mJ2HW)fPTtO6B{sr5Q9|q+Q@ye~1Us2ny&>F&>-SsSe*&Bb?xG!b#VU6sr z%OaTJr0~i5dwHQuM!EO+H&n;p^k#0k4%d2O@n-HODbnKbj&B)7xq1fdG(z%j{&QjH zafLC&#T`9tFcW{TA$T-()CdxnaxkOXK4R^Cq&Di!O74(o_V`G*VPh@?;3c>fa#6yf zb$Hh!{X9C#j{fkY3q97*5q#~!DgA-F!F$G2EwnwQ9gr{tB- zFHYiHUzDoiaT){89=pQ1GqQh{Yt?GtCx zne)A4l;RCMO+HitSNS_My}nKS8Sup4M|O6-3d%8!d3ApGtb0m(Xfy6-TJXB~>UjfN&AZH-@D~jDtUBTQG{7Zm!I%y2NU;FQ!!Z=d=v(;WTQ7-XAgM zemSqs-DPZ1iN29`Eo3$aTu~1{P=oW`8G+G!=BdP`eXCI$m!(kL``iT;UKsYwAW(%x ztL@wZt~j>G$9+L2jgq&gO*Wgiy^iYoB^5eC&y&BU4>yP~aZ*}nr8Jv25C=QDfRj5m zUrWtR0Mi`gzva{E>+74DQ|P@el|>`#rSK8w+qh28HrBq~1zhCrdaH&WT9va^Lfmym zC~CSRX??|`a4VKu9~Uv{UGiDXcSvHoY9(D-pioB@3cX*VjkAO{-34 zR7twDL~+NQpykRsaeF$FHP|=zv`LMdVY5bJH{9i^$;pqY>sCBA4{BNY`Lz&#H$O#x zjg37^v*9_He%{LybiC;fL)rnEth~#ipF8i)wZ%QY#y-56s^PRliqX9F)H`XHZEbB$ zB48pCAWv#lmDu$62!(6P=LPc`1F$NJie1H%9004VAgnb8L8ey2^CPufZJ|*O{)ioZZtrB%(1$zqseEdV+2Nmffggz?<}Z2Y&!V^v9ys(rMzA07I`AJ`i0ru3(*wU% zsmPDaxBeqoM}DNa{GUEu@YB;{r~l`7eLDPS*Y6kze)s3WKmJEg$^ZMc$ol>J&;MQI z-z)L|^hk*d)E=h%rEvAEursI*FTFJ|N#v31oL*YWFrs~uHJ~$|GuKJ_54t*V;GN;0 zUA}|wU;c%;DNIB?^DcZNZEljNCd*Rukx5}v*f*8t_HQHrkMxc0FMDZ+@vl|24c>fK zUcii#rCeM@+D9(VK09{^MPt*Es09{=&}U$dhuAu9j44b-RkasVE=Xe?q|^LBp?T*b z??FR#h}ZiI%?m*XvH+%_V{!TS8!sPjOWi7hFQ>epJ-RFklT-Vnd5p`fS}#!33;czA z1Aq#FT4Q>ok<|P^VG0BSR>K1TxXH7!l<$9=9W>M-?-I)4FDP{bg$8p>)gj=WSd#qr z8$OX=AHW0;*>u|US07Bje&dF!>ug84^FXe_3d@;9Stufgpgun~_F^j;C~fBDGbJVP z!;9cxZYe1fc>WkDjyiPc5X5lRZOie^LQbQD%&9nxZzVKz&qp5SN#XBzF{I~Qe$N{~ zCK$jI(8GA7nLY@>M95mRZcEc>4S?pp?{L1ytM>De8noiYoX%t|#H$v*R()R)9Gbu!ia;Lp<1#gxay zU6k>w>iG#16zzJy1!8XNL;uw>XfROjIHC`apcloZ*IVc93cZR9-!cem@2pSLUb>VA z`n*4vjl`k{jptc3Dj5EKm8Y(M%)RH$JH!wF7!_N*FmC2ED!Y!UjbsOJ z;!*?ZDdhxF=YHsHBlM{2vas{G3Ct;SCmHpq0N#9kp3{ONwZA>dERE7%Z4>OgSReT9 z+c%*~?};zR8=Z&HYK(fh`t7=W8SeOxA3khCYZ%+%GFzYND1*hV?*JNmMw{O}{`u)( zzBzVWcl?9=NoocmMHu7`a-`ern^cBPu&U5YS`J=V5$Ct@w#Vlf4YO{c8KxIL8YU0< z76m68q~1D+A@my)mt^<1<#538uoM4nRCnNA-MOI=0a~n$b+*oH4Jmb@NE(?B!b&(RIJT#d&?4$~m8zZ;J(k5GNWd zgfY7t+CUF)=6SVYWRm^$>)))BU<5!GoEYXfVIQcnrIJ<9kA~0;s%G3A1!ZK+dNMUC zok=@88C-(5^Nq@NbM<+kEAcldh_=OOeY}g$ObJldG4C0HPI0PMfKfXBPQIM`Dl`;% zg~ZUxt}abshe6Hy`g*j#e8>fN@eBK>USHtJfyVR0l`%ZVGsN}(tsem|h+;mpi8`m$ z;H`9mH~$80x_QaNSFPkw!RXQ)wFD9T+>ehi#c<$z1V_2Yk`X*a?0nOgFVJfr*S5O9 z*K-&;ogNse^Dez|t^tmH?&8o`tMVBbrU5-x?O|siGAiT+A|5agvclzdeFA;Z!^NbA zASr#%KZO(X?l~$bfV^vHJ$Dg?7(*RGI@CVmVHiUf%v3J-TrnfL{umfAfN5u1$x_}@ zV^(1cU1@<2Br_;mzwa@{#E|24h;Ou%SPC1bB4v1QTXnvV0@-^7XAB`4DhpLf1!kCe z;_wm#(;^PD0KTn$a==Jrp$Lp-+8w8))~Lmy(p=Rz0WKZ%8_>zw!i%c~9-|TIjT5$q zX`svE(raJN@s2=acf?w{O?^9qSW6nI)u({}vH(|Qd#z2VhlPZM^t?L9L1l&7-^t|s z29-RyK)f1OxU5acBi)oAJb19XP>9t9+7w`@l2uEji{y2G!n_`ngqtFVMzZe1hv_hF z08;wG(8G^Y;nWs0AM`Kj2D059?joe)4FQqqWNT;W)_OQfuYNil-OR?u7F%Fm!}w4= zKoxA>4I6?)smtWuMi1yUI4aYx4DD?D;7axtm>R=mg= z%{Tw$)wvpfQiXE#jqE4WEu2C^z0=}wL303Ml04RZ_kT774%hqF@|xG@Jq3tG=+)uU zPKSnm6JR3Mbc`WIQ%p(X&|uEo()RMI0_TzmCzzX;SN-6@gE_j{;m~H*OGR%o%EeFe&B<^|_>>JYOUa#0f9(rEO;gEnRnAzuBJ;t*66cb$10NEqQ zjv3dhZb0KJJZS|xur}FD_;i>i%dRg6qKEi(=m~6rjrj8A%V6^gK-lwe29b^N+gXQ* zzj9<42@3G}mqPT(+3Ne{fBe#ix$^QTY0n-K4w zG=$#V1*{68bI7W~+%D#T?FX=@daxGJl&yQ!1NhrZ@_QG)$j6+5EzT2Q&MQY?iVhn)~38EfeVtr zmYD)9q7kzwYTqxW8AwQEUI-5)gl85O><8%>80JefMk<{}%PXC6GL;23>r>y+maJE= zCia73ctDMitCu@dw%?H?=?ML*T)dZ{fUwvUX5)X|p!b6g1h`*ysqJPryIs zyDW@2uV7U$2}=Stt4GAT?PARA@m>useDK#|27Xj28kHn79k3M+3j>9o zKFP4f^z*nXmqriruDdbRFRME8+$;LDwBibZ6516^P3NYx*tiM|DRSMx)Deg6gDe5Q zI0L#V!;qG1(L}KJ4@!p|D(2=M^?1YTEPZ>=S^>M)T3`0)$LCewjyS<)e0iZsb+=WD zrGV}g<)q!+;r;^Cg_0z;;^N{8P;d@IVWY`->yIBlqMKKS%Ug+?JsOhI z72r`hLv3xKvxJ46^XA--tzP{nc*2bnG&GfjxgU+wE33bPS%#g)>+of7%R;N!qPPu2 zF43{Dq|C4UQ=kOf|8Bp;Xf*cx))s$m+=kMrM16Ee^48XtsDq@6CHC>JJ0u)fbafj^ zo7QMG@NPl~4`~e3IbDZ?;cj27fU%0Y<#u{8Ae1e{?$>0mfS>IJ0+h-MyT6+oFXEUJ zFKj=b;9T!AH1k0|1o^Y_ckd3jLR1-%6L78l(pzw+a4O*-r-H%fW@rD3=LaLM16BvQ zsI6oWGT8O!#oZ0PgkK+pQc6H&ub4ngl0}&TUg?H~O(I?czybCXAQ$F53kB;t zq=#dwmUPt*2ll<(cgu<70h^oH@e@l2HjqC`-1-Gf_8v;=w*IXJ2xj)4I@1oDVk?PY zg;MKwlYXV;sqYbhDV_n~v>v9rbSW`sMX(wS2RF32hw-;hjBMcGK=no-xaUGR@U?wT zy1J3b50$upk+PK^3M}PPGw@Bj`=k|G1_l_>PBiS_SwT^92h4ocAi1@>g{9;}Uls`r z5z61~0DGa=x3|6CvdEe>h0qPWyCUbdKBbg7aOT{(k2cBPcPE>}JdG5>LobPc09%0v zr&y-Sn)0we&yY(mC$b}1+WvZqq-O!M?;=0UvTnc^BW~=`qb!f5;pq~-&;oO80k~)S z7C2EtZgX9_p6T^RrL+HZ8Uhqd$1HjOB$X01|St_B>rj) zHtqjST$bX@q2j&s$LYl*UVY^e7GP-o% z&1eya#T6~78!hxcrpiQdo2r{61tY5?`Xen7xa7ED56 zo6%+%pK4RoOj{g-(4%+8?r_!@HqG+QFf~@*urBdnT_#$O0=m|NiB0V4iLn+SGbK5K z2hrKt#P{eBv>QRz4z*6=t^V)dzt1r!Ot$(t*aFZ!`|LI4K?Dk+W&C%m+#TT4fHu#B zd(xV}8xA*GHm(H*S4GeFlD}unmS#b_M<>b!#IM6%QCZ=oXOI)x=wWC6@;40G6Fi;T zd5C?pEzOC55lYMSZJ3_sfpOBslI)H2=}+pX>%AJH#H;Go3qUc|uk4Xcsu@ z`I2^UyV(yKgMvz+H|yt;Wtf&gz2L9nc{qwb8!}vIfoAvaTK5HXYU*W@2iVJ_)C}oI zPF{EfH#6G)f<~NxJnT_;I&fHz=DO0N8`RCs%>jmgTU9dhtpOhxM0gKn#{jALpdQ5( z6fPBs(~K7`Sc|t(p!cGX17J;CFs2eP<+{QdSAdwgt+(`oci}T`_M#@=imD#`4)zk3 z-U#e(_k3TjUiA~J#(g-13lQ0ar@KskwFSI?f{IFrkPms&poK!?%7W}r!O*P`%OV~g zC_K!EJCx7@j*<)SWc{~Iihq9@7WQr!(Br8~VyT@nq_)V62jAXZ;Y^m?affQrZ1^0X zYU*8d|M(!f*cWf)XrD8aAE3p!fMuI^wRrUaLqDeOG@mAs?I6YKG?IbZH z#!MTonlb%NKX~zcV^L~msloO5aIDa|^XJnc8pwjHhxP$XhK^B0;0CA-ziIYENP=8< zuqWMsQEsfLw5nv^Nb&1xs#&{`LMOY^ zgOA_nF`!KE-J>bI4MS9%!2+hF@_`8L(GnyM#*@o{P9)TGOiw{q?2M z`+j?C&FFox0;^?ZyMffoAqVtA)I&x=>o?isa3vg}(!2r`_uU>a!mKACm#`SaYigGh zGc4T2%F61vzrX*FPlT|^dKl|8Ceh;!Aj&z*b!k{wCy7%B9Xl+6+OGp(VT%@yyWF z>?RDmngq(=;ixf<05VJr$kcBZav1C?QYXD1FrqMwNI%c`zF4& zrFQ-W@uTKrTT8>ow&F=jDA~3|jVAzoq}RW(1`SPc>*r~@xs|2Y3XVo(!dTF#G$(3tR9CoXH%}g%F&Y9-!5+WdtK`Lf1|%Af1UNU@iu2QaRx6 zN?^7pLV`;6$)tpN!}iOa(L4zOJJWRS$y1<9nN>qTzH^_zD%z^^WUl8r1pI3mr-|*atX+haXD&)lZ z>-QyiZ`^3pC4OTK%aRK|gRp_2by(hXd5%G(3}?7Q;{8}T*WW0x_L z@td}%I&2&qhS_XQ?>7_P!>T%Niz>ltsoD1$e}`lR=R4xgYzpIZb7%pL7Vge}K8}aE z&f~f>7ri`YKuKkU4ZwU2V=08?iYI0&JRMgCrl-PbEC(0^JMa)_LSh)RwA~<)NEA6z z3mrt#prrN@sMV1$uysTH=+H>Hy^YaFnOBytci!xt3TMMeg|n3(+y%1DeP!Hqtk%;2 zwi6f|XIscKM%zE6VUZb44v>Wm3D{17#TD2+{B#f-@IXOJtE;Q$Ng@Gz+iu>sr@PYR z&)y@;Mt9scp#$Neq9@z!@W*p+dBh1=saio?=eaP7q))t7 zQ)fDoav=aT#ZvZbvpc|_ETKjmM0!;I0z2eX$bka9(zDW)t;5k;gGA>LI9OdTJyB=u z;+K6Z+?Y>$ETmT^#I85%!HEfA#tgwY_JM7)@UgvK{f5CcbSrOCMfN{-HrzKPnN@C4 zHYw}Dh84Zn3h+AG9{y{VpZ$eh$F(4iq!N4qfWaPd=S3IXr(2>BrVcPwRi8CjR%&|* zjw4e5nD~^2t6V;&1pr1tMlMCSK5m7^JZe^8Q#_3*5G2NnyP^S9MNAkq;lVx=m8n^|-_r9Pr14-NqwPuNq=cd5?_1wy{ zdb&T2TlP=;5G6^fA)gK{> zv4eX9$Ibusl7fQ5h~S}nMT>#7xwwY;j6k+*j7|7?Zv8f0kexmduUL^cN|qOY zGNlOzP{@5=S8fMl5eUPs>J^w6!rVFoxX_Rv~-4HvDf%IfMa`gk(l)^LBJrk2~5U1;w zqYCr7@5XQ&bm$`W4(^MCTCj0FunRg=9hor*+Gh|}K62uG7o1Zas+ZzD1*VeiDYCdH z=HRg)@Zki(s9NmJhBDmzwIGCt4%4FuhFiY&FlZR+MXKZ>fPw)M{1|A!7&NrlVQrf^ zuocc@yemDEf2gtGIYocyOy2TA1$^}-dAkCf%iRh=a}>FJk4)}^b6fd*BQ&pGA=gIU zF4WTldv8ooUvM)2JdaVY?u`1^pvd%r(mBA*`tT(;wqCq=kqy8Ui8bme`@W6!$b#W= zYxzumpW%H-AlMu%c)l5G-^)?W@_a)n8A`z#Jzz(vMgVX*hI)LinjgQv+?3t8xl(A zKwbdoUxB4ub8LOkDW>=dmD~EbkGYYH%a>9ZmOGwS#^1)GDEmT5lF}*2dFulDyUc6N z0*={Gi8@4v5I%f>{PbX}fL$9ItZ9&=XQ}A_Svf_Q6!|J;&y~4=6CEnr@Q73+02YZ@ z;543S;46*G?XCgHCM|x2LjWvLpO7WLkdTm`pw5aOd^og`x8(9P8Q3i!FzbOUNNuGS zFZ2VV5uknieSbbe0IQ(rN*~bKn>fn;fF?4g8QLdgfe4w{+XhT!Kdi`@9fZE|pleGy z#VnNt#vGz=L$n0Ex*psTpy4!>Y7(@r?5vmrA&7xX?v%ub>9&)cyr^0mK=#QArJu9m(|SLOI1Hu&p_VWllTK&mbsG+ZZoS`*fJ2*jeXmZ@{o9qpzf-Pzr4T*c z89m;Ob8hg)FXw$@miD=KH1CqlKrJ_zY9D+?43%x~K%Svx{E;!>G<@x3%VGS5Jr_vG zDz3bA2Y|q1RQ63X6S!NW8n^5xQ;H9{t;OAVd-d zjKn&tL7n?)6}-m>V9t#DfqJOTUJN+Zg^)(S3XzjN{yGt{huuN{ptM<;XTnQt}!zCVxq=GY`@H_$;I8?$EvKDhu_ck z_;kTUwc1(YmH{eVkVv#)o%_B4Grkcak%bXVDN7&ptgSnqf6M; z;vXvFq#D*l0frZYQ5FgZKiEL0m6t}h|Y*huqJZKQ?PNZI2NBf&9|gq!!PMFMof$?*!ZeT|e=tOW0p1_815#xCRmnMMaoMK@c8#_4$=DLdbj8J1lK@Hpsa z5hz^|unz!^aUPj;DJCY?NQ%?C6!;bFgdird2p{Ooh^WE{v(68Y6@snQ0F&4E?C>#F5+DjoE|vN}l<&R~Zw|QU{~yKmy9t>P z<>cl40H{9G2D+^`Q-cKq-%1p)DV^7ugmF4TN2(*zJ$*=-$v(=6A`w_Tll~0?p%3N& z*mG4HReFt9xj-h%0KC0Bka*zI&4Ds;A~fe<;CYag0?=x2%YaZPERPvOoB~Bf~|W&^bIEsKOkeZV>->=gu7{e-egNSAUt48`$ne z6vC`Ptb>m0L(;hrJR{lNs2hgbfaZHbnIsqEj)Q)KG0#>aQn3D3et%{Wh@>10>na%yNS;uFba6lU4R3+6%K~fEHWm*s%nFMb%SJ) z<7kx@gs2g0!0;NOB|8-lAEoN*&DNoQ*8gwr5!ng2SZMQrou(%yp^!w3FjeyetJMQ2 zmC6c2b}a;nu_~N`|NOo`lAaEc3ykXIm)3xr^#Q8Ua)9MP;swZ`K4j5B@Fd?AnEO!p z2(hTR2l(y`gxvslQUKRSr6!@;5=`+h%D%)p8y(^%*0j{~%eqTfpZP%qIJ)6BY*k*;dHQ^*p)S zJx13Q{_snzYul;SLCdfVFQ(t&HHw)pq20Q5QT^fHjlp5c;Q*Z8eV69X5#FW*(mC-~ zn_}FcTFo(OU8c{WJ5*^)#&RqQ3}QAUYQOe~j~ao14575ylt5rNT=XFD3Wfys?K}g6 zoftdFq=8%T@EHgHKnrl~N&g(Ay;W^n#|Ym!QXT;{mMv4D6v2*QeXG@bHH9U@(yU5bg}ZbAnsT1!4ldTL-8LNg$2PmY%(E0S)$RsB&^*Vsk1&Bi`De zRvQ>NCb7q#$}WAv?i2X(4NJe`U4WMAg#cnqMgztjP{$nvt7B8WT*FrGG&X!XkD7|A z7c!KiGBF<^L6-|2Bzrd?;`#JHeEGBS zq=XW}LmxlKez8S+*BK9A!ljw|G;I;iUq9pr)!b$GR^{Lnj%q>?EOqN56B6k9QJ^#jx5j=!~QDI?Wm>Za{iEg64w;+|iM+9fNO%k;)!5tO!o2Bdy zV9S6u*5#p{a0)u503X>0%!W?6gCZE7G884CF~k7Bb_1px_k+M>wB8BeD%i9Zs6g%Z zY(u!K({s+fS?@<2f6s45!Uf;h%E|<(pmM^;q~v;=e|_N_YJ}6b`!@6!guQ+ zEU3rgz5IK{caUr<{0QwtcO`iZ3KD_Hyfyh zHLaf76IFf-*ZvW37d~*fO^@CgDYN|u2HqY{8rnb9vU99>8&VzUoI-$;UEq~^3(aL? z+yi#;97tdYfxG8I-SXxDa8SC@%sA{M*{LrEAypO!i=&%^gM#J}?E)kp5O2vy6h9gq z0Z3Br*r`ih(8Y0ja~P6zLasBaOVIiq-dlLl9?UR8f5TTvNRNZm0XWQl_!^#4sw!?V zv3sD<`~g9NNhk#%EW{8>NDc`?ZfX)_nx~eQ@?*>c3^O%S`oTiNAG#fQ6so`?a_$v> znBL#K=o(HHCz@VssPX$s6y164OT#EbsR)p`IP3A6F?e|^rjn+5j- zEL0@O7_Rd1@qq*)8{m(6V0-w9yXF%~~fS}t8pAd|YUk^b_j2_V-5P{^K$YX!bb2l+4%WMsA; z-edYpL1AGFhZ+FirVG)@i=ZI%PE$Id6f&R}=&{-bk$MIwo^rJ_l$}svkPs&N&On$3 zy&H-UCCEfm2yUaaDGEU1zltmZ zw(Q<^jh!9#%mO0XaJQ`&uizuRMq4aq9`q~n2 zvFL45e6Oih!DoZ`jVk*^QvMmLDoc33prJyDH-xN&4m@Q8Q=|_`{KXN-F%*DPi+Uyf z`{uL70P~D0e6-}B`T-@<&OG|~Bdb+Ni99CHw14m@v_~`7pvuB0C7a&FV&_1kB zb^gr$sbudN`o_Ew$QW_)^75YZ{o^t>BLMRUr0Ut^*7W4xudsZ~kXvJ2_Ahyg10}_7 zeeV~u{)8wW z;$qx)>BH~(97ZZ2(sL|vhEWLykVXFfh50v2R3f20I$6KI*34p$aFmBD91}o7 z^I65>-%ojW1QA6a^cgh(F+nz@pb@_U()Yh%=OclAY+L?6Bi5hZxDF@BP?LTu3WAhe zu$XaU)W6^AzyU?!{{MeUpc@EBp)daB14@70*8RP$@Yw&~iN7h$e>GlCl13?o z>t@}TnQnVeW*K7teS$urm{{lDTN5vdeb7%DH^^0AS=ejheq|)eHdXv}UrLTbnuDB^ zH@n_$rQ5WR`--5heZR9>6miXOwc14|<;}KPO4ucCtZ`*ejikP5m0Q!g@o>F8PPRhr z_cuJHNZjq3tS;LLGi4C9jb{5kvQX21BQEa45@~$1=hFjoDBN=#OB=A$*Mt9C$BofT zMm14JV}q_RlC8Q*%>FTUtEcKe_xOO98UJ(U?1^#mvH4NgREe!i*Kr8}oZ|sAZM@L~ z>K*htb^aV29F4vtw>{l(*7C66-pFgif%NM10_hJl_L+kG){@{aWjWRo(e4J}50^jF zBA>(FY4Wb$^grjc^UB>WR=q#|7%l$eC$LzF6H5a*X7ZB?G%>jR5xe0c_1Bdv@jOsp znv{+?7CP5A&&RF99dFN&ET50Ifo01buFiyAo)afdMB4;L zQpScq|N6*L?l@ppZQUCGUtfD@Q;~LS8K==l>=_x!^eHM+o<5PIU419VX%c(&pO+C< zv36hXlE|-JD?AM%wbay~Wy^OopBw%=o<*tu0H* zD%E4I?A1xDtB-eZr+TSz2{8CT@0p)>B<~BIK3py3*7$()bqogcnwE#{*s)-?O82dC zf79yuj7vjfk$ld}4+;N^z4s1>^XuM6Csm4)LIgn)MD!qtE)ivv=)EOGXGE`qL_#7# zqDJp!w9!RRA_#)0qmB@x4TfP1!*KSz@B4Yr_jk_Uzw`U&xRNWm@I3Q8d#}CsTK8J_ zy~OVGYsg8!R(IDGFat>{8x0~Jaq!!;;-HjNgoSvy__kn=-(ChEQnj9ARaa-dK%o&V5j~&;MT1e z%(|yy4BMNZxqg>zy*y&mTi{is*lBV0H?NoR3__9*@{B$`I%7>{mC<v3W71o!?Dv z^`iX8uEXMR!n)r+X+|Wz9=3saNDjEG1ta>&t8e?@R=+1W2b3urXfjC*Rje(nc#;lN z*}XS6-4*?7d4(Juj)?}9K@)ycNqN;vf=;VzZzij89}!3^xYWHtj(SB!MZ)liWXf*u zqjpuy8mxSEur{pG*I#|*n6|k;t^TpR#^*aw5|)_I!|Yn_%_HKlZ(SQ6n^(>{<)qz~ z5so=OK?Rj$vlB8dI3H)vcXM+>)s~Ub-WxV+N$#bUX)f3Zhq@8cZ!_-C*>GXHm!*n z^U(%H7Q!+qaiuBY%8|u=*f6Ny^yMd;RQlwKlPia-f`-PC`h{bBm23ajb_aG63YhMn zat__7mU|3@6is%%0^R{#X*uz4p2=jRYdvBbg-c|vUn#yC;wp0bV|zOX|GFHlaKB?~ zP@gh^9iKnC&Tm!ykR@a`J8TN|L)w~zZodhDh zgO8Yxpz_ez<8_5zIciXyIB7S6SGjd79RGSF><1eiu#EEadUj^(U3pB~UR!uEO(tUP zP@%-pN&x+poUE*;tTG;tX5??=%=#WW8gF3MbKQdXOl;Mz4I@00cKYb#Q|l^&`p!;F z6b_GnYNy6%WrUlMZZI?3@q}#tdG>0kR=V@Lrrhol6JSIB8gJT zi6KV><36kAA^leS*<0<30`ESJi2o%;0XxAsWBy_b&xqbWwFxqZB4@U^{#IVPeEAm7 zO51y_HqyiQMu&FA?3eshY(g^1PesI-kgjqG&vtYVFU_wV1GK|t{l9BjfgWs?I(@QM zMsGZX9A-M@cY0sh#^&j?QcT=Cr$I0xtztcw@x)K&pgbHJBc)jr!6CoE6OA=syGW*^+N zvLv2nrvuBe`t*`)B*UOD$Jr7u9y zhNkPismdp`WmxDpKGrFlt!lsW5jWn_miJR(F`xA!*@n5MeAIecX zlkI$pX?C89jzGFZw|=Yd?Byu3ew@F6|R*Dvm%l`)Ant@A*%ftKV#S`Z8#BJD9P>Ft>OE-Gt098FQXov--R4 zrqS(KZ9I1}_O?n2aeY)%C_yOqPki(x@4&(+%0b+Za8=M zcvH+$j%mDilj+KV1%Il%2|$UR4t$^q%Io)*|%*$vlKUhB4(xc1?IU=SbRVr}Q&8)`fFD@~-FXOO9#Ma{zr)_VFl8p-O z@KRU+pV=7y`J*&<=(3#Z^45zLyck>pR+abj=~l5-Aum8CV)Jyl-aAje)=B)d#Wx1i zZx2BBR0q43z?~;6$~!-hMq(|=WEg7=?(5~=A`;~VzwNLm$Xl^Nb?08~dyn?2jO?`*Gz5x{9o2P`sA|hCv6coP|T8?` zJdSPejcF>v>-sB(L|)^0UVHpsN5EZ} z?%|oCugl2r>#w$TpJ&zmnay=X&VEVG@|iZb5Xf~pKE!@}X6Xp+rlgT3bU%G-<1=dU zTK(nUE+Q>0UBb~+*8TC@ApPqPy4ZxCM3h93n@QQ#nR#1hex&{3{hljnF@pKv&OPg4 zcS}%imCbCMbAe#t$M5X!l8Ur^dk-)r5}lJ{ui2H2Uv2AHmhHR9$FK>HgX?r#)RcO$ zsloMEM~Ta=`_FQko8g>L4*6qae-C+u+)Us5lNP(VJy?-i( z@N+=E?(|S@ZzgI+|5{SKL$zU4aX9$U9)=9tn9bcSP5rjKJhEVFfG(rxx#CFGZVbK*Yff^ z&S^JEAfV_&p0HN{K0^V$Kf6!G|ME&`%s# zvK!S;;7TB(BIv^Y<(bsQD}_LEoG|zT@7UX1Q`Esbs$J2UV{Fmc*ehz^u@O91p5=@*;_+5a_p07-xVM+v-cuUZqjXNk zZ?OiZ70(lj4U4lZWzo?RtZ*;;ev97&dP}CSX2wA0gV3wau?1u=l-s!IBoy-C+FpJcXLuPc`sH|c=r3B{tx!s*PyI#v7IS*#d1JP9dH4sTw)dQ}T-VIAwQKhso$ z0;kM84I8_>zq003i?hEKaL}9vYhGI3)zJX^PVr!t<}1>bYs_~3>0+LXdGIVw)pzhq z9o;;N3&*0*-&SXY!E5AAwq(9Mr5xe*=NFD1ta@U8ZrBxHB`|SOjy3nXr8Spe)o&dg zl4E2?dwb$qZ36YlN~lw+?*@7})gpl6cm8V&Em2Ip2KJ~tnMekmIN;Ft7$8p*IU4hh z1gAYQzw!HVwU`(SipNRICI=$1rmI7f|zEtue^mzsUY9#foHQ?0zb?oxL1B|JIlUz*9ErUi)xPBgaY8c zVvo<{QyTn|g&iX!^iytv;v+VelSP9{@Kz8_KzqtRg1rAkImTIBHP%#u_(4o5WW?Uf z-F@tEMV|TOA9o(}FQL&HGPxkaG$v~93@%)r+ln#Of~Cp^(AAz^;*Q=|>yxq4`fdMC z%OiwwWqloE(pD#fMU4RuV~6~Dh2o_Ikzp%wLLt9b-wbndk^k@bRJHsyxW^R^(`GD4 zLp!4Al%h*nr7DtIS%rV-)_LNN=$&UJZIr!TP>-A$2P)!SCywmXus0;#dt$F0HVIL& zy!!Yq0SJfml2oG85N(alS;si_4!S zn)!39dJ}jO{s9?Hc{jFR?5T+8 zJC%hzsKu58x*8nL8;w0Q%z3pvcxo<&Jv9aye`paexRMQ8xfLdc*!Mp0Sj*%3nmK~I z-FS(Q{12%E@|NyIKwjkXwru7DIJ#g8O<0n9W6E^dGROE?FP4g~bRaqX5_pFZ@EIBy zAf{H=+!}J6?db0nTz_)jyw%2cp6#Z?@boaAH07?x0JEj91#cTpj+OR((u-A8(PCPa zyKr77rZw|NW6O_?u)XMrM9oz*%b4^BinoN>XaODd_vd$_0ywk!%@O%;ABB|7PWD`F zJuo+^{vp?)(A!DiN6PQxsQ^xX$ALx2h}YhqJiOz7;i718efG0wru$QiKfI((TuHOh z2jjBdi$SFvGu5{6#Pzx5ZqWh^lx%TYKH#~(!%Zo0EGF<<{=x%3!A~47V)hLu#(jVb zI_Z0Fmx|>iZhCZ{0b9EJkCY+gu>NwEO7Y!zc_fDL9pmhEG;a&=c5F($Rx!+XVl`?N zxUDH|`x(3JGR0jn94gLM4U1)&1^tyHC6@Wtv6F2tRw5}O)MV7`(5knY_wmE$m!$4( zmfz9X@6bgWMtxbT6*T-3iXrSwykCHm!$B)Yul>lRvv6^>QxWj=qF{<`0e>)&v&^DR zN%nwdIKdeWm|iFvD|yX~1_d$Zn!&U)y#m-~ZM?}JkDV_(lOcPbH)cYzK+uHnZKV5M zE;|;M4V|FcVx`MhyJmzWq7ymOWqoy6(O74MXFE#kRCsV%Ew)}Yc6Ahi=%jJtb~?@r z2g}WAn_ZdaGbNNg^2ASJd5d%&5J7>Ko~@T^6kZD%(F;fs44et zTBTZ11DEb0Evi$%4#Ec|IUhf3nGN||4%{yg7kMv?LtCbtKL(pr@aw>}o}`|^D;)?X zl?}ervu6Mxk=aqbnCtE%|GEupe}`Llh~|>~ND0_PxkVt|T?*$O9>l8~IE9dBX;?y~<1QrVxxWw`CJGe)&GBS~>k!jI$sC;MoD|lmH z+>C@w#ZWEkCj4+kwHDr#wA*$0=YLnhvFz~&XxbK1uQG_~Kj-SopSi!qV5_p7=*X2` zSYH0fJ0%O2P~OihvsB*zXrzWL+a>F_QGn&FFxT!-l+{Uo%0} zsh(aX6tCYhDK_s?_^3eyf7kFaqoBoLtD=fQ?n9P#yv#;}*YRAt@u_rS3Xr(5!#>(L zOhtJy3gD;-q`OxIK(vl)9e$$w{KBS_f#s0Z0Udx!q9?sbU$yHEs+Ak(y=SVl;;QMc zc*sz19Jjt=mmLzin3%l!7-3l(dnan2w{Izvj)rEm(2D%jKLo!cDdzW=wZvQAuRboE z_JhndDgqYqu`5Q*rvEm8QI#{wnp7hXk8(m#W_9azzii#??F%m}9u$+C;K2KLAv6h4 zyq+sto9Z5uzdnA`VV$h|jZ`<{14p;!q>t$6J68)TDk|@QZ5j|^>jEERbq{9MTa|FJ z;LeKgW%@%^Mj>-x(j?bvLvo!7I4?i-=8PT`Y0)W~LH7kY5f;67CijW=sTEuI48dP< zJR#I_m0-yY)<1Kw=m9OOUP2Fp#=$>1CCH%!1#P_zfXo!qUsR&k??H=(MFFUXS4Dli zx!2W*w#^O-lF`9|$a?T)Mu=GVupS%p&% zOuCJhX79R~lpFRjo4g7gJT=_M+)|$Jd8adZRi?NP#Np0N9XC2UI!tiB*dkXfW3g@r zMhbcL zZz8SLdgN4pZaoEqJg0cz_%&iiU|pFAoYoXc_nNe{UtN!PLg739{blYoSEggu_u@*~ z3n>v=u-|R|POnp$G^=H|$ny7sI*aNqT>Dul%pq^kGC4|v)d!IZi_x#j=g_z1Hcg3> zHP+F#v&!<8e`C|cSiNyW=#jCBZALC7w)ZsH z7CRm!`#`(YKfdRmwhlqCvE43{{=^KbQMsW2aemHBTxb)xe_tQf9JCPFYbD5(zMxb3 z76ea~6LmtCjm6TwIE41M!=BVGS*59%w{Jg!0|gJTd`XWl2-Ue)iCMvqC_C|LE%rjQ zr&0MtRk6dTS6gR4U!Cut32Y4LGy8VyR!^ox%!P&CT(6Z<`IxH!!I@31r|hy-_Pz#N zId85ur-QTTH5*+FD5e6tE&!tS>=^AC1awtRMC{dGv}I5M+QlfQ1bu zyP3{MbqE}TURy&30N_Pk%$L#SOjXdGYF=4Ff_Ny;Yem1{xx3^8PJU5kkQokwO^?)K z<tux-_2Pne`5;o{~zT49fO>wg{A$^eUIsZIdswm(!% zSB!{p+9Ap_^|UhWBS7BFy0`qKLdw4XJoWly3kok+A1D~bF2PF^ND-MiYy#9Y{kXzk zLqoPDfN11mY*_U4MBiuOmkP4FnhFY8iGpTjnyE+H@2+%=fC4dtGF_gsrZb1m)2*yo zZp!4Mb!nf}#L?igg3joDDVl&YAQ^f&tIJaW_;6tnAAXAoxdD*9gU@T@P^m9X?whvI zgS*Czg61Oevb6rz)xjqfOq*7GdpCVIk6I2K$63H5{h8%@u=gpTb=IozNoH;1VMHa9hYVtG$J7g$ z)jKckdVg3uqx-evebQ+DR761pGUv6*>nV`A4PC0K8tnY17J!u!4}pYNkz&_IxQM;I zeVGNh#N48~qv`17sff|-uoDINmJpaOU?TA2ge)4Z&-beU>c^KuvKya${|)GxEQGT^ zHos(N-S;oGZK^o`U!3-poefdrkd29?HJ{EHmV8q&E9gtM6MIM&8L!uME)#Zlv6K0^ z0U_hEiDDlDg6}DSQZDg$0x2sM9Y{>4$8tb!t^@V-J7a&E=_MOQ>TVujdW~vb_L&7t zo4>DmNJtpf!bZ(4nrf2b;`AbS7?0)J%BEa+rw&jjMv+F+Gy$g$n?O4uF>pPnfq$|K z10?hZG--_WYn%S#*9cIaZQOQT$?>qnZG12A)heiDQt0pR4;KM$c!F$`*{d--*|(#jfT1_g^6gHu#1)X9|_Y_&SFRm^)6O ztuOK6VDojg9nSSoW}?76l)pq+?-`GV}#gCY(y zTW;&lJvw7kR_%*>`M7ohtUEEc4Md4i3(#-8wP<@Pbjg^s!{h06WhzP=qRN9sxxJH6 zXxpvV6knx*+!O$5XVHWfd{1v>MjUWUmbV8~I+)Xh^eaN~i@mv?OM=%=*684u4sfR} zj(d|UVf(XB^vQTkj}s=+ztIDeGCkwjJ+a@meecxAl%OD3N>8!An0+Z&EyCx1w=asj zq08B3Y%)DP$>C-1V4@($zzZ1I9*t)pdsLtAZ$%gRbKkju?9gau3mD`#fNPfAqt@}sG-y9kY_c{GiD(XPnG9&NH)7z? z{i(aSK%e}EzyddZ9BgCZF@w;id*JzLM8(g!5bB52)=Xr`z{dbn-<~jhgDb{ImYR=>UTLX}(jwO1t?;lX{n$`X4iJP(j z6|zypU?}jO0V&4K-_6Yp39>sT=svCh@Qn>N( zOId&X<65k^Xgm$+OFcTEx)xw12KAN~+pAjP0ma+W>zeC;KdO>J-KK9SsWg{_{+kpKQ z`)>K1EfeJlXK9xi>{p3kN}Lu;_fzC-qDL{ZfnEt=it|U^L}FVRVmNp(GNH}HVgn2( zJdmhOaF93$>PSWlEEx5e`$zd0KMVQ+9uuOVZ&oCJ6ltY08%4UUL;lc5BAD$EBy6Z{ z`OcMAVZyQSaCC&-w_cJ&X{6v$KMoYKuKx)(i8y1I=EIgM4K%w_inIO^}@k)+KN&%D| zkHm5T=VaO45bHWo)#ughYTTv|3Z>LUtg2Oq=E6DyEe2;AJdMD)Jvz_DS1RZWim(c! zT}Kf%O~yzSn+;q)uEuCqP8R6JIY#X{)r`74V&F?0CeB8EG~@5O`Ee*VpwD7^w9^nb ztSWNYx99OgTg-Ln9!829vZwVHex2##s~o{=IWBLxqR5Q|Gk#=xdOGHYde858zacZVOC3zDPzBJBeo`*ygnd%Fw)?RG!EO2_3G<2J0*XlAseu(r>(7B zRY&zzT+phaV2-1881N|@L|*c1gAf6 zHwB{>LpYmHpWL|LE*D1QeS?)%%um}&Z$?=D$^8VfhMT3rG`4Y3M1z@Fe7g?~gTd_Xyd+b&d+~xGbGLkyQ zH=wW(h#|Uo@8@+_-`|NtTGRBa4^bS~Y#ZMwvU@D3F}XTYf7d4xK05=v_NdfgROrjh zlwEaz?Z4#Ei49~|iEQq?4jfZCrdAEP zz`Z}sVqa$#AXEm9@I)=+Mlx_e90unxtJAecrF!HWk{;%)(uCjO0+2?!DIDwB$2tWE znxeWVPXvrV2BL*D5~3ky^%GvVgRx_-pjvwz-&$!8Pfb5JTB_5hU*}m!xTxg}DPwGT!Fzrn&%`X4R&gB{7 zc5`E=_>ldn0fTT^JQa`-8^pH)kD@?lRoZt$c>34&RztRJZxkJMO-PA%gkx(<-loU^DyH^L$XVIv6U+3eN7vqPr0F z5~!Waj2)e-NH?u5WyonpK!(l{l#>c|h4i~zw z5>_Pi0sTTtoBZ{wG#Iob;O<4%!$1iMh?%9)!^qi(tBiV{P}F~j$ z!v$kYY?|jOX5M%SMo`uR8-Vsib<+y)nM`Y*=L0edKZsgDegC~h{K*VOpwDB_$2IH5s*>Ku2GjhRrGXX3?`S z6rh}o+IFXzfEaO<&gKsDkzmxmdu+iIHfMkN=P#_poCm-C?p(%&4DpRx8iKaMV)h%> z?UdLK=xRtlgll|A6ktwe3mqpfSrxl+Iro4iOO12ZN2dIVe-GYK8>l$YUwQ#VWXN{R zO!sXTgTVXz-mITMp$44nUT4TQg4uDRkT=RLLYPE2_Ga)X5xLcRrLrv1N)9=&z+FI#Dc49)V&B7Tk&<8}n@QHxyY^9l`$MYC(Hc7~_x$qJn zMfMD=+*$1_$b@63jE2Pf~A?+aQTxVVHZoBLRG_NxXm>SQ9 zNIU`x(^c`cjx!oKZ=CaZLCgRRxzaW1SrcgFc+cX`K+0AeufE@|+Y_*bP<)wtR2a85 z*EA;T5%!me{rATqz zQL6wKkvF>MKyEdlTazHI^n}a8`IPOd60Ah>0qp>B}lmeQDtEr|ymk{0o zWkQYJ`q|O4?P5UpDx?&Jg|yLeLM4KNYT;egp58_%7m~x*<9Sf%2SWaodmCmgA0u-Y zbElhpXPw4EiGe=DfjA%CMz2rjHG!T&7Eo?lUNsOh07k95Ak0YEw?4bf*P<^6&6I7N z2|2Qr+MpP6_tg;K=1cOEi_HoP(CaYjP9e}~1vDh77k=^U?jwPd5QWIV@gHFk4|xrS6yIEs3(VOh%KI#% z$p^7{C8%a<#&f_ST?N~R_`1y~Z-S;L8aGKuLZJJv+kzuS|I3SS`dA|hQ5zQ)7DhoG z5SHsGr%8TR=FoUi`(4g|v8u|=2RDhjv4FVd=ohjz#4wx|4A$T3-x#_3wu435b4muq z=ECUKQqvanCa9XW(gwgy>T1GW9dqy1>%(oh7;&X{oM0m}1dLYkc;IatRND#Ni}wc* z&-=wr{T-eTkjHRqofy)t1FfI{?{QY!XJ;Hb{3h@h{n*&AOz93ve^Q|0JW_&cDrf{X zPF2mpAVN+=8ao@K%bdB}qh8$w&FF*=rzn12YwKdjDad#~U?FMJZMjVE@b|%g?d*DE zo7IXn0LY-$?K%7ULdBJ=82`RZV=!^T3Et~N==~&vatFt%l>F zHq=Z7YwPQyjozYqGoMXPTGQVCvf``$P`JN^m6dfAAVsI`E7T%+;yTAULFp`uM6&2- zpqisD+t_9UkUkzTF@UOl{Njb$JO63*p^EHN8V`w)481EMhUI{Ha*y3jn1Dm6hskY# zUswM`XL#Rj>?Z&^AB=nPLwH^~!ysR`0ei>gHt*(Om9359el%+c^AUT#PJ3g(4V!4v z$l~&HR%Zr*#x}L=7sKuWd#CKYJd@DYoVEwDBc9*J4nVF#8QqfAzbt{Va12 zP(rJ9*t(!N@%tND*V{B&nWir!_jue+qq@Pf&^;v-g>5YWtEgkDE-sD$qOq}n4t<~Y zE)<>w0`3B;lg#2pHnjpODpU#Y-9PNjG%~o&Lh>>|^jBmT~OwfhPde#2mt) zhzMm6LF|4D^v5{9#{=I;)drACK^1p*GqEz{__E#vi$JPrnHdP+fEvMf?cJpeI4F0B z@nm7(jjs$y|0Lwbrf2et|3s~I8cU{Zprz}i*BCft&Ca^!)zPv}rc^QI6|{t~-cNv! zV`cfoT!C2s^!?f-utoJQlRulbJPU@{9G}?Ux~2!Bd@}*S)Jqe%eeX|nLW%NraLylK55vtKh-9>7qz3Vts1j&4o!q4DxhSX~wQ2y~T-v~YpKD@bBHzghGX5lp z!|~89Rb{qWyYY2G%K59F`VE|L>43C5hcbO?uE~NvxM^XeRrcX~C*V4j8Skf#ip#K! zR+CN#s(}u+B`Nn|Pd|TOsP|44kPx60Dteb}Ju6j?TD{|(3PjJeTZ8;De?8lE7XfO7 zpPZb|wU}LJbq-vF!zE0}r#{L6y)+LG4_%u9QsQo1vB_>&LiB!XygKvtV&Np%OHgDJXcRAxIVxa?iH#J1q;+i{e?lQppHdUQ~GJe-e9%;|IU_Q4OpjSZh`0Y~r3iM5)i>tB%C ztuCbm3ybRH4Hc!3%?FiLwqoo;&*l~*3_};@Q&=FIGEYy-y7bJ*CKVgN#cv1U&}6B= zjaN_7gT;mHd#`B@M_Ycv5eCV2STk93=oVlJbur^&p!g1sLlp* zkGbxy42wFZYvz5PFax?EFeZZ6*1YZh3|@jC381X1`ee~gn*LOY%o#rdQ38h@->t5N zv(BF5hSUvysT5Z1s#|ZOoIpB`V3+5?6Hh}v;&rjp5^@tW(;BvNa^33rH^?%2vG~P4 zR(rUw@Al!Guu2QJzEsnm8y{$oVRms*Gm4P?(ze&AM6*edw6pC-e?9f0>7AR@1#5Xtk`pOP$&QSZc_Ah{gW(n{~A6Dko=w*N{<;HP-aF;6uHS_J_?La zO_iAF{Y^8L_LWm_YqN34+ z7G%gU=)~aIk%HYM-L@ffsdqObP^>hD0`UsodMc=J)3$aN87% z0wn{1rvrceUHR6YMZbRaBlDNVvk)0NRd>47z%n1r_5;v=&eYPV#MGq0BRO-iEH?e0Pjt1;bwVZ0a5om0inYCH1?Ox3z3|7r7lb74^;fkKVw88N9W z-=6XPE9UbHbsqQp3fK`=$|TI3evR9Qu+#%@3AjBY87-arw^)9N2w7Mw9oK8d;unH< z;h0~JXjCTcfhrKtT-vNDc-}qVjlB^{{|OWs0!bE1(k_ucneBm;me!Qa&+`z~=8*?P z&9~-~o}UXINd>IEevxJ9bsVVM$lCX-ey6!G#UcDB-FVUP?DzB}v zjTmiMYF@~tZj^GgE(5@`YeGQ0ncYjBaP)&)lIZTS)NlKR?E(8=w5Dly@^Da@-|)b! zTAzKhO&&HTnTL9h``>GpVsxJFK8D+N8K%`fVU8xh1__RHz_-<8{1rjVE*2G9tbL?Z zazeye%MslIh<=O!yGGc)zvDEK^A2df=j~S6^;z>?JnvsxY{1KfkcE}5xdERXMCL~1fZDj`sMmNG{n<-d@&Lqu z0XiqTfLC=UlE0zHpM#560e5~>$S#O&wdoAG~Yh|TV)B@XHSdq9h!jC z5Y@Mo!8`|2KD7>L)wnw>?AGbRo*5vO$ABY&Q>?1KULu|{_YaPDnlM5D%M0fh(gA>L zZOHL6y$0vH((~kV-H9A3K7{3?=tdt$WQV$;<66vBn%?8_I?;Xze0jK#2VEH78WRUN z9{-L+KNZI z`tKp;nB&gR5osM=u65DJ$H!?hFFSh;*+%P#L!Sz-Ql_zr@g{#}{0=nNL`N@z%CKJ( z7q5&Is>GBIYyq_hZ}`DFV*wuY769-PWI)K7?i|I~on`M9lQf25 zedo6h*rlnzxj{{<{OWi{58n!@V1cs=om95|($J7nz_sr`q|ttGCkos9?c1%1sT66q zNVE5kcMOFSsg*xiakh; zEu9&D@#2LxKrabxmNPUM@XiJbHJ=E=VLbr(0I;mI(m~P?(cRKNKN8_GzmRO`)Y;e9 zH@Yd;Knwr}hX$RB)i!>tl^jtzX4UADfG7n~b?hkB*K7aYQ_lV7&;+^*-@LL1OC32H zeu%UMc;vs|dfkxs;M60+YKxO!8~i}fb*gnr+pzG-8r{`}s81p*%avDwoW*MlTvhdL zIGj`qGQijD+qk+>C@lg@-O%Rc+2o-%=ujN@Vqq8S4b|uqhf@igckM{h0U$8aOqQB@ zVIu@`7m5+^{I*@z4->DU@9i=Qsvd?#q$+DZZV1{3qDg3HUb+jkV?}OVOg$+Q+xlD9 zN;_L$>1Zg&8?yvVtr@O`*$UzoaGAbboJyI;29#%bG%TUgcY0&JYyCD-> zm9F1|?s9a%G+Hn9%aXwT^;QqnlaNKIW4WQZ_AU)9J^#}*wK-5V5CkXB&jg!WTV90@ zC0Ru$CZW6)DajNu&#@rAnQ)rEE|yKLc7TB$$H@l%Qw#9NUxA*^@jDuA=~m-n{a^y* zCW4Dyg#vvf^d{COF7AM8uGKO4Xicf{;B8*uBXV(>?9k(5^&520lz93o@yzK=?>zUc zCkB)Rlcg^B)5rdKPXdB9GzUA*Kn=+mETT!$0WC=0YXHG7Lm1H<^zX|9qV&$D$mWDk z=8z)`fJu*H$uJTrRX7~)IJ~s{^&1CBa4XHwo*Mxjb<5KIME&yU>!oY-*zb7d60W_8VhkU}W7LPm?&c@0q# z3v%ZMD;1^nm`8WrYKQO)&I~u!7r!P7Oo3JdI^G}yNU6u=^!K+yfwqHQt2-vZw|}&~ z^aM({`~-5P5SDI4z!=S%q1%9jGq3hdD`wD<>Ek7e5z)f~h`-S-R2sPxDCBG@Xm#8x zYxIN?g;LP0bR$q@ifu=S)p~Tu05;99z}@8>kX{lV!AktO>Mveg_~^|AzygPq%~o{0 z*x*rmGGkQcgn156{6YceDzsG8Vga6_Sz_e&vCtl#z@w)N91y7&f8GXLeEC)J1T1QItj1-N6Xf9E-Ef9wy3hnZgWum2RRnCoJ~f)HM+X2$h4S+9#(+`vRPke&DY3)&~@gb zTvMgE*9-Ytr|szMe#Z8;(H@5Fg@uLqG~$+U|9_^Xxm-O=*iZ9m*S1$Le>sL0*S0>L;tpr!?8~b>CdG5E4rga z(;uqdEWILbTc_|c++ygYK())=zr7beLZG2}r`4{UdwG(E>q8Bh;Q}=D{`>pSKh5!1 zrOom8)|GA@wC@>5a%!!p*3BZZJ>IP8`6|5Rz2)g8|BR2%cuxE<0z%Q9jzX1Wj_SaxL~qMUzA)&o^Gb*V0CV^P3NY&)IkJyVaE z7XRr}M14{o=$^6sOVpX2cR&B#z$zo-8(JYlOta^}SdmU!F+s)v$lK2>vz&pXMCWl9 z-q8xSv}p1w_nvK}wMb5#(itvj)h+ZiXMQ#+P&J}B5>D)+KmpPsbo_QXDLyMfd$wvVhO9?GLn)?+FG|6(?ie`53%b$ zO_qK~Jn6xcJl;}TD8Y2X*iQGY?Q)f=vid_q?u-}a>BM6&q$|xF8#8%%dCOyF$|%3- zSlpwC2qcm+J+WzNrQOI6!<%!8*qtNFG%8LKakDBwEz%>Z3&`B9N~OzdKY1eE=Z76w z9n3XL-(~mjfu`IG*3iIBov8fk27@Tv=E#cRTJ^l8ANNB3Y;;<^+>p$!@3c?uS=u%B zs^IO*+b#y(z3X|mo`pxUyHDS>Kq*;ZJYC+r^$0o^AkXxA>$amKqo-6*n@jx2#IDe+04(k6^DBF&HO><5q%{ zmZ4#R9emU7$DmdD4ew;T=KZCQ1AcjwyN1AtLXGqI_s*qTM$wHLo^pX!M$s3Jeb0n? z>*}#sHRr5Y?h4Cp?S)+?cK7s*^hsDCnM5rOh#MB|8-Kl|6vxDN+^h!JW(btK z^z_;|Qx}77ai!<~b}hd=z)?4m@x6j$p9|Te{elL|S7$HHZM=IN@N`#o@Jr0qyL^0l zah7Z)RP0}>JMSI#=p;Gz7vkQq&TTc=vt3*@2+8|8t3}vv0VgkGi==B+aLBe>Ji{ED zQE;%9?Cvt0Sz^~6DDGbN$?Oc84Q0sLN=-d3I031Ax_v-C@{_oI%@BF?Fm_e!*VPpH zn66v`_F^sk;7LM40uOg#I#9#QT38sL;gt1St`Mh(jcm*W+DcG+uXw}*#lyir+w15S z6=#&IcK6(sTE{8nhIFZ7o37nwA&*Zvn8>ijh{L8!*+Ztw^A=AnkBV=^?+5Yn zPv?OZXyEbXF#^G}I$8T3kCzhbW}}DIn=NI8`2RM~kT}{PnMOFL-P;=dn0=5WSe_P9 z^Tn?X#F1q{rIxo|PX$OD=RVYnicd&DRJ^G6VPNQ9_u;E{H3O=w29lB!RTC}*Id)Kc zx5|x1wNlZtjqJKg*Ld20nr49I=j9byG`SE~W=br#Kd28C^qJMTPSH|y+MrM(#_lJtl1_ zyGIX5gn2cA;Tm%rF9O<X3^I28T;(?`mjlaw=AR5=e9&xR%n40_m{1f5fVTWh4s z>0Px-xl#k`x5uHRn(Z}D{L?`AraPA@?AImGW-v=>yv9*o^+Cwso2Aa6jsJ=h75{KE z=L51I*Hia~=>1zPL55trT0Y}y6?xLR{HX6uMi7e2}WO6^4@trA# z&3rbx{GVTV${J4L2lNd!65d>{Pxsw00jtZ(bo`(Go{;5-Ktn7AL3nnOM7!?k)x|2Y zvR3^BiE7Z*(+jh-x3v{7cKy%2LKC|ei3j8@*kk@uOEOU32j`4C3{dCZCRsE^)_59y zN~0}o^pRCtSy|bFUHQ*dUmv|oWEl=)gFx<^u}ak#MbNJQouU5E4c#!q42$$NqoX3wjGu2Gl>R$&1FWGq6gCNkjYBPg z|0m53IPgF&{j{s@hFuq-PF?d+^l8!z|1?f#t}}Zg3kGzVsdMp}?UwrrWzRT$a{g`` zjQPf(_%c!4nN}lykaM{_mR;OZl=;Z{i6tOvrUf#S*u5Z^)WL_cA&n5-Q1^9B+?F3^ zi=?kJhp}UWjC{u-)9_GBC_{n-DeXzq4If3?2w8OB#Yb?eP}0@`Szh)xr-8m-Ulw{! zb()!St<8l;kRk)U#%MQQ%H+Pif<$o}C8zFVPJF#BhqNjesUOlUrCWi^wM{dyDXX6d zRo*-xdo;-wS4=E;k>VG;ezeqc5m+HP?MvIe>)S=&M zw{+sB+@Ty+F#w4w^zeNn!)~zmc(Yl7T!{&oS?kPznM`>BkQpCFf)yKipR9?!EG9yZxUrB=J>JPGP>*?XM zCb1>7z+~{Gy>I6smGgvseV99)%BnQPIM?Z|e@s{7NAzFBVbgbWF!5$vU!vW5D8uD& z8j0#UjE8)@ISq?x7ADbK*l_b{A|QUA|HeI_lqQ)(mnkca!J0j zHQJ5#Ur&28thjxh^N_9itQn2$1BvJGRc6x?fm&}ZW3`fI zCC+6%8H}?VymKZCn|P?Tu0^A%Mnvo%V9=H;BI?E>P=dIoT`ZurI^;t=K;Bp|(IhS{ z-Hu2QE)v{%;5U6!Q`V{HHso6Eu!%=@orHHim)7wO4NXMrzFFVWUb2y1yO8^Im`L^4 zB1hyBDkO7}XlQJl<-74HBD_;u+m`M;0;p`J?NxO{y?rl<%7J`pVbD3R_i+Q+BOz4> z!4^Af0&Jf}W1G646ploohHp1sDPAEyb{Jw--VN}cW5yFvHa7~ogL1#5F2)k&$S$(g z98;q$O5nO2BCuiDNB5ZPOX3v5pMx5u+s)7}Ulb>oMES6ZBaBt=q(oE<*^1dHxBN0W z9&acXfBM)Q4bA+f1=W3%dOQG^lNp>sGFZYQE%_G{WY4{k1;}J;=`7rNV76p|kvv>N zeaW&W%S_KI{T;6UUBSHhYzvg8P9iWjcKVk5@d_m_DQQ%97?;;Vr}Zl{)~;P!KZGEo z=hJH=U%Z$VW^P(6RG_s63m2q&`Sx+OK~&^Uvx)r0Qj2rR>x}o*FfUk=NcBA0AHiF2 z1A6IN$KAU>5(|)}_}pmt7WKvg-#Q-OA5|BjUTGO>rbUHGf;6jM@1b(v8gIwi60=gT zhXu}=b8_vu7d@dgub~On_ra-CA3cqOdlupK77dv4>N{6!y9(W?mWA@Ohff%1^^{T7 zP?70*wRDC8Sp-<^tGwtqkUJo={R+nH-;# zL`OG*?~iCKXV|Vs)P>&Vv0oh+y(59c}dd3ijlc!zN>PR}udo;G&ocj{|L1 zdOh{UN$l>t%)A#HM_5TkR>QQqjo)WC0zZZfz*1tGGs$j=_$--mW||VojM|(00u^u9y#`Cqz4J;T zXWSI94|-EA7wySC!j$I1orV^rSb-gi4qrB1qdRx-$!hGr?;%acJo2`!xwfPOoci0IGDKbGhuBd$#Vy2aDqQRi7q3J!Q|9 zPE*^sqY16a1jh0RI9CUOZBBa0cW?D@u~6PS!Rhbo*zNjL79WPN&d3AbCnA{;LnfrV ze{Q~Y;J2&S1DB(sZpvG4UIkN=fgvktQo;_7l}`I)`sI_QKXTSrX9V{hhM=d>0vMJn zeblRk&KjSKnUWi7PU;oTz7$B}pX%LMM%42lb|*(WW_8`)o}EJa!(Zsmnp(a~!Zl|W zr7CaJVAzbGQpI#+@S;1o6)pA}hL=HvpzYI$_l0#q2A7m)DEboLRJuv)_voboLrjOc zOEU@}&*O7%YPCAZged30<>6v?P_lKv2FD4vQKeDQ{xZ2P^~#{aI%R((hafOD3iPSQ zqDeBu$l~aVAR2?P-Z{4JQT2O~Mbi4(SgYGuNX5hNkq~`Y7#qLf*g8bGTH&XWDYDI& zw$w;FJG!1|QX8W>+zOAQcxG}Cd>e!^Eeow6y{bEScl8@@n6j=i-ysZ=G-NV-MKK`Y zw!wNICgGfh%~TUNTm3xrbx#YYSb7x1`_1ZE;+nGqdOYXNRQYbWF8a;)mne?+*)YwW zrKZO7!cme|IdXFw8K%ntMX@85dTjnLAhU77eJYL$+sr~|p1q6W?3ztudFa`lFmRLemW8nyc zQ#LjdRWh6vBvf37r?=1PjC+)jY1`RIHY&97F?rXISC>b=o@FQF=PnYMeHMOm?VJID zuUcDLy4a~BH_PMci?(`bAtda>Pw?_8>qBWVwfRK2m656GowC+v&ttDkxeOMReW_D9 z{FEft4mOPYa#}kQCKFB`KS&y_4t-`w(>-fyYTD*!l?}O+Mynrnw6u&MOd}5qkwjzE zd;`nP3oJ$?&9m$~ZM#k+CMe9`1ec$($wkPE3AWaXdAnOAuyJ%7+CQpu#t_yQZT-iq z>a&ArTAug8VsNmQpHtFC`z98hnG$9?Up8PvC|xi4(VkY6(nZCgTbVhzvgFP3{riRJ z^f$9VejVB-ap=$?hL~K$N+2qXF(qIoe`boaV<(ymoXi{g{)MKFsv4ET>NsC7FMNRsRyJi^cQgZ zxTamCvauC1b}@tVJ>P1y2;SNENOvYoGS`pVjevuAR3lg>WqGO~zoa`}Lf;%1jJ|MW z*<6;sR6_6^Tdp)h*ZvJxbG#-H$;=dG$f zm2GN3_pbE!n;lG6CNW2t_k!AZHjnZj9A{67am+50xNdp#sBhveKd4%+chekKKwz?6 zd3m}X?>qP7T34~riRJv(DcPXHR@fd_R#x6k&GfwwC@OU{(R*UH6G;gjaKJAz=wN#A zII-c&Gm0Qvnita#S7aO779RnX&+BC)!9oLS&Oor=RPMiV3|PgG4_Gi+FGX^ zLOBss!H+Q)HpKt_si|B-&$>oB@Tyi*ZY7P7FvS_d?_PAHWiuUR-{5FK`O5Cud;7Y4 zQ1vU@ggg3$UpyjWz`0d0?J;p^aCCXehKK1k>@m0yu!_F*gH#k77iTn9Br7|Z22|Ih$JEBD<$7r^ zA%T=oIGRfJ>Ik$1rTm5Gm9y(Kk@?;(G(oPbu$FH@Ba5bPqsCr-R&@G`s%)370Nla{sTYes4bmu*ug@fkrJpk925KRlI zq~s(F(eJAH@SqwReVg|f^2}%}Isd`R{bg=N`z12iV}PgtPQ;P?zSa4fKtq^Sph;N; zVDnXrv$eSq?Y>r770}=`sIWtl1L-pIb=-WV5!5u%lyTX2ZM5#J>UUB5?lk%Jm0ZF! zo%hBJSoY~IY|aLmvtP_~FwjsRNi%YPXOQRPIgH+gkg8TWp6K=!zvUNvCn^2hW$x!T z5xE0F%#K&IieWT$bb2!jhn%t!);dr$Ji|NAak@Q#($ZvKOnckj(8R@a|Amz0TXUBZ zEwFJm>=S;h5F=^Vvi30~LZdq}BEqphXM&h$UHw$1kXNbLdnDq7u@(`pR?(A@8XK+# z{Ol>&nAPxX^)r?**^09QN55~M5~y3?#bO&98>)G(5T+Sq_D0J)QfO1y*(OjO}C zz~6eRB3cEj;xX5?ws|PH<3DniY4Vj9Ns|Ub<&>L)WSTI07vhel?ncV4pHj zTyt~!iS=RrD0~sTngC51l#d*w%H-N^$&dEC)1V0b62`XBnX_tiwOHo2T zD@Q{P-ys$eyf3(a)i*Xa9xKw~F$r3lnRyn#kElDrA`40Lr z2m`1Lc4X9PIu+g3ZGt_DY54qEntDP~GNdo7Qn#XDMjt`i+a=yzi0n3elg>Qw3y;Z2 z-x!;USr!37_KC1O0+7Fu+^qg>5y`(by5~yNQlQcq=72j{elZDC%q&N92trk?lejz8fxlxqV33>L zw{E33O@QlsOrsI53(VxYEqb$E7Q71&si#~z>igO6{xYSbf{;M6KlkX72RX_X+t!iP zh{A-@M-05vr^pa9=n(G(rk?y#ftBALi_5tZ)6?#x&~_|urHD_&mmxBh<}@>60HxQ) z(%-tLonq8X$zA_kNpc6`+HEjXkrfU;LqR=(Ph8WvxoHI9!#k!!2#Fd&>4omF){hEv z+bx-S&%drUKUrc~P=sImF&exfsBBm!sgk~aGz}(J9_orO53QE9{cyPgmCSnbgaLaI zQA`a)FR(OE)XzBmpaEz2RK@w_uUEWf?7UjQeItmA88wowOu}wx*tF7&PMiqJZf%}V z^k73@7UT(fD(sqpvZpQWPfJPBwI2Ny=d++j6Nch7EA@qgsQOm4CewoPHL?=SmU6T^56%;K{aZR78lT{?}O=(`pa)d=~6(vi7a zwyXyP9b<-^#`o=CJ&35!{Sr74#B(9uPvV^)XV3J158+A(jM}XpI z?p)N4pNHsZV>RyV1}GC3ii+Y;MyokguQO&0`VLHoj7BJr(p0-f(91@$;y#Yyl)3I) zpU(`0UrPUy&S_!QKxY|+5)T;%afDRv zkhXYJHl2O_Be#I#nTy@|IgLgR3`siy=>|v&Hm?>sfl(<~kd7R*vAsCNi|kUpYfS^& zIFavsA0&Ug{GU#Az0&ZpB+$Ew5-yD9^;-oU9r^;kUp)Hz&rWSuyo`g`4|k`YXmdlS zttvoc>bdYf>DJBYZpVB-4IA~5B3yh&XQ#lh6;hS<`}&(8u8wvM4*Si=t-D(;nu2WF zJn{PV@Yz9cYL?KUabOboR@dN_)0jpYt?c7>KyRU!`x`IR6t#D6nX^UE?6lDfl>ov*BfkJa$v*-yRpS0*ZWhashv9k?q14(z2Uf!rmO6 z3P0cG2_3b}_`TxNAF}tH=B2-`(Q1aoRcW~<&Rv4NmYzvg_(fCj^3L5I^!2=#FyWNg z_3y1v5=JanTY@NbZ>SjgqGok zrgffkefx7-i`~ghE!*KBW*OzZ_JBy}Ju7(kKO!zJUQBrr<2dY6)6ng``B26z_eN$` z`HeianC0xSq6yLZ35m{5olsdUwPnF(b>EIIF;dWU>75A0-MHHW59oIfL8lejRPRBk zY3W$PR;h{gmB+TC(RyzP8$Oa(-yNS*12o;Oby7zG%c(tf6a#*e8T}edovn;lqFOH= zlwLHFoP%8lHm=t+P?>f^z=*vy+S)u(Xj`6sB_<#!jp|(%eaZ&HfDGBzZ39AUx;=^Y z20%}IA^PLPMTLpgU0dL_c4zWIC31m{P6i+#79|bB=r2Su1g`2Do&j` zl}D+&xxO)C%Snl^n0ec7-P-%+;`FVUy2E>wnfQn|fNy|);K*!T7|yM2-HI4*IDGFV zN+fd^G4Eb9)Rk7R;}U$$#=c%Z>Sjkt6JA+u@@$3z@-Nb~q{@#^AJ$Gb0B5Y!x7xm1 z#FZZ+q2$6h*(M2UWfVG)IcI3{xDhOUc2BOYC~~eEs0xD(ZAn8p2Rw+&?(OHN$ZtZ` zo&lNyYhDlfC{+=%HOg-nX|D7<@uZ9}jRTpmlekrFtzx^puo=%Um!b|~uP(&1!@jKn z#p`K$4w2!RCo5u;dC{>qzx#}Cy|0Me(;IUv@R(O_W_q7(D{eMV6HbhVsL<>~iuVd} zwEAg#<=yT#NWAO%B`^_ngEay)TB?xGoW13(sv;13*9S8~^MKOAM8xgYK0W#&jLL#Z zW(p4YKScmEAV;~X-wry{%bbg81;HC!s^0Iu#4qxxGB>rNneG!|jZ!A5z*k|h&2hpx z1PB0YLj>P|VbOh6tfpoT5N#cB1U9>CaPU4_gV(6Jx0&_^D9N*-UZPdqLW@1gX%(e1 zkfQcyjVAcMoti6$0g#)VVS>@`E0G`z-|p;uUR(yUXreOM^Si<)H zsVJGyblX-T3OyKUd%gdFb5D#*%aaduEOY(LwEKk%7j)G}3T_HqIf&LS9C0lK%+kzU zH(zF8BAS{CAbBH$7#^sVaV--bXLZHtw#bNw7bC>EidbvTawnX0}yT*@Vv&^&6+S|NUzR1GCE4cfgMt z1MSFN!si`eY;H`yWH8*Iok;9!#z zQDiWu>VfZt^~Xfv{l+b2a*KE1C|8?X%rvu^rcKpNgU92xey|qf`$=#dlqnTT;6jdI zAbX55fEE{bV{UE=K@VfsSGnVeF05N@cI<3uog%wDScP#vaPjzP5dTZ~gflS8^x+;SIeXI_fj>N`7&joc>tLOe20 zN=fxR6+ABiutmJS9{QF$kl&cqtF-A?q2NN(Q}LWLw6e;AE%C8~0Nat>=hL%H9@gaJ zVHE*KU*JU7+gSY`-h9IV{?k)siNcmT6GN{&=V6YB4?O9%p+XG}4H*k{2+}D3fM1wf zR8$n+%nz9h)S$0Zj2#^4YMyX<_-+7XkK-HvA!(Z7zShY9s3OzcnT9Pka6U60znz{7$i{m)5*KU@*dV zMCPDOO5OX@VhD-xVe;t{7Y@Xq<5&VGh;q&AO`Tuhh7e!)Pd@%RW|Y%Ee~TY@@w$@j zh`%&wVuBt4yA6;dPr0hYL^xDkK#bZ&2(*4|8X~rDi~riAZg)JcN|7B=2QFl@kKO+q z+}G(~;9(v=1!zRyyuzUiT!Cno#5L4Ef1%0YOtl2NI=*IBxxDCW3>!ka8>=0(xl~tj z2=Pb!XcKPAuf1DP$pQ9PkRv=?)dwiiLcuKY?pkh5L9I$0&jn33|I-gC@T06m0r*kF zI|jT+*lA)c&-o37*kR}6^)pYFOxcipSLHbw!2{u}i^tq*;o`YKV(jHB+mOXwWb)di zzY8a5p_jD31qvZ3;?-`OPUMeycf3JeHXv?(anb&IZdD{$wq#gW^S0AP`S*j|qGEc>Y)4*zbYEZ3 zqZRZiv7FssUV@xi-C;G)6J#k~ul>6>68zo^q-h(x7E79Tcw}(qAoR*Q*Qm$(#OcEk zyj!FVe?`tER68G6(w2V#1MuEx|MR-!+QA57>^Zw%5Scx#AO=5@xdZC^mTKwe(a|WA zRC9O+@}M&GC3bb84h80`N*WLaZwn}Zct4)}on8ISL4_E2RIwP3L$3JQ2)Jvz?Lh@x z3(afwPtVg_{and&`$0J_T{0MFBen0DP%alg1_O0O^RP!rM<>z9*!XC6FWqMmD8 zD4>aPZE&_ejjfU9No>-<$P)+EMJ$J&==yQrtT!MLR-6x^3r{vi?+!lCZBtHqq(kmV% zM}3Skb=ul$trf?&8H}aRo5uSF{_PZi}mE?`Z0>p>Ids>@<>WV z^SgTJ_oBhc!$H+yAxh1xH4h0%_kSjlT$_wN^X;M2U*7=-lJkk5@9#_<_|pHzqko?| z{MWYipCEnkjK2tKsS1v$@>Uke(=tNlHrknHKf;AXhpVnYtQz#BPhesh$uY|0_m9+_k1i z^zVO$ztfFuYK;0_kJ%Nl`ChmX2*I+ZrslYZY}5bbM&+?m@8BqdfC5;IU{BoTW(ATW zmj+m9zd*rpdw8nnf9{x}AF;d8zSAu5+3ELQ-||o#IXS0(hU62sW%&N*u5z9Lnt;dD zN^5zt_k1qTba;3;NZ3#yJwN@DUj@1a-cJAL^XU=j#aWi;iHR8*zkvW%S9HEA+Fku5 zMo@YDxTxrP8yg!?#4eV`{9Uvbt_*%QKj@i~k#QL!m)W+XVqcoKx6iSAV*pSb2hjtn z>exeVft5snng99t-}tmZb%ZaKy?%tvd6eA7snnKWtot`5QJEe^niQ~#AE$T|yv<&h zK@2-K{M+9bLb`;=2heDs<m zlZl2Py23ER8>{=%f!alhLw|~j1ot|Bp{Ywskg{_ed!1P}qWHM@_^g!`xumymC6tsf zf!2^FRrCJAf7{?M;a7oGOVF8^aEACKE%8$lk+=uv1Dw_GElPyM1BcqQBFbi{^Cx=1 zmp3XL^5XvfL?4W4eyecP?8J!^vafuaIUU}!`R<3jBOo;e z)Hlklw6n3WD37;EjI|)z7x`PzEOTutOAU<-jf8SzGIM8zf^#6B-U>_{oIZU=1S|I? z()lB)@bm<6+jF!jc{_z7K{(_|iC>e&Z~N1e)U$QKx&Qe|eYWcd{qGN^n6n3@{`=$T zUxkPM{QjKj|Nr`*q4>XjOdbZJPTdqd6C}T|u+Z|m@V9N*Edc=)_v$g6#eshZ|6z>f zy?EUMBhe#A^bDK|4Mpt+qOto!cgsgQ7u+QMpidx!uf;O6vKHpM&1;2@{CQ%3=Q@l@ zo@8d$qAvFPL6zh4`VP?Q-UY=ZBrJM!U%B#q;(3W}UryR${~?Qe@xISE|Lmsg=UxpJ zGtN>^k1BJ@rmMFIa)eSD%v<=>72eiP=n@^e9lLSgC zKcZR>{@HWpMNv{8xWrdwM&Od%#=R!>gjx%94^+2X_NK`ASfEGrs@(-Y1O;tZ@%-84 z;Q=@t4%^*25wFT8YSUAp9QF#Ot5V5#gZpojWB+awsLUfRty*S>gO52FxU3#9i_^*D zSNq|=Jn_P?K>x^*BWIC+g8U)xIWDfO)YQMh_ngw6gSgq~1m&}ZBmWGpGMxSX16))8 zpA$0$3y6=Xsj0=ZTdp~aW~OT&Udqk2foFXAQl;l`&iBt&4>-QIi!uhOXQ2&UoWFda zx_+{&JH3eq0!B?1$(a6(pXa&`jXvx1cO|Zz;Wp55VU>NZp`j7SbL7}Dfjf7){`Z`r zV4!uKoC-5~ha4^RCl76a2gGyh!Qj%g^1laSV&aWSq|RAlV+=pA3>FL|HIwc7o|9pJ z{+qMpQZhceguVx^#7AF0mM7O6mKF3T$^u)2#EI?g{sJW@xeG@~?httMt5s*-Kf4CP zTiU%{OYkkZmmdB&^WU}y{tx?F^G$&<7D60wT!md;ejE7TRN~Kp z5yfqO{ZIgU)IR z((UOo{QTPDW1fDW?QZ90WW4g*YgqC{A3LgVkSr;xm#xkz8u2=BpuqUn&6_z_1P=du zp1jy^?}<8D9c~z|yQ^Ai$*Z+O_W1Hq+xJoj-4Drbub%?h#;bI5aY#=~t3AXm6z{*U zgZCHjOs5aqNf~?!^rL@lHzf)@%~p?F=#X;gw%{CY?(9T`@(VnBIVD?Qx)wq7D@YLS z&(>f;^l6_JG^u!Uqo)^2k%IciU%S?^_Cn#`>-Z9U5lN`!dxd&%d{Cu(%FJKD<;ULI zTV4wjwQXa}4tIemJz}`jrT}N^_vpn7=2px{Fyaeg;x{VF&K4rX_Nbc{MbAtqO|r7e z^2Z1N4RqhUoZg=^no9163fF8UZ19j_Begf2QL*$Y z6B!>jf%Rvg#5Fe`TQ~d3)zEZf{@3Tz3Qyh(+$+va5Dn9Ga&oGrNhYfs-oD6tgS0sq zJ7w``*!8jes*Jv+W$d~0n~T`@Pe6o6A`plK(GblNOSKE#TQx7~XEc(Y;yR%F{5E)h zUS4j^GF!ef+o8uGEc;XIDjSSWi;0Nn5XD~)jAI~@P7`0=zI)f7^TJO-%rR?TU+cqC zV&*RlO?p`5UXP5wT-_Y|$&;&l6t;iIE7Nf?dJ~KlpS)`h;p0-ni8(Fr%ZJYUmhdWBD(9>gKXQsWT;>HYcpx0bJT%8Dq8pJAUktGtQz{{+g~+8CvZ>yXT_ z*mu``saF;jHr-PeJGKKJ!)10JwN0S1cPFAEbgJFi*q-Hs3h9CG?460QnvL)WiJ$+t!PSHa$*{E#Mk-6F8fNJ6DbKxTfP_N1n8e^|yiln98_mL};%pCk3Msy-T?z;NPUkRd{0gZm7>T za{>N`ZTG)X_t5KaaB*SmqMA{k8XfFg{i#VO=itQRWO&Y;JlP}9GESggVk-?FfQ_NFUcJ&m zjob*3hQ`Mm@(SzsG_XPEVOmBdL*LJE4lg9+qgpnNk0N|mE0v5tHhiPhA_(1FAep71 zSqAD^m^Z(U6M8o6MVC_29PGh#W=wQ#OdB_1WIwjyFw*|g({(&5yNMB#ipoDgy|{r2X< zHd|9Qy?v91+&>KC=wDbpvpAHVXtzgOL174I_eAi2O3U*r*9TP|d4E(GtE!Ap4i6d0 z-P$^gqha?ADPlqHASQx?ZOO3rtMQT^WS)|trGavH(os%Hw_oWLa9j|kRHD|rgmSOf zWSI1DIHw%BTR&a#i>VLQOD$zRJWGS7$ti3{xrkKhX%~l5V9^`38T3qjM>jW>FrSrM z?eslYI=;0tPa)(f`i}ISJDM!AciM=xv`ua&pMbX3ckeq4V5?tKvO%D+Z%*m#Saf^j z8Bnf7_|?+J*Xmj;*bpwKaJEVvk`>jUwos>7u-rgT^ zwV)pAG_?-1dDf6k7xCEr)S{`?yQu+=qW0$^DH}(bJM?sQT{|VuRN&5%EUpeYViU8= z?8H4~$nn>4yf&0^o`}kY^mJoydgEK35AXZ*@UmnUtpUfur841vr(j@D{e`=kfA_}T zg0SC7MVB`;Rt^W47CvDBFA%d4cWSI6kVvoMkZ%E4(*!ZP3c4J@wmT(;2?Fr`rvmv@ zprR7`_MHbpr)+Fe(ktxxj`R1_UcP)ec&t|E?%k16UOs)Fa~rlAQ>GFAR_c#*hR<-z zezPxf9xmq9A9*=l>a!VOinT8c*uRS21mV~}F}mXrk_A#-nD4~hGMio_^l7df2WWMU zH+6A4g@zB9mKkPLQPAB8V_f^b8}^d?v~Ptk{~7iP&I3NToE{{@T@;N7$->gNZW%gD zxi9?s?1o-=A?aqF{gIjDf?Zk%m|J~m@>1x?E@x+{m@i*)veaA;uG^1Hjdomz_|e?N z*F|EUXct>m7^BsbeQBUapl=iS{UnIS_M;VCU?@j#M@6L;0B_CDQ?=N4z6x}P^bi*a1t-DeWNH(<|%nM@4_AC`Ns!W=POA)@o;~D+{1+9+7)#l^A>tXlMiz_!$gk363hXMOhyX6w@y-{oESR;}4 zD}9<$wI(J*otgJV8-s!ZiXoDCR}u1!oS)DVg6(!UJQG`8 z26d(CeGK=5#R*7Xl%iWSFPNjAEz}my^2V8DfMLBP8K-rz6@t>KDzzSOOSj+C2wCr{ z*O=kBsiYu^OMOFk1LaW#(^0cWi9u7xX;IRO`-G|)c!(SAOI=l|-^exYHIQ2G&rWZ; zHS@Bu4pRD8D2rp8e9e9 zS1{6#-0sO%P0j^4oOOGUQwZ~f@d0=`flA%nv1mq_()_mjuP7*rIm-Rpp*Q20 zD&lNhN@VjZ{utL%Fr83kHU&-GKAIwy9X&X_FQwb61HGnxY>ALET_kh+dz!sHw0#w+E%OYEk6E=pIvfIN2CGr;s!$xuzX)Ri;#!&j!e_m)J= zl?6>Y^Gz|X>%RuIylL%mGUc=MjYqm6b6g_~(D+{FGp}U8!$3l(+qcsfL;f&69TyKT zSxo#q)oI}A`BjMqf@_ueO~`a32dx@|FO^Wl`^j|{6nLNK7fE2i$DNR{+XDd9HaD5-Og!fIh@$(noAv8KI=LVEP) z*#>?C@avGjzdyUDJa@##u3|YGPSn6`wrFqx6t&uKx*ajN&(k+b{b*+jo>dyj?QWHf z`#Hh^Wc;ca>U$smK^gDAX3>v?wx-KSe~(le5icQB9r=C2=bY%$xE2=+%jz_RLZQ|* z{(|%ZE(uEM?G@o^wu4>_Q8>jW6uH*dcME_QkQDj_(PQ~vzYadO`xG{oMlidCiZk`s z@ff40mOJ7nyRDkxc$wjE-x8|BRgR@0~x4f{X0n>g|vTMoh z*eFvwyT0Ed&&~Fll7Nl*R&>Rj{S>1RlVItc(01->gIqZrnS^HSjb-ZX*@I zMHRSE?bcgu8UKBMnKkA?pN)B=Ji@)H4+Mmh;P8T4X9`r*Vo?<}vS&m^Pp+M*B=NP= zC#@nj`ugq{bci&lXlob$tgmQfRD=FU2>S*{3Uz5Q-}0e^|=cCxawe*+M5 zW?+s&K{*S{W?n_(3Dl7Tp!ki=*>~DaW!SLaFT^%QOH959>#H{K0QITO98r_LChib? z_4%J*4i3EavZae8f0Q^G^pWw&U(-uIiBG$9?RsJ!slmR+q17>*vf2gq_Y=3Wd^1@U zUGj`L9U=29!#tcz^xJPN_V@2BfQM5b(;8ln2|*TnvxBPKZuv?kgH(p0bqE~UsC4dP z!fcZBO|x^GBS3RUql9fr2Ka({*uJxP9_ppgbd6gBE@{>e85$ZgMKfbJ!yjj}8I@St z7j^u(%%p-#b!2%yo#qrrH;R8)(>xJWPX_1nq)9LjoG95UG|x{a-C<- z>aS4(KoPR-D@<#|?xkq!^h%d|)aFGgMsTIgMYg-`Zt0@W&WhU2$sOs?(FoTMxv0iK z%xk2m1n)lTPJy^-55cUh`xgY^xh|5}zSv8Xlv(e>gqJ|$}aZqsjh&(5fL-NTPICJnF#FX(QnObcFY zp6=cz_#jRy!a;-fje@1AYRFPw*1EP>Sy;8&mc@=>um78`Q;Sz!=a6&H9?c4!Gt{*9 zEcf2~uwqo`^x1eM==}K2jZao)B}N6qoQTx-@1JaMZ|B-X9WQ%5WOCpDug{)&nNgXC zBYbLKiwl_;E6^kO`0D}&w@P!|B@^F`fTb62xIa=vQ@5QSs z$cFbW^i9^^Qhqv^4yxL#vSBxMjHalput|G@=&kO~)BoPmg{-`p+QUmRx})Jxb{CqT zgZ<>>iydb&Tf<4CKd33SMlWVGPGV5tD(Ke5@ktZi{+&?t|I*zOFV1Wz|5$Ld1$`vz zS!eKwd+wXh-HxclwOf|#tXFm@vHqdve+&f$V$KalRTxA7p#MWwIKn4E;nshrxpG%Q zmABB|*yPQ~G3KRgB(yqONA%yTWMcY!b}8m(quSkVhiYo0ig#Ce9A#nQC`=qhNjOadHh&I3<7ku^uJodBK*WHdBK0N>PREIEWH7}Gj;_*8+ zJ~yGR&bIqn4{nxm)qxre^*NB=VgQG?+eoT}XHe||;t&Y4qK*BDcM()v9+pwTEmZt( zhMdcPZ?(S}3`pnxSx({pjL(j=sw<3xET1^I@p6Yu^D?S-h~lx?Dog|HNs1%>mW~zw z9#*ERqWuW_y_u|4{TuzU63YNux{}ZHGTY!! z#|@tR^0MvWdE8y_%~$iEo-1kW_+ZE_G573PAvzQu9h;B^SoTgdMb1v#db&|i(s}nV zP;@4~Q=7O07ADf_6&9uTVhAl22`z*lZ{8t0xsEK1+*Ka`!Hsjz(Ed^WG?E64G~9s8WMa~s{P6K(jt7dk zI>vRyrQ13=_(6JtWAfJu#IIh}8bPz|d8Ix$P>sKgMg98am$~Z7 zsBjN0ejImy_wy&m!c>TGA5dXzeg^6EqcvFiQD&&$G^u3u0+UK3*V^Rlu*8?InqAXJ z0A1rf8?&G5S6$9UW6&P8bcYm1A6pb5m$4vH+c#P%S3OtG%)-*ECcfd(9#;jZ$~op`w^zpo3-(D zjBN5o=Exyyz~I)2<5Ou$;{`u{Jd|%5yU3C4F&;nf?sq@2-FsrMn)QP27P`mMkB-Ue zbVB9VRZ^)l=hI=NvgEOytc;Asoj3Bl5PU~UT3_jXwJoQ-0p!@<5Jtyu%D!2ehPWQr z{HjNR4Vs$S8Y>_ws%B_tn5SQm@LDW!uH87J0lT}nQKHP4;MTkOow+aSk0Q%~)NxZ# zw4GY{a`GpyQLS&L9gPEp%@IJ&&$}nt_8*LXQ8YgtVP)GrmFa+uVo=Q2)0J9Y-HVwg zFaSbeaB$#Q>U2{$amcFC8g%nKkn=axdI7-*$TYyIgw+Iu0%p!@%pVZEW8zaj2=6_{0_fi*1J7tD}z-H+$rgGxMXYaKz=aa{8 zjyJ^vVm7R1|1LUcq_WA6Fb4TR-d~BnoUqeu#YH;C!Z|RlA&u%zSDYtTf2&WbjVO$ofL>jkxmM{^Knls*lZy@A zn{GC4R8lOBl_~V`c>D1TXRgd=ADMCnj{+;5dJwf2vsKxHWgNy<#(@+icpX9SbpFEdQ3*iDZ{xB9HU^f5QKnqE5zmL;k|~q zl6WD8yW?R@14tgP%agQT)(5flv&iiwi7b+mKPPM~90q*`Yt_DvhU~W>Zgyl&&K$BF zxJRl2X2w7sP-Ha8i~ajIuV0t1tE)4OaE5N$t@l)9vtEc6>2HO4jAt=(lvgO0F^GiE z+?dS5!jEu9Ixq7ugWKuS!jr4r$d%bj-vYKKb$NMtRp7GpnM0s!aj_@%mn4RBN@arZ4uB56j$PJ%YS5xa1K-Err zTX$j2gdy6%_9C0IUe5kOv`4}VIRunNQF23eUpD?I(2VIjLe8(6 zVPLuI-KOoJj64+a#-VD(yP=WX7qNEn_sgQ265D~GU)1K8`-kq=>+?3O%S zM*Te>=}n?;mfE<6#@?o@F_de!{n0L(D(LB|R)fQlUWP1O(%*kSr5vDv=-&L=TdI?- z0zE);?UW%qr)-ckI*I-dI*)Ih4rve_YZrWXtP*%QcBrIW@$t6H{pD7jrohbcu;|%8 zx@9}Dyj99{@#G-e{%U?c{Y|7t^-rn^UlfEz`c4}RL-VSGUpumNL&K;Huxt=*K+2?C zl4?oU(A4~5d?^|Cts$GpPTQ(C-Ng7mr7*W;7Qe+?*=i71<2A5& z*pIR215`0w>56tI50i#t6Wnzpx0eVMibTBzXy|cvQCPy;mtJVZQw=$2eM;|I7Vw~q z5PxH801e4z^Mr$msS+^QLKQz~zS9M((u3_B>SjI#n-kJideMN`iH?r`E_in&*lo{` z1~);C0Iq7W|CCwVebgXHLZ>di-7+&HgP)hTKV3sn@gtATCV9VxYkKDZM!9stid^qR zX(Fcwan<06FJ}N3=<(v;M+wrnZKp?kE6KfQ> znMeQ~y$OmzkOKsI(K9JsDS|B@pkyK2ULaZ5Lani*qL0CS+x8uAL7M@rsjRH558Ne7 z?mlUC)JW+!p2Ridi$%7bRUo^4Jk6=~9KU<`b)|A#AxCrZiYC46JpYd# zT1h3cb{X4_R!k^NFfrZX-#h?&gwukY{C!@ZP5f`9UaM=ghZYg;l2v-^DYI+XOK$8< zF0WS^LbdZhuTfzSuCQ;eCK`RUGD_k&#;i$mXS)TwAz5ns$|Zz0!6-=v6q|SrEGof` z(mR26Tz&B(Sp6{eaYrOz#3xSTgzfSAj#z4dgBc-wL~D8dwnf^wL>dx@9>1r_us>De z0Hgqxx-&UW6f348aR(CRq?(SjcNFsXsZ7oM0^uA~k2Pn2*5##(l-{xx9 z7fOYplRI4_$B3XBsz|@(Vhq?s2Q4H1PKNWe6HqSxg7g@a%+1dinVZrIj4%V#oiJRJ zSF%7^jcE9TPU=pCoc?y&Li~A^L~5Siz9%UPU{{}Wt^+I*a#hy7nESua7otom zvj$;+b$YDbapCSrxxMhl-mKteS^&`OWyIzl$xl|b+X|418Rt(`1`N7IJ-HHU>DqA| zEw{hUrOVwSa%?f)%%-!WV*qf+anLv0Ie=@cf?!4Yp(pFh#O#!lk11?h>M~uks zh;E?(-^1%nd|J0q{*^#_&>mz&057@o>souLp zihz_<^-}^hX7udd_**If`NLJBVt7A}Bx5spsdH~^@vVQHfO*FcZfNTR6{*Zw6z8mto7m^gv)0&tDrje|~idc5fYN38M~)P!i8jH-`` zEWH5KK5tVlOjTRE7dU1Ny_eI9kYZ*n(BCi+W=0iGpYMs3Fn}u>zt`7}9!+P^m1K^g zn)r+)8}GTj1us)Csy$|PaRin&2uNB6KQU#=h9g!Y zH}Z6{TTSe}>Z#!9L=xImX!#NoQ*<4|haUxN80& zbhx&FNGER{Mw5UWLp>stYoW*tm?jdox{bYOB5Az0@C_~89BYBVC5bSG0vje6@M=z??2r-WHqDufJv;bhgu9D8sW)D1h4A~7*fln!An3dvfFG-ER zR_L9IGoPVXGV)Vu-l}Vg#0k$MB@D9CB}_VB`JRXfLrIU)&;Q9Ts!QLY`a{j^EY}}_s!lO zJb19L6k-m#1vW1{s5v$`?8@renhwT04e-?!UB`AA&q?!w9`7daf5HY+bbcP3c2D6C z-^|4Hf4lSj|2s9T2<`@M+_;hCXuCf0)%;oI*TiF?^ zK4d>V{g_It@3(;-{-!;a*Eq%L$E(#i1W!|LB=L;mDBtDBvdy(TDx8`&<87uwoh z1I;zx`*|S10dpe4;vB zPh-pbK+IuqncoL)4{=JQpFE_>Cy{h!h!p;K<1kaIwaYupAeX%nJYHkZe-GTXMQfYB zd%66hNB#WmzYkZBrA3viN&rZx@h}hqK$Uy7&bhRIOidS>7V02g@(J_ol?{zKoH-%)ZK6IuoSIx1SVH;VyNZ%*15T;%uAZ5)dF4uew((r4EhY zud9bvk6jHBy`dig9gsZ6Dh40H(eH<#tZZjrQoPP$Stzt|>^7v5exdYV)^zCAe zE6y+lqGp(f3I=YQ#YLf_%6lgz|2!EVf4K2)|DMiB&yqIyrsiA`M+qlv@e-7_!F~W&lc=?@mIi;xh`g7r<;7*J!%Xyj?X6#6S>bSmH z^{TZEsd=@0ETyC{oaeJ0=$64nOUX6EZr&DOZ6XROOosJHn)U^GuM= z0luh=UR-|g4;bn6#PA8mKDAR$4_N>Da{h=xhEHQ&U?Aobh-7 z2qOQ&m%3EDOIElu6rMog*)!gGH#bA|jvVkgYhgt=e26bf?)owl6DxeW@7lMX1!AnQ z1%jRR#l3Cke}tQnunj$*mm_QU_Zx>O%rQkGuPHd^%q1{f`lvRh~>YB-7cI z!@;J<2M#dBKolw*^aB4iF?-F7XE|f_fsria=H|i~*WU-3_=`u0x(b&>$!<>nhD7i$ zSm$p)qul?|Xah@_1YX6+{*Q~E>Ga?KljVLe{Xh71Ad?xxORRYgES3RlY7XWdxjFcH z117<$?(JO_&JEUhfYJ^`*s2JD$9hg%%V#YtEL7l&04vz`R}_Ffe+&Wr3=|e^pUDpf zZggg}_3DfUSk3dH)NROxxsuMs__XhMhof2_v99c%0&TfROmq$YUkY8eUh_OR1Om}f zZ4<-MPK_j?ya`D<^ZxPi@xX`6jiUj+ps&X3ReYZ(U=zfMTPx=s?EU?FB&Z$TUXw5K zfX}P4?*P+7Bq$@)&S5G%qgBd%*;$m;Y$Nw<6SzDu{T}@wUtig8up3oDRgL4_iS6tX zF3-|v4CrVN9Gc8R=s>^48@;S-Xc0UK5U*v{-6j}sX!bY;83oAv_<<Q&kKwcR$%^2zngZbiJ-|VtNs-flQdTDFe2nr4uug1w4(UJ z{GjT&c`XR>n%tHE6>v$ZxBn_M_$ws9FmJM+0@kqb7vhUEu`xOd4OU=zQ7n)dEn4*j zjW?TzeeQWg6eIreH7Q*c4SP&=pzR=PF@ov z<%rn_Ytc6Bfi#|yg2cz|8;0x6I*x|y>-1Wxy^^u%ul1uv1CRi4Tq(@1qg0$j>~+tV ztM%)3G)D&Md^?K0BMF(sFX4P`49Cm#@IzcoOaXSU<-{Or(Kj<9U&04?^UdNkv+`C6 zVIq1|EH5~@=0zZ`e#vb07r66$ACUXA&(!JwiJiNUi(*z%Qi8xiR=9Y3OTiJl_9h=c zLa9Mr?v>Mj_;6ugM@3CdkZP`fp(&hm6Lta!3&5Aa7c!f{lWX3z!MN?16sk!6>QxvoB!o>krr)+5jqlOvt)JOIS?})d_Myy#+rP>V^_K%B-G9f+ zIdVe|4*v$g{Y_c_2moGAk^9QBo zmO#U(ct()%pL!2W)HZ9e1(M$D&(i1-oB}m0Oi3htREOcZlw_4fOhWC3oB1bws4)}W+xQr3?K!tIUOor>uue)0Dlgn1$*PR_#L^T6+ zW@BR$42(fvzfSFkgU8Rnpcjbr?Ge>0wP4 z!oB^Fl18oZI(6S-@jgU5PRL+kA$t`ezP7b8T}32E77x9)9WMRMr{;^g?|%WJ4VFN_ z*4p2#lhCfb5QOhND8}0zXa=IAI}%sa3!bgST@*IWYd3yrlKz(~jO_a%#rI{XZGYZ^ zt#OHEL6*k8iH%Z3)ef-S>mh32jc(8v^A}F$Ca(PrP^c|n+e=+$`R#{_`_spzC|AB! zldr%XnIgf6HFF2#VfRgm0}f)it}>iSfzLH3nzEIZw;wT0;elQ_930zyuo(SzDeZJCT=3+{ z<9nJ~wH3}Rf89{mJ^bYCBi$>1zrA_Qhoug3IXGIYge%2-NOLX~y@+8$) z+Sk*S05R^UB)E3clLT6=cS<9K*nj)abZ4(quu?#h%+ttH?TIl^+}fWN{*sjR9fjRf zFR|=gu~wNpi(_+2j_cL|ZNm!EZlyCylG zSTR!gB8pFa=P9siauKgzM`hF`(Z{q;@@_5ThK^!oEE(U$rs3 z!-Z>Jpes65?pbn}`h+!x`)Lse?EBhKmX{YcP1wa9GO|;Ab!S^*s9szFWYq39wCdE! za_jxQLkTO}s=y}SF`dB7-iJGtj?jNKqc^nw0z@y9!tnzeOkJt|GJ=N1s`_@tAhH%k ze$?8|8kturv|7p@S`suC?X9M$eVMt!Ecf!)otT=blL*{s#fmsDuY}U8{v0hC;o6Tu zZw@(#9$W$k*dT=%fG$3i6tIucn;z#x$@pwoc!&4r<7}G31E>pqEMe-;wR=v`tJmaE ze!@%Al1m+0K5}~%no+X5gLc)`mqjdXH_0;I+nHLXBkrW&-m%S5%JfZwXn;|nrkvoj zoD=68b05*GxJC?+Ch2UAHH&M585!rjH!&xMi!Jg~yyotL{yJGIZmxe}lR8!`QpL?A z1Oqf+%vVoccwg~{P5CCBY>if1HT_Yt43J>M8f)gE+)Yh4tQY$_FL1JETDG&>_07jo z35Wn65*tUgUj0XzRFtKa-{_?h zCsc6HVwJ2`n%)$?0MxBl!(!_6^^)72la9ufa#!oDq{#j$Q8z>s0|@16dn-*JxNPP} zdHJ~dqcyT@$Tizso6K<6jNKau-#O~UFN@9;pcqazQJ3YfyREg$@Xqe~Ul*>}v)teD z2{GlMm0uJzT6D$Y1NL>~DeOp(ajyI-n4Qn-waq>LY;5bP0^`kmKGm;FuEwvI^v8Vg z@3{6EeS4tF3GmdPqeuEPRc`q0(^-tLlLnk}DH-=Hq?O9SlD%i}HPNl&$Vk~v9pr1n zGKVp35Ncm$PPc5%clQ%&1q>`hbI4ueQmF{zvuDrR_RSkzPjyH*^rm_XYr6p?D*JYTmXNNc znTb{I`e4Ch%onGp*xN>+@QCMnm*wSYVG^30N6P8bXH6XYiz~g?m$E+fWK!|S2WlzO z3LsKrZ$pum?TI-mgiBDE-eu-1U1l~53yt+BAdACgQAj5rP3^SDAHD}1U;<_`Y@K0% zlA>v*Z9lTZ4SVR=8TLA=a?Cf@^SzH6f?0c)hVUE*cH&IA@^f-dHjco0@6 z3k3iJT_S|vg!1EXkx5Yj)yHrnXzU}>NSR)xOC(rbhOXUqF9(`?U$KH~#6%TOdjb`s zraFjn2~k^{x(leQyh5JMjH8zkX=T3SH5(*eEJ&!W-iabE8uWSiSo7 zb_n?71R0u-IU>%?OCHhrb?WXg9#6|p+7l9% z#NO^)ue|_*@Mg>L!efE1gpON$S(w`WyARVf7}gRUl>EC}4PV^Amz-NqvW=bb8J3<5l^iU|_|@ z+(!z4JW|j$7n-Wr$pxkM>7=g{v+z6+#)^UNcsb{(jHXYk2Bw$oGmhne6$lPBMFnCG zqu-Td(q$B1WoBj)KioxKLQraH>SK53#jdAnqfG4qFn%H*bSADp;k$-xJd#7IRPWid zuVjoz4v$U!$FN2sb&*aMWzXder2o0V6aK+z#g4O?O&esXnpOjF4vps8hZ2$ zyX|3LmMaf(K$HQk+5p^sG*7Muh3sAZ%agt}-3iYC2>8+gcgW&k?z^+k&5GSB0TKUR zIePgtS$Xy8c_95WOVIP;cUH2^qyi{x4z<8f&SWLtEyB#+6J+{8F2}L z=iK~=tR@>xyVieX)&*<&Wj04IL3-8~%;vWVh&B*O{+?4SN_`j~wrN+eM7gg@?^KxY zk`1H-I|Nqpcc*he5i|AI5Y0-57-CCK7W5!nZU$& zlZ6mO)rFm@r?*!RYs#shAn`5-n?(6kaof1I=a~Zz8P?M)f|$tjTH|Zv$r&jpb?woQ z)ocu{6Df@@1ogWTDZ96;`J;s$Z&UD;%g@E*RoSIB6qvbus|_5$0OIA$-+u*9Eb4~F zw%$iZ>R4&wzR8!BtfVxq_c2ZW7zbi{YD&61JEhh0%ULedHmmUdi-HOKi*L`)CW_H5 zLxW$BRzx0Uu4%U`eOXy-0UNPLE4zeWqY=y~xUK_CwSca+miBZQtFnltv0y1&VVCZ< z9Y?BYFZ5*gRq7~ByHaiSL~W+36`jT!=x;n8zqLwa*QzR&Y;?OL3sRve8zX?v9!8lK zF)ZF1`A5PaKd@aCkSLL7r}RK9u>) z8SS`^yQbT?fSp_BhxfbE-pRZr3fUW>0rC_3GUKcHvSi3*t9^C6!Wrdr7zYIa17x&& zbB^0D_6|mnj=%z7X+A;B{lAsC?|>aKv{%vne*TxiQ<>0y94Pbft12Ii)B}N+nm1t^ z=PiIu)y&`UTJw-shwU}NkcMxve)lAMRAPlThnr+%RB30t#Np(}larS@t4pjpEol#` z$B>|^b49)2l;AGxPyu)yZoI$yv)gaBB~~v}xf`C8+~qvYH^06hVwmDFR2+Kj8hV{X zVtwLd;|slRzTH(PFL6tf$z(umugVZ!lALRXRga~i+g`n#Z;26LZ)Ha2qIzVtp93aa zrQ_H)@KO$Qp9rqY*M|d)mP2qW!~wePZ4Yu!w_P>%+_^KhpejwT75b3dN-@o0--lb? zR;i~eUVgK}z%?5lYsIX`5&PzfOjQ~bu6Q!8#!Jx~0mxX3?BCmYs_|_%vPh#6jnBr* z1ZXVMt806Cuta@XoIz-~SnLVLdsrs46@tGqojc!aicn7kOBri@V?Lg~oSMAfaG4V> z-Sl62}w!*UV2YY z&-#dC0Kb@f;@AaOyhHto-RX(4 zyH3G83)x3bmH9R(gSFoBGTx)nHp8)gw5hOJ`iWzuo<*DTXCTU|s`q1AuKW#N2{BY= z8RnYdIl_V5WWSqu9+d19o#EM^T0g`R+s%1)K^)y#MK?7%^uIY2bb&QyQ}g@$_q zfQiO0(Rq{G28aL}UBetp9;=!gdSX{wfmP;ao#lLvHqyCow{fLCS;`Coxv^Ss1z^D* z1GDzkOAWD5G?kM7vLV<8AY^#b^&#wzcWf>A7@-|{ART#NEE?k z*Arf3f-+4u$@XbgGd9%3d95@M?(e;8R#e{wz{K|F`{sl@FN$~8$+>m!j?K3Si12-u z^B-LspX$erw++BLW+n=%e>ycB@!E^H!D&!ge3|7i7&3k-p_uqQ-PYY&De0I9E}1MPC0UD%pOHxi-l`|R)GbjIh5ErPFdM_hjZZ}b=v0I6DPHcS zO~=dK?NJ7aVg9H%R6_6-JwxN;)nj%(4&r}l)Thc;{3 zvt_I~XZp1&=W18dyynmVwRriSDL%~63i2Rhjn?0X0HE$IhjL)(9W;1Gglvj!Mvyb& ziJ%m)qkRALsYh8^G7XgG`MP6d9boYQYSeW|9;m0lX$MDt@F?-8qqr)?1axmwis;E6 zjU3>;0-(fiXA<-<-euB)^#BB$oBH<8|2D&C#c=tpKZ@wKAJR*EEHUPRhq6n1rh;6M zxzH;o&UeobbV7NCB_F}+Bho0^*K1mD6TC!23oPd#Z5|A49jbzH@InNQ?(3{9FFzIv z(_z$`yIJoTaF`K3w2c5gfLRKlad#IMn8mj`v`DDp*=u35?}BzUL}KIYEgnI$O)MhU z5ZA<>CmYN7xPv>cNJAf|0OU@MIOeQ)_Cm zElVB@GV#ZG5IlXkQG{1nW7Qs1yI*>7$jb!7a^IN968#qJ`W;aGcAQpqBbr-2vT)B8 zb*5lwp|gr%$oA5A$+0OZ3$VxX_j-^#yvfV)_0mE@>DR}q^i&Y%Cx>2_tD>CmHq_Sx zL#x^J8c=-gZ=>yfFh$b#I|NC)vbn9@hM)KN`dvFMSeW8U1_w!v@1|mhlHxK_O?>SA z6p}T9%3n3#i*QdfZ>rH?Ie*zJvx#7KQLxz!-fnI)=eyN5$e*gDdY&Gylldv6|T$9je6Ndtbl*Ps|Tvg$+z5CF*b+oWPc$^UbpnHhO~# zCgD5tiOYW`R}~c*9i4G@Q!Y=Bp!Bz?b>hhU`f!=;rB?gl(mVbEAhWNND^4gwAXKk# zsB^WSgMfkT^&37*1NQN;XPJa<5OGrYi+P|gYx*?%@cF&?zD-L)qB&zX&^a}h4-_O? zxqrn2-_-qy4gW^MWnc~_kNMPWB#IJ!{3Hd}ez-V{C_{h=JM{T}IOqP(6*a^NaTnx6 zF0b`l5y!U3pmf{aHpn;ae?}g_loZuilDykfWN?JU7w*;d1k4k6viNE}&aCtQV(-0! zqTHUa!2v`BuA%~>h=5mu(vx?-La}Y&TK#&YWlALoMU{J{jC^=^wVF;2l zY>)T%{l0IjcK_Vn+S;8eEroa9aNcuHcR&5~(|x)9$z;+HDl#c0FMl*@h#dcltgk_L z_N?RbD*nulB}Az+Gf#IKYO8&9DjxFZX?5*p=r@9?TlVBX`f_FSbblAQPC;oP)zQH+ zR>`IaMXdAO7V!&-HP$0yXBd|Pue!MA=5)`sT5gDp%R6JavTeqztft5$Je_HI`)eHT zoSdSZ%1|tHN?P{HZ@+x`LV?^EuqXc; zC!`UL+Zxkh$(nEfiXDW=t#bXlW~HU3mZM`HlgYn5N2R`qS-KM0t*ja%I|2eqp47oNaZ23sJz4OY56giVGgKVoP!|Iq}@l{4S{gr>v zJLtk@Cg}#&9hY31kfn)O(AM3~T`dtVY9<^y=xiuqrs0{#l#aAOXOc=#|%^TgAo z5@w-Mb~wYNr#xNy*Q52Gv+eeDCF=3cShjqq0bM>6C=7A>hTKd4*?sh5p2lT)AY#&V z7%g(PrM+(d80dFAot3^{i)lR+CPBQ;KmlopDJ?STGWFd4NVL7?Mb02Zz)_6e_|Q9# zM)D4`HBGH%^&BolE=44Iz!tQhS6T>I#kmnYeb>3UUFt~rYReniOV9M>VmUJvlWYg( zTDn49xVcXo0%LAn$&|PHM%sJB=JjC03OCXFU@txQPXLb{OlsCqz?7`sK|h-IK1yOR zL-_ni%WU1fv})JB<3cS^FU_V|5CcUIf=1v5?CHd0r+9v2oADWj%EiqA77p#-!3-Aj ztqQajTN5YS{r8IzUYdJ15BTXMP0>oSb+d1V>b81b(qcE~z^Oua^<~N*oFw6vqWl$> z=OnUk^%Q%hB&)Ntvwy=Jg3Dm}BQ5^?#FUbf z#7>;)x3~xk`^ikZX~Vj`DOCMXP!N?GV8;CO4xAv?0B8sV`DVH0?)v^2UKvLgh~>v_Ef=a4=X6jy%fSMVz#2so$)^&8Qx7npt@o`8leY=J8L2kl%N{#q)xg*7w=3 z?^ty=_z(W=@5{bz$K+V0VWWwg^iItW%IeWNt)cXCjAJnj}sl~H(<#@u+ zHQA5+u(&&fM0S#}i|%5lOrB%rd)Bp>JL0@x+S@MCA=R91Mcg>G*~cju;Uz#hi16xrM^4T6i}OzLQrMQ+ zq8_ImKD0tOdiiFXMWa-&8Jc!`4oxoap8+;WNvNLqLM+5vlW(ol3Bfv}xI#0Sx z-6@KUjEs(0_S(d*+%?FfAk!sikLNXu0uO2DgZS&fH`HWA69g;$*?tAAT3VguJUD(t zBZX@UxAI(LZOc(&Yz-J`o&YQX(Ui3PErjxd^~A4lcW%)dJ{jiAF>253nCOhQ0uVGw zxYGMP$j$uJD&Ic97#;W|1xom(Sd0l!6buZg@BA9=U7v>ns8m>H5kZW&t0;fJP-65P z!txiT+Ms^2Js(E81DN>raS)!;dCN$EXp*qfERJRX0T{W@pI_yE)U|^udMjBn&6gRmjJI+~uFYaj`8z*sk++{Us943~bm?TVb-vWVXZx}5{ zhcA2+qkVjODxw)78h{1p+k7FNtlhdtRHkm!nyJK-nE28q&XvD$cY62Xxe+ezTVvH4 zfQXq8m*s4@gbtK|Xxqbw6t1NZ6zHWkzSO9;y*26W#Kou=sezU-{-^?wE3^_2;qx*yh#1V76Roid^&m(FxJIWFdxoS@f6h_3ZVsXT0s;4V*sW{5zm0>|Dj@ z%0dJo;vC3ILH3-JB^`Hesh%rW1dy+)$ZoGnO45p6dU&Ywm~Z`KGs?xNp@Y)}9w9luOxAm9i5g3h0#>m8~AV{*DN7;jU@9 z?2-U7BV?EL@^u?CVVgg+CyHxef3}@5H+x?WnAKBUe%gx{+*0s;>&}LICjrFoMXoG- z43DqL&+x5Z3m(1R(!;ldyNvR0nus5Cq{JoS^O^oBrk4WWobP%r&}CY_)ph}K;lj19 z1J@2~O>=YmXH;$S+U6zSu0^g?;A}2Jc_x6}bkxQ*o`Wfn@M%hEPI@dhsHEg6sLBSc zApGxRNQ#h8m;wU{*FsfGzh933_MfjQ(>P*z6BJmlQLBzTs$=-^b>~&fb4~Zs%VG2) z9w6UK*kLW;ac=l)aNs%TnX%{GY@+OU+HJG`r73yO|1jf2QqW!-@o7y9Ny$smmvYsr zyAC`$`rtr#z4S)$-Asf!>}BJ(q;=yY$%;@l0V_q$+^#5~-2Hn4;acBJBLf1y{W;`% z0QP__r-W2FN{8K)l8ZJE$`Ue@<+bX%O$!8Nfz}wofSOc!n|TgFVNx_xKKdf!fus09 zba?~uMPicnL@bqYX`II2qEA4@PuPvqU}>qSs-K~Qf#~0N@&cLPYt9qTAoQYKDCAWA z@#D)v!=wnjUD3cGV0S`qzQy z*+$Y=oge?5^JiHJedNBk)VIugjq%*b*4vinF)(uGkHh==SIb6M%QDAJ-0p?^a7Q2( zqB>qgz|AtYvj>mDm9`E(5 z`vl%3eN%E9RC)!x;vk86M8dJshd^B3PqZ-$Pvqj=!2-;-6%eE$Q2ZJt2= zKNACiID>Lypt7lm0LmZFs~OFurHKntRa6#v#h3ui+Fc8dWf;scRA7{&;L4f9mwfp* z%r}TSmWxnyQ4FCTvn)Hy9+9Q1koZp!@X3PY=^C}L^2d)_l$zXlsqJzK3WF6(yp?Y2 zzLO3z5`)EB*|_C%_ngGHGaWXqLN$!y>R_hRlV?;!2$F}-nkiM))OPmC(<@ykrNef} zvL~{8l5D?Kh$8)t)Re@;q$BuN`-v^xZC-hEu^}kVvhQ zeT~p3r`fty=2>$sTU^EKaG4J9a+$Jh8Vv{}e>jM;bX@OFN{mpQ`EwVI`zH|@03O1& zPMR+oTmI$rh+yU>41#*u0wt&sDm|kjK)mOJs=1KQ@{>=7=zr!&+QOJ+BP<6CXfH3;CQ|NB(;=oe*9|(mBrPpl zC0u}v`>!rQ9?UND@hZP4$J35aLSaTqVjza;dR(ps*(uEK!U6@bwiMcW1ll3*5}HdgFAeDsBmp2FySsXJmu2Z;*3rr+O6hEM ztgxb@!a9kUc?Z)GneC)++M%;nIPUuE_s3=$I;{OY=Y@6^$dL2{b&52b7f@Cz6<%M z4>Upv`CD<_2UPa~O8e21gsSlIlJ;&fIvqHA_pax}ZEZqh=Hzn6XwT}tnOT+}9cS&q z5ka^gNE#6V%P%bZ+xJ*rGmj8+i=l$j!^ttW{&~#;{d9~>1CwmR@vMlYGP-EEYICv6 za<{s_(!GO30V%Eq02Z4pOa!gAI9Jcz_>UyS#6x~4S6qhL@?dkYRcIM#BT%8L@w26?RLcXi9MUwk-7q8pUtl#1WAw z*f&|XH@w74g}%KuT^vU1`P(oIRPn6)yw;V5;1NSyW@l%qz1wW+Fl!><@;jc~!b?rP zNXw&-+FR+*!O*F7UF!z4gU+&}FS_-_3UC!Z?|tP;i^=^(Y_hq%nf8?DKKTq+ba+SX|#B)w+Q$(&-f-sm2->9RH;6(%Yw z+W$`FY`juWNn$HaF|qx^i~8+NJUByI_mLKy z=FtYlxH@BHiHm5v92vdc!C18lE?=~G@@71+2ux|T063uaoG}CjNM!Gp984Z5$3f>^&z{EX3@1RZAQFEE2M!s<21)k7VnQK5h_M(-_QEW9wQ!p4ck>wKsjnBi@^| z7kQ1tkW-f0wfE_y(2J%=U6HrU3N;lH_un<1nto_IJlt6}{}z|6-k>{3lf;$1b9#6Y z$!F20ZZlc+)z}BpC{6TG|b=SEI`w<705b;>3o zABz&OITY)khww0zWa|raM3D{P1AGpB%n64}RJ7xTn)$wQXXSv>sF!CATP)0{*Bfe0 zmd+bZI`uv{bT}QJctUWOCn8df&%4>!Fo~oiK7a2(r5I=|U+b)LwxkD6TuufCLVj=* zUGdhbwO{Cl1kg^?10|1DGtiIz(bf-9w8A=CP{2rL1tIBs&*^7Dfo48rsEUcnvpvuC zFal%qmgn*k)egK_Kqo0U49mV?Y38Srav~VzbF`G!>FpGkdx5i4j73d%F)^DBU|>d+ z=@tsx{|;w*a{RDg0%>7Q3hNQxZ`gY5=;N;tmt$8Q$h~9i*6LHXac1O5C8rzCmF!cvFdBC78^1XEc%U;dN}@(ZHN)`W>6tmuhC+Q>F(XWo!+%3c2vJs z%)9orzl>c;vnNR;n>qP<;l-PRUN+-(HbI9wba`RoG}*u)G{bre74B)l^@8X;S~0Z% zy{nh%Ra=b>kU0-(Xn9(TxUSS}B=LG#wO5J>F%B9xz{HtMq}y37s1;ZHC?;{p|47=< z#CtFCKQqS3d^y_)SZBX}-MxtNDgkxRtjKIdtJCAa!{cDP%^7N=oG0-YCMD)<`E4AN zPRA*;Rnpr!f$SQB>L2P4zG$Jt_Z~gi} z@5t`(bg8s`nqrbJwb(N8V3EcHsEYtHL#{#_#o(NHgG?BfV1W-ib> z+ErhAtD3-L^2XP&<{=#3U`NZ)A*B1e=+i6pd`%Y+v(cw}68E=kf`CnymcLl`4(2P_ zWH#!}r9Q8D=9}pilh_q-Av~kK2J>8CIFdS}-hH1DIm~y-|5*mJknQPlcePUpy*#)N zu=FnjjS*LW4|Lo-+)nxNj?4>B@4-ARtL3PTu?DjgLyy556LmW=Q+2skA<%o{-s4R9 z*uHL`IxCol5gCDK_8&k1*%lryGwBgj?23A)3>2sODpUX#r<+q`*MA>6f9RIGzB2T& zNn}_`56jN?s_x0t=I*@rW|#D?JBTcP=fAiNq9~MyKlL!?SWJq~6ODXB4%zp2)y6Om zMTj*JNduSA3v6eOZY8Ung3Ecsu4o)U;fbqat4Ndb#c&qZ`l_; zZ77vCxdtg!TAu~KJfq^OWZ&IPeFMyLXEdM1+<5ECL^SIot9_p!zO~5^Emeuc?GN1e z(h-K&cQV#!maW;VY>r}YS|A>^4i(-gHlA`@Kg)&HA=?ua2Z^NH$nKccYDXMTt=L6L zEb^nkOrVNVj`NZ#%x9To-?7?4uq6ry`(l)2WU&mCT!#7}mHotPcddVOF`pkEF;@$Z z!h%e~IR!GlP5Yvr1LV53mPy=)96wuKA!~0IqJHkdnO(TR*>3Dm@tFqrrdS{<)y=)X z8gZO2VKtD$B>VoY{dok)9rz55Qn9Kr-G{t!X=ilkw_UeTfRb<8nVws3L!Z@I)+u~* z!-V63b|gwJ`kSb>YQ4LIf!kyy2`MT65Re_a^1tn12UWTg1}mK~oZazT>1(xKZ)Jp& zz+as2_BRx>+q^*!e1N(s53ETT;*i6~3&bHGb2TnJ5#im+ANAZMHuDdL2}*-o@wJX7 zl|EjrH<=-qrXHQoX6CUHc;Y5&`8_q6Nzq+ifm#sofw!fNK&rzn^gB5XmV=&X{1q*& zqKu7o^xB4LcNdpfdU6AiT!fY!g%RhIJ8GG7i`IjaqZ_c@N}ue1>krGDE^G(|nUH~g zJ-f!7m#^NifG0atFq&Frqi5z>lsI$*X0^Ab#U@MPq-g~GqJwQ4J^O$X)aqg3I-lD3 z_ii`Oll|=2Yy3_$Kdp=$;t1(g65j~dEYi>aDs#2cqg>B2MIZLfcE5VAI0E3(@WrHf z9IB)!Sp&l&1&a-#-FbaQ-_2sI;$!JCoZ5%4=Y~hmC0fx$EDu6bDp!*uoztMsI(m3_ zBBLIdLGjm8ddWw#AxB~Vpf&Vh9rEb)o5GF@C(`y`45pD@k0V^yCNg2kp}X)9fw?;6 zt+bzAWTB6PhEuiHT3e$)CtI&w*ENOE#pdRacN-*Sr!)ucXO0y5u^A~;ocu8E+xxP>Hk&FzB@R-gth3mvq$k@s|ieIS7 zv{Ncxb+H^)2gj2^R^)7av2xNt5qr{&KC#GHTLu@a_rwmD{rqhPU%?zSmq$3Cy+uKQ z7_n8Tua(kbSxIdtjJRMO^0fEmEs;~+n45w<$n2`dIF6}aG5vHOh7Rl9(@KEYqST_a%i$PzV-%-nt>2{136yV;-G4yOX(DlP@)dx9dtp0vy z9Xb6_RB`al4=Y|f`RRd-_jySK{dXOnT=_|-&ci&rVrQ?(Ph1p-1o->OIs+h?Ygn&bTAgYb9&u&h#tOah-k{V9&uGstQosM@<^9l` z_h>GiY_v^BB(i)tJ=sGCuhG%@&7_Sx>8_t)u9d70bnNMgfBN+4eQjx>X?HR+`05Y} zZ`E^l7Zj`g4A>RqgA|!B0-p$Pm*$@H-Fad=DfAoisL(5kQlCE)Dqlsk>%n3itWaY9 zRoW)Aw7GEMMnxp;yZgu=vw>z@I&>go? z*@)zw#ypITSSUG4e%NoMGk8Kc3!7c*<}yFF(zE+8cr7K$dG62PKx+(@FwMJ0oc$+h zX*@g>G-9p=O|+WlzpP)7?794O?}59!d)8;#>ZssaY7b&WY~-6^f$tMSM!&KN(dLKS z#m@VW({Q#3MCSVY7W20ctVvrk|7SR>+wc0gh|9{%um>cOLQS;7 zIludzBFZoe9cp!U`{}JWS^2FrwWg7%e8WlMJk3?hNpANVfQYyjRDY78CT-bcS;#-> zimO)CSbb0Jc~p(5n5ssJRoLI-@$-KsVAkxjc_>BXFHZ_DJV(Da4eA`9EqvbGN%-J3 zxiV!AROTYwdnoiX+Cf!@0sA4NdOA53XXGvV*AJJmLb%|w%q96X0a2*zY+8EycGFvw z&l=T@cy89;9n5r^q01UhXN-s$p3A>>%mOzTDX6Mt0Civy5_$x1jfsykXRVTJ-P6xk zT|z91`K<)x9M5;>sc)>W4)34bu(+aMJ4A4TA*1(_7C)`OqVIQ7`D1zFp!Ghd%4ZhH zXd2f7qt4TKg|t@KND=F^=LrRU#^RXaNYSlnBM*}sDMuOO*5bniHTC~A3lmKkksY3R zH~xc=oZ>-tL+&ZCF+r#$Zls4_#X=K!Du&)07w5M|Kw>GOqBqlXhqZOH&ISE5tr=nO z(V=i+A+YGTm#$kGvI&wa0$LcJ+J92~2DL}!lW`VYiH>Xi!L8*SdJGb#cGx7R1Kg)p-w*@WEjCh?JaY|K5400OOy{u#}7n$^lMW z{MvY$2YFx$-NpT>BRbCo+jEE)oPs;qYzFTJEj7Ho?>9#E`h2N_h->*=3_+qMU#~nV3WpXC*_BA~@HLrsL;|PY9i4hP4xdl93t0WLhkx>)p6?bz% zu0+Fzrk^Qq64E%eW*jlQa?kbK(R2gq*4fIp-4i>NMbCZnN?m@B&29Z8w(OloFEL@K zpX{sC>tgSg>*=oj`h5j+uDp7xPw)2a+pSfOi#!3iTCTzQny4rWv5n?m`rgRku3)qT z!UM{EE>+kL>JRJB|21TNx~e3$F+d85aAjTH1PJ}JWN3Xfr@ck8!{vz)k7OJj-vdLC zWv9X9t$4if1DOvr(F^t524&8H0@f4kvVi2Bv?$(@9V<6WnexUG%sZi63vn(@@tOYc z1!#9OGAw=Lh^*16213(pCyY#hc3}sI#Hr$N6#zFo%GduwJTkCd?(K|D4_)MHJl1Uc zz>M3+q%Ys$f&{J>P;=R2fh?5z`N1137}w~Sm@L@StTrMjm%Xr;8}{drI>}vPp-p`z z`4t<((;!UgGShVX4yIUsW3Hz|k@1F4fb_*;l|YhsrBr5FBcGEbAd=kx;3^wx5Gwu7 z@Irl`6k#~CR6bxD1!a=Jc>?u(S-{C(8x4BMF5es6DB^ZBTkdTLorOw`=jUBWc1gE#s z`@k~H2R5FlaLJbQh|w!_Xrt`j#E%RLhriWuf1+8)pM@MoCX&lfyN-A3Bx4&kBE`hX zXn7_uzQ6+N`^J?hxD1Lea&`>|Y02)Jgt&S$8EJ7gSwoZk<1QUf?}ZbgBS(Zi@mBTe z${1tog_v&%VoFe58OZ=zDf;9R?ZPuj21=r*{ym98=_%LP`=)$0s6B@X43CcMmfUWM z1Ny-*7YJRzT(BGsp~O$l1n0Po$H9i?%vN3$-CgfTAputF-JfXi%C?Sn+ez4IU6@$I z?WIF{Tr3O>BgSH@RLbr}7RpLEZx%M+9gLCQK=GNf@%S5;;D*zN!>d=XrZo!geNSBX zi-K`U{k-R~>crA~je4LrE^&Lw%t#f84AS(Xz*k#u+{&YM|1FCPz`D;IKK(qaQ3h1d zkc#$h{C3`b zpSI-1(x~tmMXh z@3Q5AB8vjjOFcPvChh1WBh|`Wx=CedS|>&n=chbc?zpAr|Aj+1eWwi!(=|b@wB(!) zgY0eJsXKMU)s6dVelc7|J;vLWT}FN_(17o;;|F$Uhu)XIpMrfPKt__AOEmKg<&^sI ze3Jo=ox6U-#YwI2R_oV1*?L#r4duGJp{oTH3KpF=DV0wGT+LNrK95Y`hG7`v9 zvXHCdpI*yUgTiw&yL!bW(M%wb0ms}ihK!68SSQZV7L}2KCVcd{=As;>&v|c}OF@TI zJP#o_%~!x)JA8V4Q@n=>YEd;}k+#tcV#nWo>YUmUP=dES?dJxbgbpQ}G#hK4cU714 z5g~=BmR$B7nX4KF+T0rXu9fepxRU3$XoMWqVPpRY$@s#AK_`6)NHc8cP(O_3S4qE)ctx|%NuxiWuAFO%;*wjWaY_nfw zqCony^WnqmTQZcd>kp3zNc8C2e4GOQrD!a?Y?0Uem7Cr6I06^ry8N2)I*0P|Klx1b z#^Tr6Mb;7=)92iKdB!#Q_nHkRY)bL zre66syDY(P=^pyTG?JRXUb;|EP%W}Gv`mz#-o3`QrZE~kp*;0meFS9f0d{>Bp@zFO zd?a7isn0bsoW~KPai9P9<)_b4y8cP1Q;0&@*S0gN&qiQ$1;mD2J5)w8@$Ff7*h^WMftneB7 zz;k*pN`D840f`Lqu}b=P0;~m4)~+bfHln%o<{VzT2=Gn-=DQZGbFEwl{x&VoEeVb; z&e6kL5R8;9(`N5ELw1vp@N`)>ItSqT8cIJUMUWt6RdsdYvVD|SuV$2dpZUV6nha#U zvQZj2rD}(15_%*3<-Nj3P&X}VT0R~fW12pXPd`1ndP(*YsO8G`XxYAc^^Jt2?C#xm z)tbOAO|sHib6x2`;z;qr|K*7tXKB@an-tHU8ZvJx4>% znVH#Cqenw&?-5?KCF8f`6vdg5Nmry0Bq?yq$?~y4zj$o_<5sugBnf?ZjpO<8VC_lE z@^RuKg1f4)YZP=~)J5xmg}lHtQcBXZS;}INU2wZaL!d;x?OVm;?|oDk+C=6rFHb*c1($?Td8Ap_&FSUeK;R8+=a zEC_$8LkO{d=Q6W#8eD3CP<4_z$Bq@xAB{UYgD4L%Lv}d*CjcPq!=e#0VUjBp%5d^! z{5;!CIYRqf;#5zLaM~Pve+imx^cKuzn!hicOU)r|_GG`@bq*mUB}0hdB*Ts8GCs@d zP-Xzc5<+no5RzA@-tK7A#}FeN13JXs>i?5F;>lIVxn|iUVISeP zwoz*VoUMF5rvfT!-gj>RJK}VxkTWQZMnz7}@)M0+`6$lfj;`+h?IG&l3?tmDAh*@{ne~j1iDXci$2hbams>$ z)`KoGI<#(30VsYpX9e9hkxT}D;=JzKv zg8w0nGPwKba(F|qzkfh5;|12Vzf>rs9zGh-B4mDGQlqT$2L4a=w_4U;QoBV_qh)%U zP;Yy5s^Xato^eOPLbovmySd4L5yglSFGVhhGo#J=(T5vtMwQRLN+XY~h6;+N!X)yt z6mFUvI27pZg_(ArN<}PSvFX*)r%HjOBI^W*g^Y>odW;smQ$7iT>w-{Xqx0hj22>!H z2Y;+~BXjktIeFsy)*rZzbLmy7%Ez8&x?sUo(qSjWu|p@_)+b3W+CA&HxUOW!};5x}16 zR?&Y&K&fUq_2C!$JIA$<#U9-LJhHcfzlQ6$En-}irGSiIJ6%Vu+g|jtc$_so(3^rz zmGRtFv9=y_;Jptg!PaqkfcRr@OENIRLaRUVYRShjC{OZJxgX%NLZ;ju`0f#2GUYVMexv)hCf-1^F`X!Wdv*wwZ6V-;AFw#LC3|A zDX+5!g$5|`$<;R(Z{{7XTcs5G;1<5ygo_ze+#waT#wK&*#VgPc)eQA&6DQ|e_L$%s)Fjr%qY$OvjxR#HYo15da z9FTz&M`&p=5M**a%y}8vbO04F1ol(}0|$&eEGts<8_O1;H32TbpB%wb|Y!ZyhCX;yePo;A!wLqw+2f9F4 zO@q%-&y#8c5Q9-bY#*4Bhvc(}O!yTi`~DQdo#*K5X+KmK4HoD?my4F**a{z+P6vDf z`Q=xo>n!(b(n_a;Je~UiE8LmHbu-x@7ypsuHZ{8~*8z#l6Pwq}h;Srs0QmRXj6xPc zSarEl%ZSI9(7Ejv2j7dwa_-`8YocOfKaXMaA@sj_TO>sf50U&zR?$fX-59ns|BmkZnMq4A`q-8|jAkQ2AvfSEBmVX> z8`q6$zayWQr8L*IRq2$Dm0?_mGi1rFzqrWq#McjG_ohRO1jt{7HWZLED`kzxLXo7^ zc3fD<_^0liVp96Oh1L<)%%~3CvIQ<#7MIt>T%{V-iX1yjlQ7QCOODl5(-juMD#Z;Z zT|cF3lJd-YY4bF&0CeMgS1WEQ2whdc6Kazuz&CLw$05+$XEbda{TE#}U|$*C83rRzPNAf*P$WQpCkh4p0zx#9t}FNIiQt+gj$L{jA$s z0Kztrkn!2spfcyb?DDz9VBRQJ>(x4j#uPs$1|)=S`Lk`33uF@5Z;>KIrCi&FK&Rto zS9P`da*0oLcqhqU>@2dvo_zO9OC4;j&`uaz<64LLLsuI{mVvwsC^q!68CryC_IMG4 zF=JO@)pAX$(^UYuhP;0aGQ~fPP8>MIAvsI9b0n!@S5(P zIe_A7t+N6@posR{h(qh@S%ZMSxVU&-f;+!_8$LoQe6o~(**dCJzw~){|E@;<)N^(j z8B{iIGgnDw{HG*?^b)?w4*%r zqGRNvRod6s^`SoegWl132&ld=WP60Wq^jkF&tHBJL89k#yY$iro-&wVDomfBv!te` zI$o+S=e*dD_M_7%v@}x06D-D>wf5xIi$}Ve(xJ;&HxK7~gUdv!l7)T7@)~Rmo>@J5 zpHUem?xwid72EB4v_-{veGg*T2@5@0SQ1&kemT8|H=2>b5dS1g-!AN%v9*bH_xd#s zIX1e%l27wSUlHwzo;#NLO6!KMFFlr5#eHZi8>VGzrn23*P&RnVv{%F%+h zZp+R_oV(}S=?=F&223Ni)rXBb6mb%%*qQ(_va(AIBpKviw2AmIy(zr|jj^q%nsgOnVlVXbP{Fa`F;J>Hr+Sx_WgP~Cn==?eWx){^~=HXpp_?tlk{u86o zD<6hrUtYC+~JPx`|>-0 zkc0LIEdp3(`O_G#e{9j)b51<*i7D3S!ujKgM{TN@^@$Ml{TD{IQm-<~U+;?LG;pk& zYzel&%26nxnbyE0@Eu1Fhd#}2S^ZSEm+a~~ezG&nTBu(s9YH~%ZpfkgdimRExxjd$ zM2ah=`F5?-=GI-yZgn(#jfQNO34f;V=HB4k6U8Yw^#76TYr9Y69eP z1omWRw^5U>Fnkj;re~t9m?0!7jN*%1zx3-b&pV?A`tTz9E>@FYy~_7|!ArVCtw>*| zVY8F;QZY4P9~P~fKhm!(PtBJ)9|@a(Cyh{*qh;SWHmn=Rm$dC%8_}(EwlJn?aAs-s zkhFEVS{A4APg2tEkW{4$V*Rn@L7>0?^VaAsSg=jq*o9m*>eEhS>nh~;v7d=h3(v#+ zk2%uAlc37VzrZ-6U$kBL`@6Sqo$Hczgw620VD?P;xD^Q4O|h~tUZCSWyhT~Y8vT^x z_U+HnoCN-DU27wj4KyCdulskw-R6!V!0%%^*RALU*rLCGREZeRX3gAO(LHOgUF)&I z`In6H)VQ6ashfqBRrf(q2VkFBBlJD5;!af`-|WtNexNFgC7u0 z>$UO>WfKt*QTe=ub>1~oPU;e%eMEFm_~aTN_tk{8X6UK;G$V^4U6bQ{B#1ex9YeXH z87#i1W^0r^B=+!fivAnX8#7s2DfigK*kiKpVxC@hjKPs{+ux!20|W%}c~3tF*jgDC z>T1}Gl&C~hm~;sQz7Z(3I_s4{hAif3*p=$Uj)U8%OZLuKgw2ck4vA9-BEJqEw!gyriQqAz62`x}w2He`qd-wQVXYKOiE}|dDC5ZJ>C-Wdcb~cwa$K<2 zpmE#4VjC$!@IF`&k#$QOc5u?~%gT1Rt_~NDKLwebiO;46uH(~w)03^63H;cVF}c2F zeyh+$uOg(`o$7Id8V~(CXIU=vaowd`atDtTB_+8-kIXCHQ#>;~KO?=|k5LJwlaQWo z3%iY5XE$M;V$=v}Iyu&7qU_T6ZGZk!J(4(1)(A*2tZ<505ODD{i!|?yip2Rr;8llk2dv zxQw4^mnff|q6SS*ps=sPb=6arMeF@gp2pYoF@hSQ=H}*Ki?epu(XJKRB<6Fk$*eiQ zLzsrxyeikc5E0Eb{7Z6QDjS=vhs*+xNtNox3ku;>eK6QeJ+$2Wm zfBkD#h^@MHNZkPC3Vk2cil&l*30Kr?ptAL|o5JI`CQB(Lb7Z6|B1521FIPQRvbt(- zW8=pIGO?UoHMRTuf>YjlO1ta%dUl0bw?y15`Z79Nt3Y9*Jl`R4eD&oD>>6TLr-s>N z;&E|n0)Zry1H8T4TGr6`7tM1Y<+cv=q5_=<=-x9mP+C-(8+?eW0_*L0xa1r40WW<0 z#hw$JfBn1O!h=XC*)Kb>`!3d=EOx|@dF(t*u3zIm%5ii<%rx;Dr+&o7n_+M6C1cbk z2Y4^ko5%9fGmZ_gLmaO3Z?I|=g*X_28^7aSVHa}*{rSEQH2RE(2u5wUBG=7#mZNwO z<)(@%2b(74-l7Ph{KvESwUdN5J$$SytIRieLaBE?^&1O-3k)fjENDhBF$FEG0p9gX zJJ?F{Gso`y`U>Rz>aQ-a2ee~FO{`sp7s2PWf*=sN1JHt6Boo1Ero zav(}Pv9dXS9sA*b7eO&TcYK$UGXJ#Y2A$Cn-A(mE>a=&{;_46(t_(pCA)n&Q(_?`+ zhdB7i98&03jaDl@TQ`YjGow3SY3FBFH|yy-j(lS1`Rv!z9q)`Fl_EwBxU$$N+OBTa z)wY@_TyX!;)TV$#xrw*23D2L|p$(&n@Hd+w*=6rsVhhlxBqvXw^xEyT-h|#LgZqCI z&>JK>RtanP-qPz$6v_wbmOS$xX?eQ*4^7Jt-c%|nDh|!wX>*?2Cx!sDC$5RVxPoZ& zZ>1Z0VlGbz{a2ob5F++AqPPe99gVcEl$c$y%R+1tD0%QIw2Y9@OZ<*1jpnA5^liRQ ziddUxG0w|gY%EH9_}`8Jod1i=HsQJdR!!FU|IZRDDqNGte?K$$|5J%u|KCnSxL^2_ zUife2l|3G`QvA>7|NP`)!QfChqDT?p988gzP~+qgW<)!Q_#OPF?w7d0PFTFUjtxH2y8{+wU3KB&}=n}<4J19 zJxS*M;Lvwf(szDQ>RC8UYsBWgMn4o?ulY5H7$QkS%)9wG=3Qb$*_X)1obY0Tb~{lE zeO|Bc#2N^D!fEtmnA;onO#!6nG6^HxQe=nx6fFgQrrZrZk4Sj8@lOnS;>FDV+S(Ps zxKbgS3hTR*d5hu?CP?VX01l^(Cth$7P1kwB6aQj2l}i~BPH9ReCL707XP=iVPT``ek4C_1UlM+N+~6$=H^%9jq|yTk`W8!jg84}SKd@t>oj`X5{9?AFJp#| z{mlj+H8xId?*3dN2p7!oCjNa~(lXpOK3}Aak1(rm5Zh~}$KCuFH7?t&4Q{o$%aX~n z9VvQS1XCCF7Zw*8C5-68@~VjuAL6>14i4kC9$K)$5H@=2;Dr-`l-$9=p}X}(c_>~r zGU}BmB|$80?Ck80^acXb@9Qh>F~=+5k3?qs*>;n;o0fmJ7U<~hb(;Q-N6-JXLtZP; z%L}=P`D~PhXtL(%kRbCuzs9Y`6>Vy;~GSZYT2BVR$aYzkT_`9=ym;jw6uD>vdrbJs#9 zzHejgWayJwCpLsq^f15-Cdak|7a`AaOuEl{`!sKk{3^gtiQ@+N_T;?r^&tsZkAu!f zc6Q}Z;4!CQSm(*FA}bqGh;R&x)siD-OA!^QN_b*eM)6x?Wzk(c9lxEt9{#(3^fNb~ z113*XGHWfaBi|}}Kz_`BHmAm1$xs<^;HDwATXVW^= z+|0np%*ghRixV-?{$djp+ao%<*o;C8VLp=R9#gn6N^NXtIiW-lPuqrv$LFnz%--x^ zh6Z{I-o2BRQ&eOkN6cs*>^NB&J_I2980fS(SQ%&LP^V|7vz{}tQn|3^+YNPh?bTv18 zN$2t>=jmT?`>C4>M0h~2u=zlQ&4sg3La3scEL;dVaq=>P_Sjuo`MyZHwhTh7kawWR z{eTKfnfjsC(BTCP${TiFA8s?JT**EbG0w9VT9`_>_Og2>y98H{R zWh~%F&U4KyEzHeqIrlxF9UpRsip#KX_ZABB9ak6`n)9tkc`OG?lI3F9KLH7e?Y4O4 zuJlsh%gv3Aj?kdgw{IUaGRs4s9e|&fc281-oHQLApd=JPm1rtnou4=0Fu}ED7n!!k z4v)_3c*Bd^`^&_l2V&GP`SBlrn9S8kaNUR^I{>zrrYBI33>s8qW8^BO*~*=WQVl!_8Re<2P1qmH+o6 z5>!2x(v3Sab&0e_VlSB+WpHCz;{lt9ha|vQ$<)8=q3Q}aCK3Qnq}J5j3xe?nwDOm< zb3dcdNpQ&i{yxp5MELlWr&$3I+t_5Gv0}0GhwDH@3&@_KvcoO;@U06YFGZh@8s%Gm z*^%+uRSsRQ{x$RI4s9M4sf zz?JEXVYl1#N1shSWuJC4@@m-sB8jp&6|ryEUK^REb*VYCl~o1au~XwK!T481WY~`E zEtj68ESr4y-ANi^&I-78;O z^zLohT2$_EyW{R02iiW*e|Ld$PrDLWP?LNG1M?{-O3F5|(KJcv8 zxRbQ8Ms>IUX4l^>B`Z;S(ugx^|lj<#8RYxMz2JAs9v64+MR#?$@W6vmQg1d18TQF5c4%x*A!`^}p;1M4w ztNgpQKQaFOG@X2L(~`|#WhgNdwVw}LB<-Jn`to0MAX(dBreT6Q)MlN!zQy{JKDM+JT*wpak6NQxR+uvK-CJF$9p2>0QUW1ku8U&<Dq1Ri;W)0eKX{&nxqifj_?!sem? z3|ZWzkt3qfpt8i;D~B(I#)SkiUH04|W0;6!z_2MA4T|B7mwt3%7ic^EM;Bm6z@K-+ zqxMqd1QA__4joE!Pi%T}+Se7|#-6RBQ^%5ItsBgeYG|Zy|Z51YUBIxW?NhyQua7bJrG&Q5p z+~$kOiAvoJ^u=orA2M>B<~SQg*P=vdo|Di}Lu}SysaqwQ7u-w8qMST+%AnasB-Wh; zUBjr=`TO0;?8gI0TRS_&rP%|)K_hoHwr`KQ`cir{{2$(3U=ZapRJ;tb+VlotmQ$LD zgv`=|;ERK%19|9DVDIIUXes1rWY?B|{Td;2acHlS>OP`l>5ZO#d}Y?PD9b3~M{JJt zx?bp8>(4kO>s$(Ps8!eUxpqW}dHDEjXT0JvA#^jsiyQBHvPn!>^ z1Q}()MLXLioDR68;qb69*VZo4YT#Xmvji1`VR-UE`t6v~{X|E{(C&^6KNKOWC-*Iv zvqkzTrB>0!C|<0krA3I;YF&TI5t`R#zH7*Gupt`{t$`$>+1ew1|Lv8S(H6NP!fpV>~x7Vl@Y!jld@0i<9Z_cIB0?dATADjXe z@_eJa9q~CJEZDog(gO}PFC*yvZ}+5(fA5cDiyF4v{96sZoek~#Pd|Lve-@qxP1d#$ z#2w?gY_QzO529df{R3)DrmxX`w?8_w>ph<}{~=xl)N$8iKkjb0#DfS#A2uRv=!QBwUK`- zxX8WSNL|I}P~7ldC1*E8oBbpA`*r7Xwr`#R9(o`d;Cy|+6>CQwrdwr|a72#{qI)P% zE#9s){Jwj)zl8_c0+=C2K)pqh8_0DA4F)ZL);Jw-aWi}ycqfl)*i7%w`Dv&YDQyhE z?c%V9!iTB+d?(orpMnO>&nKJlHze&HwirV9Q{V30hkQfH5PGd(f76vxsu7GC44gI< zDWN%K0-Yy&UH;E^?ufm|&J8V%N}8_y*veXD0Zf7da!obGzhf{-Y1+0bH_DfiRs#O` zByzI8%}MmA|D`KW@woy660;k|J;T#>0JtmiBeU!3e$h=+>O4*amvDvL;E%=OsMo8P zqRY$$2&KyIAH-5HDGh!e1K=r)3F#qc277v36_qEmc!|}eQ+KcLHv`864Vy$8Ue9L~ zN!Id7p(Sy@8u$kYYc4Fq`0>x3%>$(up#*klfrv}ILjy8-azj&)p~gs78JHLPs3W2X z)mSSqj3&RePpLR@@L(~Fvpvp&2~D|b{&kj-N4Yuh-1L0+iVF3qQ}b&e){%48*umM9 zxYqV3@kJbeRw5IP3em<67n?htaC~RjB%BbV0)wzPHjRH{2iIcVuWkNYd|Y|i=X4-U z1FOGA;O2Vyxs;Gx^6euX1UHuc0#%7CJY5=|S2Yw`I*6E;l2zRfEj) zdXT{@VGom*S&zwTi?jD0N0l>yo}g$uDcVkFyVImI!%pix}Y+Y7ofi{xR%iEBcAhQE)M_7Wqg^vaa!== zH*LE{29;Du7sf4aW|qM~bCY}b_%8U3x$O(=@Y;PYK+5Lgr9%?W`;m$RdgzHDM;$*Z zqb~bpTP;LLDe~jO;LKag7IU61ilR26yF(fp>!xdxg=l@e*#A#$^OW6xe0uUqN50h? zI_;K*C%tw4)jcbv4Z8Ff9woL$3FUiaT&JEHkTJ}yhE2W)Ri$F;Xro*+{-b%kPQH=k zC-)2}q%3%H$$E=rdCA*Dzof`5V`peI4@1)Sj5Z4^6~s$4PhY4qlL=m<-}BEAg}Mg4 z>pc58N!pZ+lqo|H+H<$xT^GgM9pc=OJXGBvc=zh{i~+28XB6p5hNu%$4R-r5KXQ0v z8SSc@a$ztru2og0)1dmsfoZ=p+I;+~=j>wzpg2?#d-|#RS&dkO<}JpYZWO8Kro(vG zf6|}E)9J@y97$E~zNp!KLYx}&W8k}Bo`?=sKK)~Bs|(j}?I*Xfm;ROIUZ+_^l-_Yz z_a?@|?MCT>#sK}I1L$?L8G3i>Se;~v!^Ua;0L2eQ?s@3 zt~1!dY%i3Z>Zz}RPg%&pzhwV~c^fqj7^Y!E&1c~XGnxI5t+e-)8of8u^jdw{j9lHb zh!DB>Ip=Z2F`QiIpW6Et^5TCf=O}M<7Up`w%~&-&($p9}M1-kh^x*17OJ;`D86^`m z+WGw?{ZL#(XT1*v*>Mosop1;()qBIMV&P2UKD~l=cUjU@!?ZWm{Z5{q#b}f+*WK94 zD%Fd$WvbK^oZFPEjI2z1*WXZlM?*P&_R}Zf68C^WOv~$PFF9$9D3^WAT}EBnE6!r< zCKkr70|hZ1HMhGlF1G;9mPP(Fvs0hp-(a11PWbsVV>>&$8vDil;iP&UutO@_zhcEw zii-3R$I2<|8_BU>TYetQ3K^C-PB`G6z=USh9Xrw^56XKs0;jP^buw>zJG3pDyf8-f zB8T)^FHAI5Eew@V%e&uSaXz`-mlbPKez&5TS}ZZqb*tR4^8W4`w*xLuszlN2&+sX0 zaS&>!sEtofIFspJQYOW(Do$X;H7!{k$m=n=8W|FLslHntcd4IZ?e-_S4(HEJ^k_#s zzj#lESY}FczMEitJ?5HIvsOwql5`GOqb=^(hIPegEE zfOhWDDbJMCCWM*ab{EX}B92C0g)Sbiy5;_y<0*u)2KosC?S=tR)}YOvm#v9ux5Z?5 zHX!wYfBA34i2(5)2W!2UG6CoD{J)r@8od1#YKA@bs8%`Q3G1M{43oQiPs(C}3 zm9aN*p*Kv&7)E2zF<<2ylPSYf@1Y#-xS(=OuJ!)g*lhk*d`^XEXfZ_gi7yX!VK(|C zw!UZS!?qrScU0WcJ^t}c!2$9`E_fgzzeS(R0A$T^D{f5oY-n1>HBNLX>=||CBPGUa&+eWIE+nNW zEB?d)Eb07DjG-dKGarz{)pt6$DT+{kybfaDXCN+ZpPWRHYIBAV{QGnX+kH`=k;^rLE|L9xh3-yQp?W~s-p^5!}iSPxb9?u_;Np4 zNh0Z%Goho7quM>cqDbh_|Bq3=cxw9wh`~$CA@p8&NCMSk!{67+jrck04_gnVOaOg>QHEd&r*AtxPE9193mq`AH_$S;B zmW}O-jhEVGKXN9!a{B(-mb4XFwvm;g>RQ?w#qFv?R+tS|G;6qw=civT{AF=_bMFGl z<92OyEX4b!HYn2EKUd-VKs5g`jOQ$nIvHQ%*{7iUN=4GLtzIDeSS2OQ(;6#i8IQBT zp|ijTo5Is>)~yj+iPxUqYW?LUsg`CiaAqH!mOyo8IU6B@jBm`|t#Kt@gE4BRMn=`a zOq5BpjbJfEdo?~pktmV#eIpD=y6Zzzv;QIWUlvH)@mI>gBkISG_Y-46vS!vdc%a^o z%7{vQYIkbp?aW7@ZRNhexqGa{ni-R;msV0jGFTw>u!8W99(5Ekr(TVU^%{mnSyI)ov3wTY2~m?}+S|t9#j7{KH!{kpN&VGNyAome-O7 zK#I>KV!@1Jgz7;^INW zlOnveoVjpDHEbV-TP@wuo7Rh_I!n$jyi1V1OsCI06Z~!@I*ne`CKXF`lj?nNmeGs1 z;CTpMXm6u0_Es4+7>zv%hBzugPntOMaV^QPBT?E_#$y6Yf5rjne-D}PFkxSvjAGD9 zgyB)pE7-|0uaQDl4%);pJ{#(5E_+Mv(Z)z`9n0h?X3$hiZ2aIaN)z>ZUj zN~~cn24^jC0M7}2dPkHE(9-cZNz=mP*?f(D!`Z!B{g26&%1I8fA8LJEmu^y?n1_bG z?f#fMOnBbFsmBMd7}RdX3#%&O8|FWj#psXu_;KEb%*B185~X=yYY6vZrRGn^MO(UJ?`(RNzXCZl#tH zlIDF$n;03TpD>WQJ~biy`BzGJ>+T)e5x#%zbO8g5=4w#dJOiT31fme7<>)eX@4*#K z=F(^$a6%j#JnA#UJh$$ImK&dFbih>Y|Jd|qDuvmVi!HrZ`SAACTG zlD<@i8!fE$8nV2sP}5ud2~KxhDM76qkZG6EQ?mClb|23QYoYt^kqq;CM&h04A;4|* z1~YXAy`4jrhbW~0uU}_EdTUZ5SO^3>uaNau&l)`6Gj6s+nWBl)8Zx#{8(vHi+H>S) zijn-+S8VMV81uZCbI$`<<_h-xAF?4xs<8A>NP14J5^JQ$1BptDvtFdhURv~x`MK>x z*iyO(1c=Q~>Ca~o7|Xl2ejZWr zST6lZ`smiL&kk9uO*+Z|&eQxGql8L6mg_Y=aAs(kbJwv^;odNzbc>49LJmWje4zx_ zwt$;N(t39J3tB=DNoF;dl0)b-4>DWZpLZ=uC?kq^&-{xcgRmb78!GuSJzZ8t8qzbZ0paIJ*AVEn9r27|;XDBV7jbbAg*`K|sgx zz_bcvijr*9>`%XLD=x5=FU)M3ifd!eSrt;j`n7Xn3cNkVpТeQK^*YKpPvJ*Ug z&$p;O|3Gf=-E_hL_Ch2Wm{#8=ei%ks+>>vg`s{e&?;Vm^C4c?+$woL>WXNaysd|!S zl}lyDQWn+!>b_nXE4ngN!cxl8TxAQ3_OD$@zeBZ9G~bKv7l_Vru=n15eGbNW*S^rpkJE+D zNs3=sBs;$5MT=niicl=>fauN~Tj58F_EXGOs?vF;7j+66Ue zCgw@f}?m?tW{L3;tBC|~>0RZ^b{yV?^Gy3mLHMvBk0vfgEs*lP+L zUIU=Wk4hC7E!GMyK$?23Tytew4&;F`FfAlumBAQ(_%k1Ulrnjo#7lPc8JKZ*EeLrp z%368kr)Xv3cJE$QRI= zMQDAMazH<*ghyPsWCHNtH68|Wr#ZOw$U7lr_uAc=;`4NFQ?{b(uxm?)qCfTVBdOzu zHlcXf|E=QC2@y(d@(|suOhkzZ@q8h`+=ZYgsDnLq>XhPYiF|Z2sNw|)>{`MOmj7+d z%kr`7&Ep8(zZ34t-XA@WNE(bxH7tRVn0E)h+CUjW_QlJCo>#A4T^urfD!2%fl5==? z`rnVkh5l}T@+|Je@*=H9=%PIhaQ`HDsxsTVzCarCeW*;dy!x&*#AenIMfV<6r(TCS zYPq2h<%UxuV{=%Va+XlPd_NYavj0UrZ zr2l;Vs*}}Q3>`hkV|Z5t#p(X}S|~z&g9$talbRaF9|~{jm|R@*)k;(K84fIl8p*($ zmd>SGX#_r1GfBnbsLswtzch7nksKk};dwaE(xJB{ssB_!(I>lC(};%u*?Pgwq|P<8 zyK5ufn8V>L46yE@G$rk2GysvGK+&$GfO30-KH!|*Tj{5QdVUmEcT_$)C|`blIg2^C zwV(A@9)xnqD)(LjGWhqyhvH)GxxYUZtARQ>njMt8MW@=If9NCyJ_&-Op_#HXD%;p` zDBN!Ok5OEPI=ql>@TZ5p>jO-mToo%1S-5LneZ3u_d;7FR&XLQPAJOOA>;?;_o0OO# zDJ7l9HmoOQbSl?Eb64p)(w1MQr{kuSdM-eDeQwge%vfU;_>{xLy;`W0<$oT7JuX9F z((QRkKjdKR?eIz5-7M+9WsgSk?rX9)$@cuHoyoo+_XqH;&W1v&#vsKE^{M#M+@)s^N^@@tlAOboVQ#{-kHqlQ>E9xP_0;48h$Q!s&4Oy1^7? zR2dh|Z8}et=C$_yL))d z0_v8(Pi3`wKm5sg)?!;hKKFh+-HLzySwzI}(x{K(M*Y;arj_F8PCQ;Sm_j99fHPuT zcznh)1q%9~ivzOr8?!eFEIKtOk8i!IkpSqghSrQg(`M^m@seo_jt@#T-6$!jq+#txPPwD9Q(6^% zaO$z?gU^n-4%R6qhS)9=Cy$<$m%qbno4eb7P%pahgZEwAWh z+}U{F91#;^3b~b4!v_|mIAfyzm!JW{V92sCKXkK_SClp)LEYDP-RX&5hH4SiyZ4kH z&c==gmO`-#CdS+mzUXVltJJ|eRNmUk=haPb`iAQy+ExSHW?kob=kFsOs=u2Xw0vv@ z(c05I(;dnVJz^z!!@e`2-Dv*l$B+LAt9sv>;^Qx{8dU)Gt)odLmk$J}1g%2VLHGqB zBLmVa=R^A)#2~`ql@eo5x6M#!c@{WhQz&YmcMl|Mo;nr1NcedQo2`(eQ{XZrwaU2E9>?34vY}}XhX@0^I)4}6yEEID;@6-Gqi#tB;YtFOj>!7aGA1t4|(`IV7<*n$ZcLPYn zC?@kWlm&+XgG14CrdRJ3*r2GwS*e|$4m4BI_CHi9KCvz;GP|W*${W7{Kgth?MQQc^ z#+y;LN-2J>^g;oT-*yDup7htWz$u@cmGU>W7f&VlA4eeM=*Tv?z1}xB-e2L>48;mS zFIK=^G~*8s8B6_-Kgh)o@-lZdMnYqdAshhXQa1ryVw2}b=yijghh47!DMJ4Ju?O1) z!N11g1>O3D{X!wx{A$0{p2Y1X5_JLwSB6X<%K{$@_c7HV+tWh3_y{odzqaV;;doM0 zpiCnxnNu4`wHTG=|G|5@r1yi?{K<=-@huA~e^`Sv_|WZ@a3fx#ByC}oDAzX6(`Ry? zpz(MFeGX6$cL^|E9A8l2#dd%4%_fc~@>tI^b--+*W4!;pBXvdtTuU&QOf)VTlU080 z$a~>~@fQehgbT0>ADl47hpM8q93y}mpPPZ^SOYrkHq6UK6kUxBf!{C$pu*~!-3Rj@ zZ=_7`fuR(h%{(dwoWF)(dLh*C>Q1gD@&3Hy;&pU8s-8C|W84$yuse6&8sp8y+3|^V ztDgLWlg9hM)l>g3QLz7i!WH~~%1r;?xh!SG@5 zkEC9j^7vf!*Le>O{5(i~cVx-^s0iAOoqoI+t^YeOubA-<7zZU@>8-Dw1;O;105Iee zLg?*DG}lmWf5;VxWE!X1cq|6q2P}I1s;+>95Od3A)mOV8Y;E!i^_cR~h=hK6Kd@A8 zVBt`fe0I<^%DyorGy9hngiM#FE;33Qou%5E-}Va!@qjYvwP2mh{BMG#)Dw-eLrdy#A(zi|2QSIPenB zkL`va)>j;HR2KSFIvuv;E!DkeUfIeaOlH4C03FcBlKY><*fGQZ%hI_Uw{8uZ#u0{! zM}3Y63Av6>0=(*nDK~sdw0-y(G(Vq zP}an&&VvqirE3+m2ctBdBOqD5?}LGw?|Oga_>P+UU#M z@|iN=%_Z4s(Xio|rXiHm!@rId+ZQ?qZ9ds4eQ0>>&YL8Br>M?(!IPrVSmHu{okQe%NudymQtL!t3G-R@eMw@ub^VgL zP;MNM0dn8Gc@xC~5*En0zJb^%7Md!pQwG`im90pZ(Kq#)Tw9*ab*YNsA!t62Gq5;i z%nvgTt7h=xXv5vEkr?{4K4FYZ@S1i208y1(3&6b*S0a4m39cYP#=MkD@?Rj$#~lW0 zU>z4o9(3qZQXij$B*S$xLeZl7`}HH$X3trm*q8~Y#ec+s{Zu!&{JJQQs&+GR4`@y` z9L^dUF%ypbcMx>shKG^rVqSKPId$tA-?J+I=IJ$KH1lJE+2cA zc(l{ft62Y&K?`Ui$Sb!adT+yrG@`mv>5rkeI)A4=-9OOn#}o5_TNN<5;x|O+zC#FD z-}9@y$@k}HlS}_xlf1H1;FQo#zCXWQdnp;h@5Q~Z`o|xCT)kRkfA-6cq&WYou-lE$2I#Bp2-D0&(4=QFrpC6nas99&#nyhn2=_Y>2u z=bFjP$zf1`QBp&kCtj(^Ds$G4sjfXQOXz8wOU@8;wtB|q;zP_QxSSG0I9At1Q5PpU zJwuyIjxp{>OSS0hBMwtriTWG2{K@ut#RN`p(DpOToY3T31wN7Qu?BVcY}e8zP3_f* z1I?=|F`~)c%A{&*dk^f)D5cf@%e#GR@p2d?k~<3zBOjFsc+@R4eX&m)>k+!vv5j8u zH83`g7b+vDwb;sMY>u1iu+xJ(cIVVydp`0;7d5dlV5u{;btdKa?jTBfLbq~=J}7;p zx?%#ah=hLcyZ9X!?$ake*&%)`ZOFwrt&!`Az}{z)1G?YyQ4@8QM^$1p88M;k2yUX- z=4c6_qauS-=gk{px#QBoogt0SF@Bdz+3)plvP@Cj2)DN@FT~m(>}30Nv?Dq~uDiun z_0(q%t(D=o8VuTBE@HgxpCT*F=bKQ>(eoZnbQAn$NzrN%BBqC;8R83b3FDyN^0FiE zL^&5fie@3>#D<23M(twAtMB=j$BX?#u3Wj&qvq)_5ZHe^VY3 zG*UH~8+U>9Qsh9{FQBGE!*GYG3m)4WyL0$=?b<*7mohmJIc!x`F~M?3K_=7Ctp3#C zv7n%DQ*Eo^@F^Oa)jEt3JDdq=Yjt-<`%0T|Jdhy7+S~T4pY7FPW^ALo$OR=O-81cF ziOZ_Ib^E?NjL&JOd1E#^CvTD*GyHBA{k^zLsCaUre$B00XRHklX*ZdRq9Df(!UAEc2-$qFH#_PBVwK#Gus;F3ETkbD? zETAPlbzWTO6%P<8%CS46n~oJLC@${W$)crXS3KQ($cK^As^`q*>}rzZP;&YipJGLw zh{xmZH}bp1{rz}XE$eOVarscM`b2S2^w^zgo4|_dCf0IE*)7r976$&w<->B?obLw3B2dJrHgZaX$bCr+M4@(U}}SyS+vbL;(#n6gGmE&Ot> zy@;0U@KQfRQUW6xxp2&W{hPX>rDcjJ=Ud+9R17+q%^<0_3A3KLHMFJo&*Dl?zcWq%HjV*t$DB@N>piZa4_4nEGH2!a4|6JR`Lba^A26N zGH;vBNpZwWKDqvA*(q)i3%AcNC@^f>U$9 zKXaMvJ)xR6`}${1EMlBN8V^C(X}{Fkp2BZW@iS*Db$YweqNCE<1_;F2G-6S%DG8Gw zaKoDXNKBi#|K;nT#&mATH>ck8PV|No{UEd0Ovc~hhbw;9826w_co+Y~*_;xOdq$G2okVzxrs@W^2`FYQ2KAG)NGxV=j#n8X0!5xXKktB3VibuVy zw)ezlq9e}wx_!4I@dlTJy4j|mw0f=4bgb-#&s>)-5^47=-j68s*_m2eam}uYDRAE5 z933y#{$zKVxHg<-J7YE8CVjr=QjTkWDs&G0U_Oo;++a=FFZlM3Gs^=LQhjCm_+PIv zdRq&}yeI1O5^Y#4t@ImMw~M|jk4J!6%MjK|Hi`H`=hL4TY>@b*9VV5&o7E|0RRc>Xd2iRcm&wQd97!K6EVJQ=Y z)~}95OkL72`5{F%(xv zn^(^cN!UZ7ENCu?;=L+KnK31ZG!3rlYtHYTC*dldbUNo{WLTmZMO`VxTg>_bJi1AXX5nj0m9N2>`WW?Y%a-}uAN+FYh#yTmhz*uqh)Sd z%jXg8C?;DGTDvZcpB6ciL(-CFdRd9=6e>`=)h5_d=T!;igt;C=QV@_uMP*c@|~n2VH1 zO&=Q@8^>_3)N?s=(tUjU$IB0k5yhM9_u`rr_CDCHnW_v3eJxI4zU&ljfA=s_Gyx~*9xbHB#=gYDJYMV(naK=dcRzRJ9^i!zr|+O4Nfy&X)Y-rEeoH>v!g z#VR)Rp5ABb2*z{Z>qc^Na_Gw5Hp+*+Jx+bSC;4a@k>@T}T}Ti|uHj2oBQ48ZkCvJm zL%pT}r!aSh+~6mT44_RQ3)-yx+f_${2BT$dwANp+eOAI8b1N%nt6AUEMLX_fsJ&@f zl!M}WN~_tOJF#VHXj}Y7*tCwgcY^QtzeT&btmWAf%^iM0S}d%VGVi=yii@zVFk4#q zTc&fZm+(SNKr=M*oQSqgm1A$a^fW>jx8Ob+x~5BNQrPNN&U5Uak(4|dIZw*d2mkt~ zv9Ym^2Ef~zWp7tq9GVo>A1jUh1xncy-Yh34r%&Mfy4D5G50qH?O{}HsR$i0d>3f|$ z-^)MKRdwzrc6@t4^nOvvwfpe21x3E{VuGa(iBowHSX`w9V{)!Va^&E8D(R* zcmajL+ftwuq?pjq&;XH(?L@Gv|8V|%{K+f397}43Wfmzv%(vQPM=Gl}+(JOOoH6D) zi?Z}EXKb=rOdW_J#zI5={QQ(iZUg@GJK!Ipltr-cuX6jby7rZdm8mkPJN0YNerU_?_+Ov!-xm1?3jf6t{%^kI zf7eTftTi4C=~kHCLZMEEJ+)of8+JHSyCQRBsQJgLnw&opf21J)MqyE*fw6h)!a^Zz zi!Z%=!uXe^wCvkXpWbX;K6JIU-cw6;!q`t&Q8aV#@b$TP ze5SlpS>dfKnXkqg!Xyt1HUv8r$?gmhNd7>#y&?}GyX4_te!Tu&2Mq0abvg4S?{`YS zjhc$VtLEgKL`tQ1{x*u5gRkj7FK-oYcgI+yY_D4S{OHM;fPe(j~gg@o$FS0cc)h#T-hB| z=8B;(B7^s-J4~+Xjyf0)Yq#wlA>~V&9{cah-{iUt2!8E3eb*!FD;KYn$@$McNV#+< zMJn{@!S5xG{LpZCU>xgKwYK{|dJg{|KI#9LbNj#RNd1p5y5eWb|KNznShaGM(;XD{ z{<{a9`QdJ4xN$p-8aN-q7#Gu?#H0GF9Ak_Nd4BZ4P<%Gs`pfp-bKMY}Yh%ZK2mA!o z?%!@CnbmpT6G3m#k`!j?OL&(1sAn@>J>*q)S2EpXnZSK-WNd5?>0hUXYf1fOmK_}@ z)mxy&-h5~kG+U#MwTs@o4}SmK|NNE*X-Wd;=a#Klj~B6G1W_MHG`%y$(7gVu)bFT8 z(qwN|3`awIb$o5wdpIetv>YE8U}imM1nF{AWnnEpTxq%5f%38stpqiB4=~ooL&l@4 zP0glG2)Vc4ZJk5lhR*}0_WSYAm@qn%1_;*}dTYKyx>(eLsn=5y5TQC$)2hh|-9rys zUvAQS`f^1!Iev6ugT=E1)luQ8r5|A6R%@#7D$$#>nOOWC4+cG^pP_d;z$|kr7<hvawLcP^r@|lfR^T(E5-5qi@(i7N@~j=q%>s@8~B8i#_{n27w_L5&M%mM z(x((1urR26-fyfL;*irq9v4+Kf_+2Zb`DIi40^_0z#|KJ-0B>39^j!4^kL!8nFg0V zz-#low-VWRItN|8Hn%>Xbiw&%-UKW9EJTfLdO?ug!r=ORlisdyO6X zhf3NgEh}6}eVL*~kVgeG-M!F$xWc!Lv6=DJD#XlGxTtJRteBiOOGR(svq_VCOQfgh)@ zCLcm;x?aURCR^BR9l(}{Bsi}d{Zwq!H!jL&y)UDSPpi&HDq-W&y`a{p5cw>O@)^4} z%|w`vTkW*=cCF`As}MiZCq$>PsHl@l*hF265s6DWpAzoX{1llk7s@#w#AV^mHPwRt z&I#Y|vD#P>jo_JpEDzTjaz(FZ#`zu<7}DN(F3Y=TG!LheTf)p4uQxt>Sizx-yxHx6 znOyK=ZM5|wDe$4IIw+ZJa|}E zdGH_QWws7Isqx&m8BnHDnIY*}PFFdTgoU0zxwx`ha3#B>p&l$9a z#q{kj;Y8UrL9%zJ+-isPw9mXaCDA4X*`NJfR4a1`o${7G{FPSD=(wsarF5}Ge>6!f0 zWPX$h^a@7p9IWOK%G&)y?u;u z3rXDuFUv<7zAF!So(5y%_EUb+#l;eWO*+QiYPp`NpYs=UOUS*0i4VD&(x&FvGY;^g zd6E*fyWf?B#3B7f6ct=c>m4|oGX(Em4a`mKTN=tzt|Y0lnu)Ita96x=Vdkq&sngL? zj4uJT#$kG-U5#8=V-mE|2^u4H2AhC@xbm=UKxA!zWyERL$e|)@N7dA01d~(&3Qe#? z-w9^V81|gC?$ksDQ<_D=?7Nqvmz!w4ImT_6kndBl3n$^+xUba+s`=iCR@weseKbsy zwd51DTqnBYC50?@L*E_u0MAD{CKeeA252~l6Yt3@_M0iUPI-N4v zEvt+m7T>=GQ0}>gZxIfEQ|D5eBtn)qGmC2&7D4_?Yu%+fA$}$-9!?;sBBJK&c`bVU zWJ(i7Bgxb<<+@0a40ogLkTy14B=`a^xvw2_`3HMHhOG6VHSg!j@=_}k z{+}4eU0>b{&2~zJp7YSlu-W6cwLYIFm+`$`3Fr0Yp=h@$VYS8JbHJB}tXZMS4V&e9 zocuS`7Z3g#>AQ%^=gprpANx%G>X>-5>UUB6RmLc#d2^AC=?vGb(@7y&B}gQgNp4rf z2xAWswm17ZtO=aG=2jo-2V@hY;qPh~!t;m_YJo(Uf_R&7fYGpJ0J%S;k*X%8PP?ce zfu5}vYmd|Cs81u(cxXXQ0TnXOQu8`aw}(|S1Sh4{(6ZR*NNZwtrH=Y2xQ#dxaGHNF+r>Cm_&vg{@Zw{=d+OQ*qEhw#3ZM2`wJgYT)Xvoy|m~!GSRJ(%1TgO z-Ych~-bll5Of-2cmUxfNJ4$HJTP$Dj4!81rZ=N$Y!lL2=@ZVd?5=?ZoHolyu?ipdx z7vm0DY^olUwSXC8XfM@lusnjeIK;h$-Ckw}B$W1jiX}M$uDAB?XMW0? zNI8A%+klV%=JXVJv$Q+7>KE>Pa0OVnRF29e&`?$8^|{O@k50FWmN`}CcqCR#ne6M@ zqfUFSw`s|vjTCPsaH~~Jq%*kVUaIt$hb%7!uDpJ%Wif^h5vI--6j@ri+kJi=WihBh z8!k~}Ej2mjX>?V&qph7opr~W{b`tOb*=w z3Z%DNg*145+|19SBWE7(+6ODw*lSb2RJk` zfbnx9c3MMAY?6nl^_V6z^}Iz%6?*WFsGMyWGW&mU0g#j2caOU#gw|N5+ z`X_QtdD3mbLCZ44*MtReRgWLhI!%MQeYfhfx;Dgh-s4`^6B!8f=4@r_SRc9rdafs% z0)ORtwF#4DtUM~$@`bDD!%iX9I12jEvI!*DUJEOwRe!mzJWw zfN`%z6H(9lzH34+Yx8PXL}WXCdYfKYVmO`soJ~pwStd2@uc;cIQ`HtUy#*I%Qc_zr zY~ktv##4$6yZ4_o;}Ik5KaYf%dKVsFvg!G@iU@2Hzg z7KN&Yyz2Vr$CbSkA^tbXbaB*|pGn#J853aNO?W}g`JRh%yqGq0sr+%(TF+?!(>4bh zps2Y?;l|~qqtAdDb>pysyLPeKb2+sh>EhadP}JymQO)9(h_NHcs}qUJ3BQ%|fl=kK zw`A!fQ_V^qwOYKy>_pvrgJ^f_3{6x*NSB&xne|wy<>%mkCru#isb5<4NMOW(4Rgbf z=5|N(QqF$Hhp2oF?{5nE%vucyS+UYrSU*oIAvzH(-kR@SKv6U_?C)Pnbz0G&BvE ze9^OfbF2NRrLQ}RUTblOIomyd{wC@1E?M=u5~#2}6>`75dRf3!0ojz~l zkor+;x%75cFoz|_pV1C*%M~v-LAZf;}bpCq;&TZ7fQ5f0ZrRADAhP|RulP>HGZp*-4ph-pCRa4 zT=D$0Y3dO_grMNj#EE>DD;rHS^85Df>r|;+)PS0$fJX2wXKt!buhUA`qwGz^8~w$4 z+m-9E-z}~>rFg^vwK0yNepL4;#p<(L4U;c`B(6?-Yp664=(LMp|VKx>Jq8n>$j(3n6nW4nPXeOu%r<%mx8Q* z1_WtoFY$$ig_R_Cbw^mx9*j+>WY+bVb`9nQO{4XCO6x3Xl7q(6OF9!sJs#I5`(&&L zGuXAN&Ysf%Z~WtFbuB0n4nsoiq)lm76JTZGH@kcpsCi<*O0QB-f2AvL-tPUcA6sS> zp%Rwrp-}uA^pJMvB%-paxwD5fGcvDOATkLviX(o+JN4_>HPR5VVNn+S=fX=O0?`^f zJ73J83=PkLmy!#cXk2=0sHZwBoyLfId8AL@k`gkBZ_=R*La5cP%)TEsSilE2Z-Nl< z?${#_yA!(gN`H=mn;3CBvT3NVRdsob6k4V015Rd+G--1pUyJim>wMY1oD(8uw(*Oz zi3rfu5gvlU>H`frgH9Y`99OTx%x$gSLyjUJ;7)m&{jHc)IdO1`sv}3hj>6=YI3G;erdg0g*E<^?b^~`E^e`ncD>~+q>oKI0jpyf1re>yYA^|qzDmR;*Oxskq{bd%^)W2e8{}$-QsWH z5QDtFU3AHdElKD!p9tTp5ppiw@X(oC(Ap!@-9HxseFcnhJ7)i)^0=o?BgvgM`PH^! zq?Dfoi}4^n1r`IqBUdF720gyl`U1AKpzwO3n(x*snAfTX^>afO$~k~DgFD+@bn zKE_>Umb%ado8V15~_C)U9fdn zym%>nn`kH-%^Jv9?#)~HELNl}q&#sAUXdj~buz5k-9uMGiTMO08g1q4Arnn($#s7Ob8N2Cczklu?8 z6s1U&4g#Ts-h!b7MY<4r3lOD40-+_85V$LV=l7j^=l*xjoqK+BGQ$knS$prb*IrNi zJfHQf2ow@2nX<7l9+Wla)P^>^9Lb3QUh2q359v56sc$DO!g zNUTJRALV=yg&OYn^{ufkWQ}arV&&Hb>}9u`%>ne`hqa{{>r{V1bWel$e`NQW?edw= zMo((W`(6(N0h&e_6|>f1wFR(b#i(0?GzbzRIe}Gxxxg2b!S)nZ_)@ebm2+`sJMToP zRZa9qfm&(6kZ!e>alX$WGaKOZ`v?I|CEVB*so3sva?Gc6y)0*JUB}Qq&VPI217{0RgyBG_ zIO{70r6-zD@u(f*D`_CYZES?OtfVM&8Ok@#8^-D8VGv%9z|%?~v`S_szwleHlF|SZ zFgiM#>YggiEFy8)pI9*$2f@}(*NH^p5Q&1Z*xt}xt;4eY+E`|c8FrB0o9t#mY+uNP zT|-a+OD$ruR}vC@1wckwjgo44DrO;FyaTK}o?cD6hT zz!MzVS~V^AdI4O5l`aa4+)zCO`BnHL{@#wUVe3wp!`cJ8QcV~sep2PNgRxdXNAdS{K$2iLug#X~mHA9qzi zu$);?oe$8k9#yhp!mu=HP%Zi}34jh$WGo8SAVglQ8W7g~DA-O7QZMNmMU-~brmhXE zh>s4>7dr^{6l3K{9qr;+16n3R<;0@ZD_0ejLZ2Co?&Mp+lo4BR)^Ai%1yBXm_Bbzo zyD%dwcspi%b5rB~9Ecue0J!uFPH-(h$!Nl7oONA%#A8($Z9_W)LqQNB?L--GqDzJ_D)E8uPco>U1yMpeCM zsargDibcKGe;2}1Zmufxjo9flZue2?<{JK6<+SVi;*>9}#NtGm1!M|vnE)u49La~; zU3gI%)i*~43lX)R`lQkIdcat1K{hHu9V&*dZ+3+h{9?9FEa_bl#JnhVEkNxIQ^Q32 zdAp4wey4^3L8#7ZH8mb{2F#EtKGV%-2~9h0V4|d?j5_(sRL?Bt*UCQctn--&R&x6D z&RX2t7v!IRh531C1L$IKJn>2TqiH0&Y2+D>3ZA}0gt7G1MJ2bLuGa+aLc7({rUOxlhMl)BE%4zAu!l<%+lF?K^zqgRI1B7ywGxG zvPIv$=8z{EmH;zEbWkFifD0;OHlCsE@9gEZzL=zPn;AKxO##jTh+RTJF!wgExRC|W z8&=(pJ5P}6_cED!4E z-(^0iudjaWphSMuy}G?b@&9Q;RrECfXI%D%;s4B+{|zStyf{)-vKZu+htKY-fIL|I z57(>Y;3uPf`)VL{RqWk-li${tYJUtX(nA!^%{Pe289ew^FO;jE_$1v;p)h&WGJ!7u zkVSZT`64)@cggJ3;dpAfMgj!n=>h(mmI+MzjE(f*-F690YQ~>NPoJOPNbS6J0@M9H zGDrG+ZZ4<7yCG@_OOy9DhN*!2V`)1qFEW%7@J0Fn1LTQT&WXJmL>tRQA}%Yuqa>&*;BUz z2cT3@MIeqkuAfG1vd+z)%E`-pjAk1h+x%WI&H2-EwlCcqjHHGsHhkVT_Fw859@b^n zQ(K`N?yW!Gz0=at$?*pjpY{Et_p ziny;@1=h^5(xj)fs{0n-cm*%+%%#p#D(P5*NZUG6C^u^})Oc<<#O{}u}{ zR<;GGBY^&u3RX(Y=@>^pB)+q{=gs(MpEn%G8wfK80?+b$x%&^Z(BD zq5r`z{D0>uQ5i`jF?)6#iEB&)gA`Y=eCl?b6TN>I$1Dl-P14nGyv5A)DG?K?@%)~Ha-v1K=;!+ z`|7`j0MM-)Y!nL6J`pjonv&^wy9{s$9z+$UrGl$Z37mo|H>L+p^Z)vS5_|zsMvNn! znIeZ)S%ke!R({yM`{a33;h|G!^PBN4VAbd3Er|<#O+^M31NI1#HubHuJqUU=z(p zHYgbEx?SFrNRT|p^wnVnl>40gq!iCpTk3KH{Zba$J97ht4*kaNcXdxwmEhBIohLU7 zM*6;|03VIBW$BNh%kpCbFB?TUfg1aJjVgI+5k4;^@)Ankx4ni5w5h60I(aA$k z-w^lpgN`Jh^CE`rFi99fbdsxeGq<-w^I82&%}sQAj(#Dg6<#ImVdW#YgPlc6ix-EN z4`eqZ!U|i+w>{ml37~$vHSq1h;Jzlhzv7Z3ffq-|B6_O^c=F>&MA zYuj3PRA8c`oOxyKK*gw2{S8owF}hhtj?WqVJ7kA}#z6|dh(NR3l{5i0QqLGx0KHfG z(ziMM-Vb7*5g^(M`)r)g$;nYjQ*ZwHAyPvZ@pO}$szW;kH2w%1c^Mjy<=WE?2ttrq zTl0Kr!NHT=>E2d__El5)!_Ne2@(bWWy_PU#PYB@bA33&$204DD*PZG&@iNMw|I}1O z0ULEb)Y;0{jj8Hli*HR_KQH`3U0XmV7fj%wsmhJ)Waiu1yx_1Ckrhxc?+_|;G_#^@ z3)TtP!jJNDy3c8caP#xRuJdJoFZHl)j6qPvrWs9exLNvgI1Qp{Gqt5ySn_q*gsTNj zFkLF(Wzy6|Tmc?EP{0>EQPj(@+)7=pj|K~$c$lchxAOzAGE2DQ7q}D{FquK|ICyw| zpw4uWu@|Sl&uJ_@HKCIsWQ6ddjON2&FfDhKjR`o7rqfYl{B~}>(xOLo@z3VZRc^TP zu1*t`b$n>k1sKU#Jrn?S7X5jRSF|ftOFWp!fD-{^6xep^$?)UANJ7?L_y*Irgw`W1 zmyovgdA7$24L`F6xhfT>Et?c9PIn~5a%O6`6;1MzV4KvpeYXiL37T!$y zuFH@0;za+e_2ZA2=dGWfMOdWowASb4#xEN8)EFi48}x46(&c*gT%~85+VfQNfAIFA z-hOfs7D;0N;Qz-}*`6ub#ml@DlBG7<=d;o)@J1+ZZt=qB#+Y<>efEg09>@RZ@T9fe ziQqPUh7MfLGonxsI_4Ey4TMrHazI}U6TWVKs=_eRmt?=aJ|pW&+$(pvvAZzp>7zxg z*Bl!Mx*< zLhw;N-AG5G`@f`-dmCF$&qo&DY2Db%4Bd#BSQdJn$+J zCrHmxEXJ;C_v!#{GN1cCQ2GR<> zfvDtCX3u#Qq~{`(CN4qaNOtv`&zPNr&-%l>G>6@5mqTvO#rTGqt@~xIBeO}h^Klb9CB2iI~_zhM3Rxqo_tqm-e&-mu2!&g3`UT}SAn*F=PC>BWb? z-^#EN$q@V$2~$BN8zSz-?^e4EG0BhJW{Y)Oh;k#u(4NXjlXr#kV8@o>3>Di3){A@o zA`OjAmGojjV=mE;lBS&rV(LJ)M$w$jg zorJ$!09z7V=h7#UCJ$n#>#Ork*cwyh%6Hp_v#`K(NaC-BA%#K$T$mVi^mCxjZQ_W4 zP3yABsH5dweZMA$({WaMaW0@{I9^si%zyRL+}Dqpd3i8g<3#r=S}_bFurL<6J^amD z$$jJo{l&@Z7*go&40ndE$>$33d3ITX{#>-~LX)|?(k{hfy>>6IprOg26PHlYWDG4Y z^p!?&epRW$_#f%X!hV|Hmdah5wEJUK@2cPhb=1Rd8rydRuJZSMkNd&g8ShdaXkUVQ zvKI=3@m>~TQhVG)=g3`C;bwSWS6aP?(r(S6WmN594k>glz zwAFZlvbBoJo10jEyGA~GwcSr5Pf%^mW_rkkzFEqNaWx@}ti7XeG>}Z_`P5Ub070W} zH^A-uSbMjUbc$qVLf&8hRDyB&b!RlI$+TqnXLaRG*eO-0S1x``&_1{5tPiTGGlyfL zb7#i6p0+9PrSC6w1RDkFCdc=MvDHYD@K0F5rmMQ)dSkhViWbg&L)7WSfVWpjx>Dvt zrM#&H!$?!_&6Mul8Yq7(!{?n9EvHk_Jssh}#}&h5uiB5i$d|Pb9R4-8k)>9)Gqw?1 zhPSdCWmFA^A~T)W4^-Btdtc)kYh6F9VH4v=Ql&48h`F2T#f})}v$r$Y*gb5^H{T+P ziS905HugdyTIe6r%OR&Ust{X^#x3>+wJyP|+5Sbfo419)M7w>{)Fl|es5xYYI4}1Q zVmyY6va*$T-M7DVyIVyB&uwZr=5y-On);id!8St#7mBpjLt3Gn7R9D);|4Hog*E-P zzV5`0(VBzTkU`bX6ao>1Bfb%J8_@6XVT~wve^;qGt=gvN(h^LqygoeJzK) zcctab74gh-?d2bAM%bpuH5ax_^r3~`V;+TDM0-CNy9oxAJ*d@fBN0!k!?^1rQ{RynpdGW~(n+6j>9$Y^rvBU{Qk29)HfWC-B@L$N#(R;H`n{DL@~=E4XuJ7C1yA#r$GR_;#mZfs z>x(sHOGh?tx#JjautZm@2;LX#>YH$1_uEsmG~pWm;F&YB!_o*Q_&(x{x0u5dEq@=0 z-j|F$h*l0StIvkIs){lhhHeR7_{{fXybik|574ZHt|Mr@Hl2hocje%*jSFV5q<^u|SGMno zdLhjZ$h*D`fgW-xqgUDh2$N&mNs!v1HIrk;C4U8j>o7^wbtW;~ zZ>cGq;aEFI0RmdZx-@l+?%RCm?6AFCs<>yOE}MIgy}5n!Hn39j3OeuE)E*P^#fXJr z3p=Y;N$v)zh8a9y8^g>_edf#llT7udS@rB#OrBc%9Jv5s& z`Z~J3!W!<7|D5F|VbFRWe)=dz3(dudcQo}7|2pMb+T9dYsojg1%iD`pK|W56VG z%+v~!Ocdiwl=c0prNzlA$r&9BmY^CIqDX=_q?*QdX3a<|UlEy+sxgXT^VQ2rijg=< zpI}B?VZ=2vn~6trC47?Bviman*gMcB-Uj09(H`#_kw`o2$XD&Bain z9TaDq)cWV2s^D&H)Jex$VWUROM@2LJy*aa2^7+?aF>C#fhzoaPFC#7Y3vTgr-nZHr z{CEaJ^tv|b&@uhLeaMAX9WH)NJ zlOA>igfuQB~{30_x zh>Tx->!uw)tZT`OvCj5SPJ6;taNasmyaL|cnxR(#|h))V(jo z$4Kiz$0{vAd%&B*c&KoNM-zd=e(Px8|AVZsR-{O~|0g&!u|;^z4%_ zFyr^-8s07T>*DYGmSivnR=NCm=_mTi&8aH0!)0S2ahQb|&ZBR>pVNKqqTw&on{uGoX1^q-h`L39hmR=p6+Gk(%awH_ zHkEL%de!6z9J2Yey8jweo&Nh)w zn>Zs!*+UjU*c^HlEdN}#?3XXMu)o<5U6-?EuIqo(y9AHFsi2`k&;RTxT$X(vec<`K zD5KR_q*^kHq09nvl=3I``Kc<(uptApdGLi<0H;TC`SwjBA}<-UW+q>=<(#&&haONd z4A8G9Zgrh&C;b$iq(|tqAm9^^O-iT(M-GmH|LJ^?G6_)}i_^sq9_V~E>y1~pn zu>G^R#C{2|Sx$i0ylmivyRSPF-fGvznc(BT`Z(W5*D%Idu-A{JrqgMgnN_!vyfbYer(Xs^^B&0TbtcXy-e3dP2RRjN0?95aq2&#C9O-Q!#Ilgt2L zCEcoz3l6EucU${Fjd+H$|`CY*1t17Ew zQLz)a zuk--7-k(9Ujwc3LdHCGs{@_Yk?vrqSmom7$S#DqxJ6%!qx8LGUo2K-^4eBoo{JUf~ z+j*7#Dmh*Z#{%{EkesvgOwqSe#%1~g4Hw7o3Zzs9OvQlWdcD_z4#LGzM3 zmkZVie)I#Uu^Y@(S7jyFCE)XTt5;S&6KtA4`1VNW+F%-VRIJpUXe=9{&XZ?cC8Nt@ zQGR&{w zc=)h^Z2`o5OLwHC3?(FnT01&yJwE+XXU{nPraU+2#H!$Y2P{X3N7F2C!*yLhb{ zlaf;S`n8W${+^r#l*Qcm=3sYXP)h%ng>6?Z@UE4gFOa?fwJE8UaK(jS5vVkuFs0#` zmS(Ob&0;#;4MfKB_(slM@1GNn+PVWg7xTa>K@CwBZD}?&SN-Q42n>JW031veAb3;L z)6ENRS;(9F6Gq3ve>e?KEeL10N zzO%Nsml5^cx=Ar`N72x@X!-UVrckU(%1#f~ zxUxIlxvb#GMSW!O2?)(ad)54dHx~T&#N!BCxip{#_kW?8&ECpG$dKDDY}kutCM^^q zY;)~2j7}c>nwmaeI<|!ukIeL~OHN4OQ^?K9dBS-=VZZnTsJf~3`SAlWF`YV-gF71w z#O^7r=xFj^Z2<;7XX}%=tgNYk?r>92e(QU8b-m|D@#J-O^;>!g~>$lzA zITC^YCEjECpl)%UtgWZTqcB`kb*-WAZ>wRr42J(EZxoiG<|vlL!ExA4f#psr3TSgc z#{o6byqHWwZ4Ao|Cti_`Vd-&WED za&LD~!Y3S@sTfGN=%^2EJc^Gv#}HKRJTpW4?tVlDkP@cz=bLB12E~8N@6C>0`~04l zZ(V$(Q}o{x&q0qMFoKskGjq`Z#<*BMn#_BNZDw+khg)1;|5d=YU&;v7off`t(P#pa z1=+&*E{A5{KBuAL8UhyCv6%4P*GQ!ed*#CcY!%_w^pyMoEoIx3H0n;T$0~7?t zN>*W|y>1boQ$W}31K{#@H=D<=(mr#sg$O#^bn78Xjy%d$I+!dPjknr=g{F!xD$Wu< zK8r_*Dt}>c;iwOXsV(7x71*MgR?QEjc})h?osmUe$(3*j)G$8Qi8F?9rC!SN)HL49 zSc6PYzOe99Y$abBc_y&T%UilHlO|gP_A_6~>e!d|eU4Uu3CF}?Qgp&uB(cSSHBW&$ z0fSnA0sn(5aT&RFAC&f`e%5@LnTi~M#c|)gTgi}73V_3%rO}&K`}@Juq&Br>|AXK8 zcJ+trm3G#=VuL#e@&++({0AQ<@E(}2=6~UcCgSYt=eceB*Q7yeJ^BX^EVSs1iq+4< zaTXm309|VkRNeo%(;29go%M&sTCre;*6uCU0qnGvZkea21C`rNSyWw_X)b?3iRL}! zF<(rluCoK5GNs|;_>VdbmJ#Sf%q^n%gj@T^I~jpStbN7-NqnW;=zcJO5N*&Hhv=zi zZnp3V+V66wCbgltEX6F~|JE?*by+NpFf-O{|B7o`Km$9!(Y#W+%(_3`ch#av2OK!i zWbXZc6~p~s>Yq#J2&5vGyy<+&U+>mWd^=U2d*{oY@!O`7Ls{m>sx0GSJ8K4dVpmJy zZTR!0u#LV#wUzRux61a|$j;ypNBe^JS=oQJ+;geJBxppW<@4zwQj_n`>io#seoQ2xs9O&YIEFMX!d^v)Yl4_e z%|{+CttY}tD37v2eF=GYaeTy=&e-6&CbhRN*xk3pum+)M2i0s?*8@teez&=})m(1r zwsFaBH9ncbv_zI%Pdsoxo|+%?<`cLx1UIzOqbNh;Si7t2Bpm{spP0&l;Ugl%y&vH& zaQEAXiTz^ncZ?$@nmv{!+r|h;x5z+HvhDB-V;^}g&yDyAy`lATqgj}_v@dBiE^)~a zUvZUpEUxn#q<1EpkR-;HY0D}X(w!75m1W$536O zT0K!RWa%`5i~;=3_XqW>*jk=cyAGma1c^;p4Ag+QZhn5Ua%~*#td-J*U3#pZ@*+fB zpAd#^>vF8mqsXkisMI9;2;K~PPGKl~#8BLr;#6(;&ete6Pmwb}HU-zss%tryAskzh zFo!I4(!5W_j6|)_e*SAx5|Yp_SHm#1v_h55omHsCo~1h%KX*`Uw1Y@JgW_3Rt_v4x z6v$CNAQNMm(nu^t0bIp?6*Kno-rdp;5}TW|9_p)*<}Pvf>lRsx)bJk6QDXTG8Bb~3 z%I0z5sI@Upn|?vNwV}mH^x6Gcco!pt5<7ZWX6}_W{>oPMGL+u!PNdh$os^xK3jFmy zNQ-xH5P0!davP=x4o-lFYQ2>=K%QZ4r58<__gl`9`8MA#L}xQ>Q+plvxO-M3-7HBv zA>r5(t1G&vT!CQyBYV>BzB{_L8&t)3-aR-i=b!+dQAuzlE^=+;_cb|_ur7-jRM#wo z3~?J=aFGo`>`rT8&Hf-879y^Ha>8Yjoe~m{vmw^95}aKqn?XUi^My^FVYT?4Hv?AU z(&9yRT&Ti1Jh73`zSY*5a_2y{Y+eUIj!Hx22!$VQ+n{`PVsrpZf4)nv7If|#@pt)7 z;1#j+;>7KU>>YEc-s<_rb2$kL3#%8AyDU{Tt?~BD8T$R2YPRtss%){B*ODbm=8guW zZ&gjf?Hm|G;J(ypF!N&fkLiH7-cO&md$E?dtXs-R!DsvN9a zv@dMw=)}cus^7|C`W0FDZ7rphC-;z9oASP}!vKCiZzO~9PVX)+|)e zBcK3+X4xh2`Nlh%Y7q*GcOEe~;tFL;Itt$ot#}_OgWn(vlmuQX5hdFqGcJj$xq78X zIoXCy?Th4UKMe9K?B_y7cb}SjL%D}*_0Z!A_^h)RB8HHgZv-#OWI(*U2Ve9kd zOm4o`qO|@Oe7i}fjx3bh`QOhxlIKe=pv(Sa(+MK?zat5rig9?%`3^^9^Owlr?-qm4*02WbgPJJh+-uA{tu zK+XNSLACyr?0F1CJ@^#9o`TMFyE4YzhRm_45mYH^^0TOAolo|E=EN6gc4Wa_*xzR0 zDE|ASZ%#U^`M(pQ8E-}Wcu|;T=M{>mNZAa5ZW{nmki#}ZHTi)eX_`z!1sNC>F1K6D z$K1h99Y9m5lrP^+tXGrYCYzqsu1mS>6Uyfb&p;7JR_ISI$PD^~FKD?o_x3nExHRlz z0IYv@HX z(3kC#&i{G8wiuuN%n=3DL(}QfbqSwT_*Kq=+-P37^x${WQf1`1wgpL*el(0-%oar%M&Oie_i5T;_rTbC0iP*gTGO`vIA$kiP?czAY z7g-Zgt;8^9bR zEd#~uy*d&;yAdBdz?)1N=Gqu*<`eAO-|r6nx=EQ7y!S9PJo5bX@SGj0Hc$YPzPVH` zgc6s~Gbt8Ia}GsVNvA+Mdj2gMFjXI1`oKjV(4GEu%KO zD3O+t@RLDMHTlC>%mkmpQ3+%6UcqCaXo^5CQ$_j~dhv}gV0j!_H446bwauNizv}vV zaYG8mG`hN7iG5J&qW(bWWYih9Z@7+NL|gt_{}vXhjhYQDK0R6cC{~(XyN3hqUvATh zjYw-n^b?emZ}|7=`$4vn@-jNT30E}*t1tB!JKvA@k|%FXATrX)ISg^gl?#1dOL0ZK z44kH%7iF3=v-}eC+pZ5Uyc!Ae6~Q?9m^*4#opu=&{0k9q0{14~11|kFq^g>iufpKX z2!v0m{Bw+99KKND`BP&xqPT7&u?O`JrUvbzDvnjFIw+-pK13O{Mscqy0udlpeo%c87Lr!BnqdPP;Pyl56@8v@tOyYP``bj zVtCSl!m@eCjJKI2q297UV?4)&1UAE7a{Nm2URd&dk+WJ~ng4@rW@Ms9viO%`>6@!I z*sFY9qe$@r^-*6*c{;b8>_T`NVWXB;^N(JWS3Kh9M2wY6ap57>JfU)*H>^db9woX3 z$S|IBpKE>mRf2vH>m8nBw8@_{#4zJqi|n>*HBGim4(Ae0Z+C6ItLtd>sQ>o#B0$6z zDy#OE^8v^Pdao||Q3d`3u07iArT2aPwjk?8%;&(VLz@8b>aAcK4f6JmX4Oma%acEX zgYqD)44ne1p;gg`$-dpX9VPW+Q||W4h**m$gybE z4IBU_v&x*%d_5FLfpIhpKzzBQ55bsANjw~DDx1g?GTVvGIkDM1BlHXJT3 z@wAY3xXJwGlb!%?qa1m`(OB=FzV-!0O(!-ozSgRs&a#f8wXHXa!xUeaj^*~C$@=3M z@=feB|3x(v6ovgN6X@q@>{eQeHx`&1*NhLXDHYzls0!&CdiWQ@YEM7z%?GFebKSeM ziI*%Z9^QFoOX7RtJEN43)~h9cFleuBp*5~ylPmQP&uoBCcsbY`X@7qk$}-zO=?De+ zm+n_Aniod;)_n1KpWxSTPHp51&DWGjOZ@U%ro4y7nSp)8`1s8f2?Yy!bvp}!n`=`a zWRCiDr{0g<`X|1^cF4QAm%r35;YHPWX{CU4gXFu9r|3aaOtgijqt})6s_otYI@Nl)X5Dsqz7phm zdn(PK#xK9>`j+B}mrR#7W{#in_57-4u9*KFz4h8qa93Ja(C^=#EmlX%O}Zq#X^>Pi zTJZvoNcXN`yw3=hkunv})E{@fDmHQqCUZK5wU2U$eu z-K86(%$h9}onk&v=Bn1@U3i-JrH%T%)xtQ~+Q|49KVlf{50aDLL-wnK@ucRH(HYSf zsa=yl9`IaaE)k@?Tu{QBR*qqmtoRxK(8TEZY)9Nq;6NDL@5LoQ602~ZEPB{&<4SDc zX(zD}(c5#Bn;~K+t5NHEAv;_3m!sDZ6=<}ODDFjJ1`K2B8(OGPw=*L3#CJm4&KRYJ z^b9oq9Yb5Vvvlom4F3BZ|1+cl=Wy+%4*6BXf0DmQwozfW$#b-T$pr{v{iTGT=fDQ1CB>CFGqDed{LwBj^)j~v z4tLqEN*IqD?pHAz6>%buy*-~FoIC#1>G@a}WdjnZ$~@Qg!4hhZN$C8c_AEEg*4tqh z1mmK)Kl2ay8gIE+8?onw*{px>N3d)C?!LhxEeRXU50*R0Z^+&wc|7H_CV>J|Lo6I% zkh20{At3Vd-@*5~=JnuU-qHM~`FoUxhT&k{Ao%Tn{^1PYI_Slm0-C$>OaF<93d@R+L&Go;P}sj zk%bE8qa&PWn$-i%d=H?bfMg!f7$1-}$GDv%a3tW!UTW|+?SYfv1p_zWnhRD~dwERw z4Ayvmbk;1G&lJTnRk;wO?gC`2lR;PQWOm*ij-8JO5ZdLIFKf6)8VCmW+G zT_;2x{sTk<2wQ|p)31OPC*fR+)B!4Z7aF%czQ0(724easvk%0u5Fg|7SEa&XAo?Ki z!_^{d^}Wsutvm;jbNF2BM<)dvOlpT7RT0BYf#pa1{> literal 0 HcmV?d00001 diff --git a/pyproject.toml b/pyproject.toml index 1311cceef..edc3575c5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,4 +80,5 @@ zilliz_cloud = [] [project.scripts] init_bench = "vectordb_bench.__main__:main" vectordbbench = "vectordb_bench.cli.vectordbbench:cli" + [tool.setuptools_scm] diff --git a/vectordb_bench/__init__.py b/vectordb_bench/__init__.py index 018cd7732..c3822a64a 100644 --- a/vectordb_bench/__init__.py +++ b/vectordb_bench/__init__.py @@ -35,6 +35,8 @@ class config: K_DEFAULT = 100 # default return top k nearest neighbors during search + RESULTS_LOCAL_DIR = pathlib.Path(__file__).parent.joinpath("results") + CUSTOM_CONFIG_DIR = pathlib.Path(__file__).parent.joinpath("custom/custom_case.json") CAPACITY_TIMEOUT_IN_SECONDS = 24 * 3600 # 24h LOAD_TIMEOUT_DEFAULT = 2.5 * 3600 # 2.5h diff --git a/vectordb_bench/backend/assembler.py b/vectordb_bench/backend/assembler.py index 6b0e3c81d..e7da4d49f 100644 --- a/vectordb_bench/backend/assembler.py +++ b/vectordb_bench/backend/assembler.py @@ -14,7 +14,7 @@ class Assembler: def assemble(cls, run_id , task: TaskConfig, source: DatasetSource) -> CaseRunner: c_cls = task.case_config.case_id.case_cls - c = c_cls() + c = c_cls(task.case_config.custom_case) if type(task.db_case_config) != EmptyDBCaseConfig: task.db_case_config.metric_type = c.dataset.data.metric_type diff --git a/vectordb_bench/backend/cases.py b/vectordb_bench/backend/cases.py index 6f4b35974..6c43bb910 100644 --- a/vectordb_bench/backend/cases.py +++ b/vectordb_bench/backend/cases.py @@ -4,9 +4,13 @@ from typing import Type from vectordb_bench import config +from vectordb_bench.backend.clients.api import MetricType from vectordb_bench.base import BaseModel +from vectordb_bench.frontend.components.custom.getCustomConfig import ( + CustomDatasetConfig, +) -from .dataset import Dataset, DatasetManager +from .dataset import CustomDataset, Dataset, DatasetManager log = logging.getLogger(__name__) @@ -44,25 +48,24 @@ class CaseType(Enum): Performance1536D50K = 50 Custom = 100 + PerformanceCustomDataset = 101 - @property def case_cls(self, custom_configs: dict | None = None) -> Type["Case"]: - if self not in type2case: - raise NotImplementedError(f"Case {self} has not implemented. You can add it manually to vectordb_bench.backend.cases.type2case or define a custom_configs['custom_cls']") - return type2case[self] + if custom_configs is None: + return type2case.get(self)() + else: + return type2case.get(self)(**custom_configs) - @property - def case_name(self) -> str: - c = self.case_cls + def case_name(self, custom_configs: dict | None = None) -> str: + c = self.case_cls(custom_configs) if c is not None: - return c().name + return c.name raise ValueError("Case unsupported") - @property - def case_description(self) -> str: - c = self.case_cls + def case_description(self, custom_configs: dict | None = None) -> str: + c = self.case_cls(custom_configs) if c is not None: - return c().description + return c.description raise ValueError("Case unsupported") @@ -289,26 +292,69 @@ class Performance1536D50K(PerformanceCase): optimize_timeout: float | int | None = 15 * 60 +def metric_type_map(s: str) -> MetricType: + if s.lower() == "cosine": + return MetricType.COSINE + if s.lower() == "l2" or s.lower() == "euclidean": + return MetricType.L2 + if s.lower() == "ip": + return MetricType.IP + err_msg = f"Not support metric_type: {s}" + log.error(err_msg) + raise RuntimeError(err_msg) + + +class PerformanceCustomDataset(PerformanceCase): + case_id: CaseType = CaseType.PerformanceCustomDataset + name: str = "Performance With Custom Dataset" + description: str = "" + dataset: DatasetManager + + def __init__( + self, + name, + description, + load_timeout, + optimize_timeout, + dataset_config, + **kwargs, + ): + dataset_config = CustomDatasetConfig(**dataset_config) + dataset = CustomDataset( + name=dataset_config.name, + size=dataset_config.size, + dim=dataset_config.dim, + metric_type=metric_type_map(dataset_config.metric_type), + use_shuffled=dataset_config.use_shuffled, + with_gt=dataset_config.with_gt, + dir=dataset_config.dir, + file_num=dataset_config.file_count, + ) + super().__init__( + name=name, + description=description, + load_timeout=load_timeout, + optimize_timeout=optimize_timeout, + dataset=DatasetManager(data=dataset), + ) + + type2case = { CaseType.CapacityDim960: CapacityDim960, CaseType.CapacityDim128: CapacityDim128, - CaseType.Performance768D100M: Performance768D100M, CaseType.Performance768D10M: Performance768D10M, CaseType.Performance768D1M: Performance768D1M, - CaseType.Performance768D10M1P: Performance768D10M1P, CaseType.Performance768D1M1P: Performance768D1M1P, CaseType.Performance768D10M99P: Performance768D10M99P, CaseType.Performance768D1M99P: Performance768D1M99P, - CaseType.Performance1536D500K: Performance1536D500K, CaseType.Performance1536D5M: Performance1536D5M, - CaseType.Performance1536D500K1P: Performance1536D500K1P, CaseType.Performance1536D5M1P: Performance1536D5M1P, - CaseType.Performance1536D500K99P: Performance1536D500K99P, CaseType.Performance1536D5M99P: Performance1536D5M99P, CaseType.Performance1536D50K: Performance1536D50K, + CaseType.PerformanceCustomDataset: PerformanceCustomDataset, } diff --git a/vectordb_bench/backend/dataset.py b/vectordb_bench/backend/dataset.py index 2b630eae3..d559eb6be 100644 --- a/vectordb_bench/backend/dataset.py +++ b/vectordb_bench/backend/dataset.py @@ -33,6 +33,7 @@ class BaseDataset(BaseModel): use_shuffled: bool with_gt: bool = False _size_label: dict[int, SizeLabel] = PrivateAttr() + isCustom: bool = False @validator("size") def verify_size(cls, v): @@ -52,7 +53,27 @@ def dir_name(self) -> str: def file_count(self) -> int: return self._size_label.get(self.size).file_count +class CustomDataset(BaseDataset): + dir: str + file_num: int + isCustom: bool = True + + @validator("size") + def verify_size(cls, v): + return v + + @property + def label(self) -> str: + return "Custom" + @property + def dir_name(self) -> str: + return self.dir + + @property + def file_count(self) -> int: + return self.file_num + class LAION(BaseDataset): name: str = "LAION" dim: int = 768 @@ -186,11 +207,12 @@ def prepare(self, gt_file, test_file = utils.compose_gt_file(filters), "test.parquet" all_files.extend([gt_file, test_file]) - source.reader().read( - dataset=self.data.dir_name.lower(), - files=all_files, - local_ds_root=self.data_dir, - ) + if not self.data.isCustom: + source.reader().read( + dataset=self.data.dir_name.lower(), + files=all_files, + local_ds_root=self.data_dir, + ) if gt_file is not None and test_file is not None: self.test_data = self._read_file(test_file) diff --git a/vectordb_bench/custom/custom_case.json b/vectordb_bench/custom/custom_case.json new file mode 100644 index 000000000..48ca8d8c4 --- /dev/null +++ b/vectordb_bench/custom/custom_case.json @@ -0,0 +1,18 @@ +[ + { + "name": "My Dataset (Performace Case)", + "description": "this is a customized dataset.", + "load_timeout": 36000, + "optimize_timeout": 36000, + "dataset_config": { + "name": "My Dataset", + "dir": "/my_dataset_path", + "size": 1000000, + "dim": 1024, + "metric_type": "L2", + "file_count": 1, + "use_shuffled": false, + "with_gt": true + } + } +] \ No newline at end of file diff --git a/vectordb_bench/frontend/components/check_results/charts.py b/vectordb_bench/frontend/components/check_results/charts.py index 7e28d1e66..c2b2813b8 100644 --- a/vectordb_bench/frontend/components/check_results/charts.py +++ b/vectordb_bench/frontend/components/check_results/charts.py @@ -1,19 +1,19 @@ from vectordb_bench.backend.cases import Case from vectordb_bench.frontend.components.check_results.expanderStyle import initMainExpanderStyle from vectordb_bench.metric import metricOrder, isLowerIsBetterMetric, metricUnitMap -from vectordb_bench.frontend.const.styles import * +from vectordb_bench.frontend.config.styles import * from vectordb_bench.models import ResultLabel import plotly.express as px -def drawCharts(st, allData, failedTasks, cases: list[Case]): +def drawCharts(st, allData, failedTasks, caseNames: list[str]): initMainExpanderStyle(st) - for case in cases: - chartContainer = st.expander(case.name, True) - data = [data for data in allData if data["case_name"] == case.name] + for caseName in caseNames: + chartContainer = st.expander(caseName, True) + data = [data for data in allData if data["case_name"] == caseName] drawChart(data, chartContainer) - errorDBs = failedTasks[case.name] + errorDBs = failedTasks[caseName] showFailedDBs(chartContainer, errorDBs) diff --git a/vectordb_bench/frontend/components/check_results/data.py b/vectordb_bench/frontend/components/check_results/data.py index c092da3a0..1e6bba00e 100644 --- a/vectordb_bench/frontend/components/check_results/data.py +++ b/vectordb_bench/frontend/components/check_results/data.py @@ -8,9 +8,9 @@ def getChartData( tasks: list[CaseResult], dbNames: list[str], - cases: list[Case], + caseNames: list[str], ): - filterTasks = getFilterTasks(tasks, dbNames, cases) + filterTasks = getFilterTasks(tasks, dbNames, caseNames) mergedTasks, failedTasks = mergeTasks(filterTasks) return mergedTasks, failedTasks @@ -18,14 +18,13 @@ def getChartData( def getFilterTasks( tasks: list[CaseResult], dbNames: list[str], - cases: list[Case], + caseNames: list[str], ) -> list[CaseResult]: - case_ids = [case.case_id for case in cases] filterTasks = [ task for task in tasks if task.task_config.db_name in dbNames - and task.task_config.case_config.case_id in case_ids + and task.task_config.case_config.case_id.case_cls(task.task_config.case_config.custom_case).name in caseNames ] return filterTasks @@ -36,16 +35,17 @@ def mergeTasks(tasks: list[CaseResult]): db_name = task.task_config.db_name db = task.task_config.db.value db_label = task.task_config.db_config.db_label or "" - case_id = task.task_config.case_config.case_id - dbCaseMetricsMap[db_name][case_id] = { + case = task.task_config.case_config.case_id.case_cls(task.task_config.case_config.custom_case) + dbCaseMetricsMap[db_name][case.name] = { "db": db, "db_label": db_label, "metrics": mergeMetrics( - dbCaseMetricsMap[db_name][case_id].get("metrics", {}), + dbCaseMetricsMap[db_name][case.name].get("metrics", {}), asdict(task.metrics), ), "label": getBetterLabel( - dbCaseMetricsMap[db_name][case_id].get("label", ResultLabel.FAILED), + dbCaseMetricsMap[db_name][case.name].get( + "label", ResultLabel.FAILED), task.label, ), } @@ -53,12 +53,11 @@ def mergeTasks(tasks: list[CaseResult]): mergedTasks = [] failedTasks = defaultdict(lambda: defaultdict(str)) for db_name, caseMetricsMap in dbCaseMetricsMap.items(): - for case_id, metricInfo in caseMetricsMap.items(): + for case_name, metricInfo in caseMetricsMap.items(): metrics = metricInfo["metrics"] db = metricInfo["db"] db_label = metricInfo["db_label"] label = metricInfo["label"] - case_name = case_id.case_name if label == ResultLabel.NORMAL: mergedTasks.append( { @@ -80,7 +79,8 @@ def mergeMetrics(metrics_1: dict, metrics_2: dict) -> dict: metrics = {**metrics_1} for key, value in metrics_2.items(): metrics[key] = ( - getBetterMetric(key, value, metrics[key]) if key in metrics else value + getBetterMetric( + key, value, metrics[key]) if key in metrics else value ) return metrics diff --git a/vectordb_bench/frontend/components/check_results/expanderStyle.py b/vectordb_bench/frontend/components/check_results/expanderStyle.py index 9496313e8..436eeec38 100644 --- a/vectordb_bench/frontend/components/check_results/expanderStyle.py +++ b/vectordb_bench/frontend/components/check_results/expanderStyle.py @@ -1,7 +1,7 @@ def initMainExpanderStyle(st): st.markdown( """""", + unsafe_allow_html=True, + ) \ No newline at end of file diff --git a/vectordb_bench/frontend/components/run_test/autoRefresh.py b/vectordb_bench/frontend/components/run_test/autoRefresh.py index fe31d8205..034ab5017 100644 --- a/vectordb_bench/frontend/components/run_test/autoRefresh.py +++ b/vectordb_bench/frontend/components/run_test/autoRefresh.py @@ -1,5 +1,5 @@ from streamlit_autorefresh import st_autorefresh -from vectordb_bench.frontend.const.styles import * +from vectordb_bench.frontend.config.styles import * def autoRefresh(): diff --git a/vectordb_bench/frontend/components/run_test/caseSelector.py b/vectordb_bench/frontend/components/run_test/caseSelector.py index 49b839163..58799deff 100644 --- a/vectordb_bench/frontend/components/run_test/caseSelector.py +++ b/vectordb_bench/frontend/components/run_test/caseSelector.py @@ -1,9 +1,13 @@ -from vectordb_bench.frontend.const.styles import * + +from vectordb_bench.frontend.config.styles import * from vectordb_bench.backend.cases import CaseType -from vectordb_bench.frontend.const.dbCaseConfigs import * +from vectordb_bench.frontend.config.dbCaseConfigs import * +from collections import defaultdict + +from vectordb_bench.frontend.utils import addHorizontalLine -def caseSelector(st, activedDbList): +def caseSelector(st, activedDbList: list[DB]): st.markdown( "
", unsafe_allow_html=True, @@ -14,41 +18,49 @@ def caseSelector(st, activedDbList): unsafe_allow_html=True, ) - caseIsActived = {case: False for case in CASE_LIST} - allCaseConfigs = {db: {case: {} for case in CASE_LIST} for db in DB_LIST} - for caseOrDivider in CASE_LIST_WITH_DIVIDER: - if caseOrDivider == DIVIDER: - caseItemContainer.markdown( - "
", - unsafe_allow_html=True, - ) + activedCaseList: list[CaseConfig] = [] + dbToCaseClusterConfigs = defaultdict(lambda: defaultdict(dict)) + dbToCaseConfigs = defaultdict(lambda: defaultdict(dict)) + caseClusters = UI_CASE_CLUSTERS + [get_custom_case_cluter()] + for caseCluster in caseClusters: + activedCaseList += caseClusterExpander( + st, caseCluster, dbToCaseClusterConfigs, activedDbList) + for db in dbToCaseClusterConfigs: + for uiCaseItem in dbToCaseClusterConfigs[db]: + for case in uiCaseItem.cases: + dbToCaseConfigs[db][case] = dbToCaseClusterConfigs[db][uiCaseItem] + + return activedCaseList, dbToCaseConfigs + + +def caseClusterExpander(st, caseCluster: UICaseItemCluster, dbToCaseClusterConfigs, activedDbList: list[DB]): + expander = st.expander(caseCluster.label, False) + activedCases: list[CaseConfig] = [] + for uiCaseItem in caseCluster.uiCaseItems: + if uiCaseItem.isLine: + addHorizontalLine(expander) else: - case = caseOrDivider - caseItemContainer = st.container() - caseIsActived[case] = caseItem( - caseItemContainer, allCaseConfigs, case, activedDbList - ) - activedCaseList = [case for case in CASE_LIST if caseIsActived[case]] - return activedCaseList, allCaseConfigs + activedCases += caseItemCheckbox(expander, + dbToCaseClusterConfigs, uiCaseItem, activedDbList) + return activedCases -def caseItem(st, allCaseConfigs, case: CaseType, activedDbList): - selected = st.checkbox(case.case_name) +def caseItemCheckbox(st, dbToCaseClusterConfigs, uiCaseItem: UICaseItem, activedDbList: list[DB]): + selected = st.checkbox(uiCaseItem.label) st.markdown( - f"
{case.case_description}
", + f"
{uiCaseItem.description}
", unsafe_allow_html=True, ) if selected: - caseConfigSettingContainer = st.container() caseConfigSetting( - caseConfigSettingContainer, allCaseConfigs, case, activedDbList + st.container(), dbToCaseClusterConfigs, uiCaseItem, activedDbList ) - return selected + return uiCaseItem.cases if selected else [] -def caseConfigSetting(st, allCaseConfigs, case, activedDbList): +def caseConfigSetting(st, dbToCaseClusterConfigs, uiCaseItem: UICaseItem, activedDbList: list[DB]): for db in activedDbList: columns = st.columns(1 + CASE_CONFIG_SETTING_COLUMNS) # column 0 - title @@ -57,12 +69,12 @@ def caseConfigSetting(st, allCaseConfigs, case, activedDbList): f"
{db.name}
", unsafe_allow_html=True, ) - caseConfig = allCaseConfigs[db][case] k = 0 - for config in CASE_CONFIG_MAP.get(db, {}).get(case.case_cls().label, []): + caseConfig = dbToCaseClusterConfigs[db][uiCaseItem] + for config in CASE_CONFIG_MAP.get(db, {}).get(uiCaseItem.caseLabel, []): if config.isDisplayed(caseConfig): column = columns[1 + k % CASE_CONFIG_SETTING_COLUMNS] - key = "%s-%s-%s" % (db, case, config.label.value) + key = "%s-%s-%s" % (db, uiCaseItem.label, config.label.value) if config.inputType == InputType.Text: caseConfig[config.label] = column.text_input( config.displayLabel if config.displayLabel else config.label.value, diff --git a/vectordb_bench/frontend/components/run_test/dbConfigSetting.py b/vectordb_bench/frontend/components/run_test/dbConfigSetting.py index ffd52721f..8f4f35c93 100644 --- a/vectordb_bench/frontend/components/run_test/dbConfigSetting.py +++ b/vectordb_bench/frontend/components/run_test/dbConfigSetting.py @@ -1,13 +1,9 @@ from pydantic import ValidationError -from vectordb_bench.frontend.const.styles import * +from vectordb_bench.frontend.config.styles import * from vectordb_bench.frontend.utils import inputIsPassword def dbConfigSettings(st, activedDbList): - st.markdown( - "", - unsafe_allow_html=True, - ) expander = st.expander("Configurations for the selected databases", True) dbConfigs = {} diff --git a/vectordb_bench/frontend/components/run_test/dbSelector.py b/vectordb_bench/frontend/components/run_test/dbSelector.py index 5fcbd8c08..ccf0168c6 100644 --- a/vectordb_bench/frontend/components/run_test/dbSelector.py +++ b/vectordb_bench/frontend/components/run_test/dbSelector.py @@ -1,7 +1,6 @@ from streamlit.runtime.media_file_storage import MediaFileStorageError - -from vectordb_bench.frontend.const.styles import * -from vectordb_bench.frontend.const.dbCaseConfigs import DB_LIST +from vectordb_bench.frontend.config.styles import DB_SELECTOR_COLUMNS, DB_TO_ICON +from vectordb_bench.frontend.config.dbCaseConfigs import DB_LIST def dbSelector(st): @@ -18,17 +17,6 @@ def dbSelector(st): dbContainerColumns = st.columns(DB_SELECTOR_COLUMNS, gap="small") dbIsActived = {db: False for db in DB_LIST} - # style - image; column gap; checkbox font; - st.markdown( - """ - - """, - unsafe_allow_html=True, - ) for i, db in enumerate(DB_LIST): column = dbContainerColumns[i % DB_SELECTOR_COLUMNS] dbIsActived[db] = column.checkbox(db.name) diff --git a/vectordb_bench/frontend/components/run_test/generateTasks.py b/vectordb_bench/frontend/components/run_test/generateTasks.py index 55f3c8399..828913f30 100644 --- a/vectordb_bench/frontend/components/run_test/generateTasks.py +++ b/vectordb_bench/frontend/components/run_test/generateTasks.py @@ -1,17 +1,15 @@ +from vectordb_bench.backend.clients import DB from vectordb_bench.models import CaseConfig, CaseConfigParamType, TaskConfig -def generate_tasks(activedDbList, dbConfigs, activedCaseList, allCaseConfigs): +def generate_tasks(activedDbList: list[DB], dbConfigs, activedCaseList: list[CaseConfig], allCaseConfigs): tasks = [] for db in activedDbList: for case in activedCaseList: task = TaskConfig( db=db.value, db_config=dbConfigs[db], - case_config=CaseConfig( - case_id=case.value, - custom_case={}, - ), + case_config=case, db_case_config=db.case_config_cls( allCaseConfigs[db][case].get(CaseConfigParamType.IndexType, None) )(**{key.value: value for key, value in allCaseConfigs[db][case].items()}), diff --git a/vectordb_bench/frontend/components/run_test/initStyle.py b/vectordb_bench/frontend/components/run_test/initStyle.py new file mode 100644 index 000000000..59dd438e1 --- /dev/null +++ b/vectordb_bench/frontend/components/run_test/initStyle.py @@ -0,0 +1,14 @@ +def initStyle(st): + st.markdown( + """""", + unsafe_allow_html=True, + ) \ No newline at end of file diff --git a/vectordb_bench/frontend/components/run_test/submitTask.py b/vectordb_bench/frontend/components/run_test/submitTask.py index 26cb1ef70..f824dd9d9 100644 --- a/vectordb_bench/frontend/components/run_test/submitTask.py +++ b/vectordb_bench/frontend/components/run_test/submitTask.py @@ -1,5 +1,5 @@ from datetime import datetime -from vectordb_bench.frontend.const.styles import * +from vectordb_bench.frontend.config.styles import * from vectordb_bench.interface import benchMarkRunner diff --git a/vectordb_bench/frontend/const/dbCaseConfigs.py b/vectordb_bench/frontend/config/dbCaseConfigs.py similarity index 78% rename from vectordb_bench/frontend/const/dbCaseConfigs.py rename to vectordb_bench/frontend/config/dbCaseConfigs.py index ed101ac69..ce8a3a4ae 100644 --- a/vectordb_bench/frontend/const/dbCaseConfigs.py +++ b/vectordb_bench/frontend/config/dbCaseConfigs.py @@ -1,43 +1,147 @@ -from enum import IntEnum +from enum import IntEnum, Enum import typing from pydantic import BaseModel from vectordb_bench.backend.cases import CaseLabel, CaseType from vectordb_bench.backend.clients import DB from vectordb_bench.backend.clients.api import IndexType +from vectordb_bench.frontend.components.custom.getCustomConfig import get_custom_configs -from vectordb_bench.models import CaseConfigParamType +from vectordb_bench.models import CaseConfig, CaseConfigParamType MAX_STREAMLIT_INT = (1 << 53) - 1 DB_LIST = [d for d in DB if d != DB.Test] -DIVIDER = "DIVIDER" -CASE_LIST_WITH_DIVIDER = [ + +class Delimiter(Enum): + Line = "line" + + +class BatchCaseConfig(BaseModel): + label: str = "" + description: str = "" + cases: list[CaseConfig] = [] + + +class UICaseItem(BaseModel): + isLine: bool = False + label: str = "" + description: str = "" + cases: list[CaseConfig] = [] + caseLabel: CaseLabel = CaseLabel.Performance + + def __init__( + self, + isLine: bool = False, + case_id: CaseType = None, + custom_case: dict = {}, + cases: list[CaseConfig] = [], + label: str = "", + description: str = "", + caseLabel: CaseLabel = CaseLabel.Performance, + ): + if isLine is True: + super().__init__(isLine=True) + elif case_id is not None and isinstance(case_id, CaseType): + c = case_id.case_cls(custom_case) + super().__init__( + label=c.name, + description=c.description, + cases=[CaseConfig(case_id=case_id, custom_case=custom_case)], + caseLabel=c.label, + ) + else: + super().__init__( + label=label, + description=description, + cases=cases, + caseLabel=caseLabel, + ) + + def __hash__(self) -> int: + return hash(self.json()) + + +class UICaseItemCluster(BaseModel): + label: str = "" + uiCaseItems: list[UICaseItem] = [] + + +def get_custom_case_items() -> list[UICaseItem]: + custom_configs = get_custom_configs() + return [ + UICaseItem( + case_id=CaseType.PerformanceCustomDataset, custom_case=custom_config.dict() + ) + for custom_config in custom_configs + ] + + +def get_custom_case_cluter() -> UICaseItemCluster: + return UICaseItemCluster( + label="Custom Search Performance Test", uiCaseItems=get_custom_case_items() + ) + + +UI_CASE_CLUSTERS: list[UICaseItemCluster] = [ + UICaseItemCluster( + label="Search Performance Test", + uiCaseItems=[ + UICaseItem(case_id=CaseType.Performance768D100M), + UICaseItem(case_id=CaseType.Performance768D10M), + UICaseItem(case_id=CaseType.Performance768D1M), + UICaseItem(isLine=True), + UICaseItem(case_id=CaseType.Performance1536D5M), + UICaseItem(case_id=CaseType.Performance1536D500K), + UICaseItem(case_id=CaseType.Performance1536D50K), + ], + ), + UICaseItemCluster( + label="Filter Search Performance Test", + uiCaseItems=[ + UICaseItem(case_id=CaseType.Performance768D10M1P), + UICaseItem(case_id=CaseType.Performance768D10M99P), + UICaseItem(case_id=CaseType.Performance768D1M1P), + UICaseItem(case_id=CaseType.Performance768D1M99P), + UICaseItem(isLine=True), + UICaseItem(case_id=CaseType.Performance1536D5M1P), + UICaseItem(case_id=CaseType.Performance1536D5M99P), + UICaseItem(case_id=CaseType.Performance1536D500K1P), + UICaseItem(case_id=CaseType.Performance1536D500K99P), + ], + ), + UICaseItemCluster( + label="Capacity Test", + uiCaseItems=[ + UICaseItem(case_id=CaseType.CapacityDim960), + UICaseItem(case_id=CaseType.CapacityDim128), + ], + ), +] + +# DIVIDER = "DIVIDER" +DISPLAY_CASE_ORDER: list[CaseType] = [ CaseType.Performance768D100M, CaseType.Performance768D10M, CaseType.Performance768D1M, - DIVIDER, CaseType.Performance1536D5M, CaseType.Performance1536D500K, CaseType.Performance1536D50K, - DIVIDER, CaseType.Performance768D10M1P, CaseType.Performance768D1M1P, - DIVIDER, CaseType.Performance1536D5M1P, CaseType.Performance1536D500K1P, - DIVIDER, CaseType.Performance768D10M99P, CaseType.Performance768D1M99P, - DIVIDER, CaseType.Performance1536D5M99P, CaseType.Performance1536D500K99P, - DIVIDER, CaseType.CapacityDim960, CaseType.CapacityDim128, ] +CASE_NAME_ORDER = [case.case_cls().name for case in DISPLAY_CASE_ORDER] -CASE_LIST = [item for item in CASE_LIST_WITH_DIVIDER if isinstance(item, CaseType)] +# CASE_LIST = [ +# item for item in CASE_LIST_WITH_DIVIDER if isinstance(item, CaseType)] class InputType(IntEnum): @@ -53,7 +157,7 @@ class CaseConfigInput(BaseModel): inputHelp: str = "" displayLabel: str = "" # todo type should be a function - isDisplayed: typing.Any = lambda x: True + isDisplayed: typing.Any = lambda config: True CaseConfigParamInput_IndexType = CaseConfigInput( @@ -146,7 +250,7 @@ class CaseConfigInput(BaseModel): CaseConfigParamInput_maintenance_work_mem_PgVector = CaseConfigInput( label=CaseConfigParamType.maintenance_work_mem, inputHelp="Recommended value: 1.33x the index size, not to exceed the available free memory." - "Specify in gigabytes. e.g. 8GB", + "Specify in gigabytes. e.g. 8GB", inputType=InputType.Text, inputConfig={ "value": "8GB", @@ -157,7 +261,7 @@ class CaseConfigInput(BaseModel): label=CaseConfigParamType.max_parallel_workers, displayLabel="Max parallel workers", inputHelp="Recommended value: (cpu cores - 1). This will set the parameters: max_parallel_maintenance_workers," - " max_parallel_workers & table(parallel_workers)", + " max_parallel_workers & table(parallel_workers)", inputType=InputType.Number, inputConfig={ "min": 0, @@ -514,7 +618,8 @@ class CaseConfigInput(BaseModel): "options": ["x4", "x8", "x16", "x32", "x64"], }, isDisplayed=lambda config: config.get(CaseConfigParamType.quantizationType, None) - == "product" and config.get(CaseConfigParamType.IndexType, None) + == "product" + and config.get(CaseConfigParamType.IndexType, None) in [ IndexType.HNSW.value, IndexType.IVFFlat.value, @@ -582,22 +687,24 @@ class CaseConfigInput(BaseModel): CaseConfigParamInput_NumCandidates_ES, ] -PgVectorLoadingConfig = [CaseConfigParamInput_IndexType_PgVector, - CaseConfigParamInput_Lists_PgVector, - CaseConfigParamInput_m, - CaseConfigParamInput_EFConstruction_PgVector, - CaseConfigParamInput_maintenance_work_mem_PgVector, - CaseConfigParamInput_max_parallel_workers_PgVector, - ] -PgVectorPerformanceConfig = [CaseConfigParamInput_IndexType_PgVector, - CaseConfigParamInput_m, - CaseConfigParamInput_EFConstruction_PgVector, - CaseConfigParamInput_EFSearch_PgVector, - CaseConfigParamInput_Lists_PgVector, - CaseConfigParamInput_Probes_PgVector, - CaseConfigParamInput_maintenance_work_mem_PgVector, - CaseConfigParamInput_max_parallel_workers_PgVector, - ] +PgVectorLoadingConfig = [ + CaseConfigParamInput_IndexType_PgVector, + CaseConfigParamInput_Lists_PgVector, + CaseConfigParamInput_m, + CaseConfigParamInput_EFConstruction_PgVector, + CaseConfigParamInput_maintenance_work_mem_PgVector, + CaseConfigParamInput_max_parallel_workers_PgVector, +] +PgVectorPerformanceConfig = [ + CaseConfigParamInput_IndexType_PgVector, + CaseConfigParamInput_m, + CaseConfigParamInput_EFConstruction_PgVector, + CaseConfigParamInput_EFSearch_PgVector, + CaseConfigParamInput_Lists_PgVector, + CaseConfigParamInput_Probes_PgVector, + CaseConfigParamInput_maintenance_work_mem_PgVector, + CaseConfigParamInput_max_parallel_workers_PgVector, +] PgVectoRSLoadingConfig = [ CaseConfigParamInput_IndexType, diff --git a/vectordb_bench/frontend/const/dbPrices.py b/vectordb_bench/frontend/config/dbPrices.py similarity index 100% rename from vectordb_bench/frontend/const/dbPrices.py rename to vectordb_bench/frontend/config/dbPrices.py diff --git a/vectordb_bench/frontend/const/styles.py b/vectordb_bench/frontend/config/styles.py similarity index 100% rename from vectordb_bench/frontend/const/styles.py rename to vectordb_bench/frontend/config/styles.py diff --git a/vectordb_bench/frontend/pages/concurrent.py b/vectordb_bench/frontend/pages/concurrent.py index 0c1415efc..b4eae339c 100644 --- a/vectordb_bench/frontend/pages/concurrent.py +++ b/vectordb_bench/frontend/pages/concurrent.py @@ -1,18 +1,14 @@ - - - import streamlit as st -from vectordb_bench.backend.cases import CaseType from vectordb_bench.frontend.components.check_results.footer import footer -from vectordb_bench.frontend.components.check_results.expanderStyle import initMainExpanderStyle -from vectordb_bench.frontend.components.check_results.priceTable import priceTable from vectordb_bench.frontend.components.check_results.headerIcon import drawHeaderIcon -from vectordb_bench.frontend.components.check_results.nav import NavToResults, NavToRunTest -from vectordb_bench.frontend.components.check_results.charts import drawMetricChart +from vectordb_bench.frontend.components.check_results.nav import ( + NavToResults, + NavToRunTest, +) from vectordb_bench.frontend.components.check_results.filters import getshownData from vectordb_bench.frontend.components.concurrent.charts import drawChartsByCase from vectordb_bench.frontend.components.get_results.saveAsImage import getResults -from vectordb_bench.frontend.const.styles import * +from vectordb_bench.frontend.config.styles import FAVICON from vectordb_bench.interface import benchMarkRunner from vectordb_bench.models import TestResult @@ -30,26 +26,23 @@ def main(): drawHeaderIcon(st) allResults = benchMarkRunner.get_results() - + def check_conc_data(res: TestResult): case_results = res.results count = 0 for case_result in case_results: if len(case_result.metrics.conc_num_list) > 0: count += 1 - + return count > 0 - + checkedResults = [res for res in allResults if check_conc_data(res)] - st.title("VectorDB Benchmark (Concurrent Performance)") # results selector resultSelectorContainer = st.sidebar.container() - shownData, _, showCases = getshownData( - checkedResults, resultSelectorContainer) - + shownData, _, showCaseNames = getshownData(checkedResults, resultSelectorContainer) resultSelectorContainer.divider() @@ -61,8 +54,8 @@ def check_conc_data(res: TestResult): # save or share resultesContainer = st.sidebar.container() getResults(resultesContainer, "vectordb_bench_concurrent") - - drawChartsByCase(shownData, showCases, st.container()) + + drawChartsByCase(shownData, showCaseNames, st.container()) # footer footer(st.container()) diff --git a/vectordb_bench/frontend/pages/custom.py b/vectordb_bench/frontend/pages/custom.py new file mode 100644 index 000000000..28c249f78 --- /dev/null +++ b/vectordb_bench/frontend/pages/custom.py @@ -0,0 +1,64 @@ +import streamlit as st +from vectordb_bench.frontend.components.check_results.headerIcon import drawHeaderIcon +from vectordb_bench.frontend.components.custom.displayCustomCase import displayCustomCase +from vectordb_bench.frontend.components.custom.displaypPrams import displayParams +from vectordb_bench.frontend.components.custom.getCustomConfig import CustomCaseConfig, generate_custom_case, get_custom_configs, save_custom_configs +from vectordb_bench.frontend.components.custom.initStyle import initStyle +from vectordb_bench.frontend.config.styles import FAVICON, PAGE_TITLE + + +class CustomCaseManager(): + customCaseItems: list[CustomCaseConfig] + + def __init__(self): + self.customCaseItems = get_custom_configs() + + def addCase(self): + new_custom_case = generate_custom_case() + new_custom_case.dataset_config.name = f"{new_custom_case.dataset_config.name} {len(self.customCaseItems)}" + self.customCaseItems += [new_custom_case] + self.save() + + def deleteCase(self, idx: int): + self.customCaseItems.pop(idx) + self.save() + + def save(self): + save_custom_configs(self.customCaseItems) + + +def main(): + st.set_page_config( + page_title=PAGE_TITLE, + page_icon=FAVICON, + # layout="wide", + # initial_sidebar_state="collapsed", + ) + + # header + drawHeaderIcon(st) + + # init style + initStyle(st) + + st.title("Custom Dataset") + displayParams(st) + customCaseManager = CustomCaseManager() + + for idx, customCase in enumerate(customCaseManager.customCaseItems): + expander = st.expander(customCase.dataset_config.name, expanded=True) + key = f"custom_case_{idx}" + displayCustomCase(customCase, expander, key=key) + + columns = expander.columns(8) + columns[0].button( + "Save", key=f"{key}_", type="secondary", on_click=lambda: customCaseManager.save()) + columns[1].button(":red[Delete]", key=f"{key}_delete", type="secondary", + on_click=lambda: customCaseManager.deleteCase(idx)) + + st.button("\+ New Dataset", key=f"add_custom_configs", + type="primary", on_click=lambda: customCaseManager.addCase()) + + +if __name__ == "__main__": + main() diff --git a/vectordb_bench/frontend/pages/quries_per_dollar.py b/vectordb_bench/frontend/pages/quries_per_dollar.py index 10c1ac8f1..0bb05294b 100644 --- a/vectordb_bench/frontend/pages/quries_per_dollar.py +++ b/vectordb_bench/frontend/pages/quries_per_dollar.py @@ -8,7 +8,7 @@ from vectordb_bench.frontend.components.check_results.charts import drawMetricChart from vectordb_bench.frontend.components.check_results.filters import getshownData from vectordb_bench.frontend.components.get_results.saveAsImage import getResults -from vectordb_bench.frontend.const.styles import * +from vectordb_bench.frontend.config.styles import * from vectordb_bench.interface import benchMarkRunner from vectordb_bench.metric import QURIES_PER_DOLLAR_METRIC @@ -26,7 +26,7 @@ def main(): # results selector resultSelectorContainer = st.sidebar.container() - shownData, _, showCases = getshownData(allResults, resultSelectorContainer) + shownData, _, showCaseNames = getshownData(allResults, resultSelectorContainer) resultSelectorContainer.divider() @@ -45,8 +45,8 @@ def main(): priceMap = priceTable(priceTableContainer, shownData) # charts - for case in showCases: - data = [data for data in shownData if data["case_name"] == case.name] + for caseName in showCaseNames: + data = [data for data in shownData if data["case_name"] == caseName] dataWithMetric = [] metric = QURIES_PER_DOLLAR_METRIC for d in data: @@ -56,7 +56,7 @@ def main(): d[metric] = d["qps"] / price * 3.6 dataWithMetric.append(d) if len(dataWithMetric) > 0: - chartContainer = st.expander(case.name, True) + chartContainer = st.expander(caseName, True) drawMetricChart(data, metric, chartContainer) # footer diff --git a/vectordb_bench/frontend/pages/run_test.py b/vectordb_bench/frontend/pages/run_test.py index 0712bb6cc..1297743ae 100644 --- a/vectordb_bench/frontend/pages/run_test.py +++ b/vectordb_bench/frontend/pages/run_test.py @@ -5,6 +5,7 @@ from vectordb_bench.frontend.components.run_test.dbSelector import dbSelector from vectordb_bench.frontend.components.run_test.generateTasks import generate_tasks from vectordb_bench.frontend.components.run_test.hideSidebar import hideSidebar +from vectordb_bench.frontend.components.run_test.initStyle import initStyle from vectordb_bench.frontend.components.run_test.submitTask import submitTask from vectordb_bench.frontend.components.check_results.nav import NavToResults from vectordb_bench.frontend.components.check_results.headerIcon import drawHeaderIcon @@ -15,6 +16,9 @@ def main(): # set page config initRunTestPageConfig(st) + # init style + initStyle(st) + # header drawHeaderIcon(st) diff --git a/vectordb_bench/frontend/pages/tables.py b/vectordb_bench/frontend/pages/tables.py index a4dab68a6..c088dc930 100644 --- a/vectordb_bench/frontend/pages/tables.py +++ b/vectordb_bench/frontend/pages/tables.py @@ -1,7 +1,7 @@ import streamlit as st from vectordb_bench.frontend.components.check_results.headerIcon import drawHeaderIcon from vectordb_bench.frontend.components.tables.data import getNewResults -from vectordb_bench.frontend.const.styles import FAVICON +from vectordb_bench.frontend.config.styles import FAVICON def main(): @@ -21,4 +21,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/vectordb_bench/frontend/utils.py b/vectordb_bench/frontend/utils.py index 139854af6..787b67d03 100644 --- a/vectordb_bench/frontend/utils.py +++ b/vectordb_bench/frontend/utils.py @@ -1,6 +1,22 @@ -from vectordb_bench.models import CaseType +import random +import string + passwordKeys = ["password", "api_key"] + + def inputIsPassword(key: str) -> bool: return key.lower() in passwordKeys + +def addHorizontalLine(st): + st.markdown( + "
", + unsafe_allow_html=True, + ) + + +def generate_random_string(length): + letters = string.ascii_letters + string.digits + result = ''.join(random.choice(letters) for _ in range(length)) + return result diff --git a/vectordb_bench/frontend/vdb_benchmark.py b/vectordb_bench/frontend/vdb_benchmark.py index 0be43470e..b859c68b8 100644 --- a/vectordb_bench/frontend/vdb_benchmark.py +++ b/vectordb_bench/frontend/vdb_benchmark.py @@ -6,7 +6,7 @@ from vectordb_bench.frontend.components.check_results.charts import drawCharts from vectordb_bench.frontend.components.check_results.filters import getshownData from vectordb_bench.frontend.components.get_results.saveAsImage import getResults -from vectordb_bench.frontend.const.styles import * +from vectordb_bench.frontend.config.styles import * from vectordb_bench.interface import benchMarkRunner @@ -24,7 +24,7 @@ def main(): # results selector and filter resultSelectorContainer = st.sidebar.container() - shownData, failedTasks, showCases = getshownData( + shownData, failedTasks, showCaseNames = getshownData( allResults, resultSelectorContainer ) @@ -40,7 +40,7 @@ def main(): getResults(resultesContainer, "vectordb_bench") # charts - drawCharts(st, shownData, failedTasks, showCases) + drawCharts(st, shownData, failedTasks, showCaseNames) # footer footer(st.container()) diff --git a/vectordb_bench/models.py b/vectordb_bench/models.py index aa9c930ea..56034796e 100644 --- a/vectordb_bench/models.py +++ b/vectordb_bench/models.py @@ -94,6 +94,10 @@ def k(self, value): self._k = value ''' + def __hash__(self) -> int: + return hash(self.json()) + + class TaskStage(StrEnum): """Enumerations of various stages of the task""" @@ -250,18 +254,18 @@ def append_return(x, y): max_db = max(map(len, [f.task_config.db.name for f in filtered_results])) max_db_labels = ( - max(map(len, [f.task_config.db_config.db_label for f in filtered_results])) - + 3 + max(map(len, [f.task_config.db_config.db_label for f in filtered_results])) + + 3 ) max_case = max( map(len, [f.task_config.case_config.case_id.name for f in filtered_results]) ) max_load_dur = ( - max(map(len, [str(f.metrics.load_duration) for f in filtered_results])) + 3 + max(map(len, [str(f.metrics.load_duration) for f in filtered_results])) + 3 ) max_qps = max(map(len, [str(f.metrics.qps) for f in filtered_results])) + 3 max_recall = ( - max(map(len, [str(f.metrics.recall) for f in filtered_results])) + 3 + max(map(len, [str(f.metrics.recall) for f in filtered_results])) + 3 ) max_db_labels = 8 if max_db_labels < 8 else max_db_labels diff --git a/vectordb_bench/results/getLeaderboardData.py b/vectordb_bench/results/getLeaderboardData.py index 50f458533..c6484514d 100644 --- a/vectordb_bench/results/getLeaderboardData.py +++ b/vectordb_bench/results/getLeaderboardData.py @@ -2,7 +2,7 @@ import ujson import pathlib from vectordb_bench.backend.cases import CaseType -from vectordb_bench.frontend.const.dbPrices import DB_DBLABEL_TO_PRICE +from vectordb_bench.frontend.config.dbPrices import DB_DBLABEL_TO_PRICE from vectordb_bench.interface import benchMarkRunner from vectordb_bench.models import CaseResult, ResultLabel, TestResult From 243888dc86f974d7dacd5efd6b8f7d911de782a9 Mon Sep 17 00:00:00 2001 From: "min.tian" Date: Wed, 17 Jul 2024 15:40:05 +0800 Subject: [PATCH 072/327] remove redundant config definitions Signed-off-by: min.tian --- vectordb_bench/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/vectordb_bench/__init__.py b/vectordb_bench/__init__.py index c3822a64a..f7664502f 100644 --- a/vectordb_bench/__init__.py +++ b/vectordb_bench/__init__.py @@ -35,7 +35,6 @@ class config: K_DEFAULT = 100 # default return top k nearest neighbors during search - RESULTS_LOCAL_DIR = pathlib.Path(__file__).parent.joinpath("results") CUSTOM_CONFIG_DIR = pathlib.Path(__file__).parent.joinpath("custom/custom_case.json") CAPACITY_TIMEOUT_IN_SECONDS = 24 * 3600 # 24h From 57544b6061c250015bf067bf7b555096f317f0f7 Mon Sep 17 00:00:00 2001 From: Cai Yudong Date: Mon, 1 Jul 2024 14:29:59 +0800 Subject: [PATCH 073/327] Add client aws_opensearch Signed-off-by: Cai Yudong --- README.md | 25 +-- pyproject.toml | 3 + vectordb_bench/backend/clients/__init__.py | 13 ++ .../clients/aws_opensearch/aws_opensearch.py | 159 ++++++++++++++++++ .../backend/clients/aws_opensearch/cli.py | 44 +++++ .../backend/clients/aws_opensearch/config.py | 58 +++++++ .../backend/clients/aws_opensearch/run.py | 125 ++++++++++++++ vectordb_bench/cli/vectordbbench.py | 2 + vectordb_bench/frontend/config/styles.py | 2 + 9 files changed, 419 insertions(+), 12 deletions(-) create mode 100644 vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py create mode 100644 vectordb_bench/backend/clients/aws_opensearch/cli.py create mode 100644 vectordb_bench/backend/clients/aws_opensearch/config.py create mode 100644 vectordb_bench/backend/clients/aws_opensearch/run.py diff --git a/README.md b/README.md index 120a2431b..566b6deb0 100644 --- a/README.md +++ b/README.md @@ -27,18 +27,19 @@ pip install vectordb-bench[pinecone] ``` All the database client supported -|Optional database client|install command| -|---------------|---------------| -|pymilvus(*default*)|`pip install vectordb-bench`| -|all|`pip install vectordb-bench[all]`| -|qdrant|`pip install vectordb-bench[qdrant]`| -|pinecone|`pip install vectordb-bench[pinecone]`| -|weaviate|`pip install vectordb-bench[weaviate]`| -|elastic|`pip install vectordb-bench[elastic]`| -|pgvector|`pip install vectordb-bench[pgvector]`| -|pgvecto.rs|`pip install vectordb-bench[pgvecto_rs]`| -|redis|`pip install vectordb-bench[redis]`| -|chromadb|`pip install vectordb-bench[chromadb]`| +| Optional database client | install command | +|--------------------------|---------------------------------------------| +| pymilvus(*default*) | `pip install vectordb-bench` | +| all | `pip install vectordb-bench[all]` | +| qdrant | `pip install vectordb-bench[qdrant]` | +| pinecone | `pip install vectordb-bench[pinecone]` | +| weaviate | `pip install vectordb-bench[weaviate]` | +| elastic | `pip install vectordb-bench[elastic]` | +| pgvector | `pip install vectordb-bench[pgvector]` | +| pgvecto.rs | `pip install vectordb-bench[pgvecto_rs]` | +| redis | `pip install vectordb-bench[redis]` | +| chromadb | `pip install vectordb-bench[chromadb]` | +| awsopensearch | `pip install vectordb-bench[awsopensearch]` | ### Run diff --git a/pyproject.toml b/pyproject.toml index edc3575c5..2a812179c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,6 +62,8 @@ all = [ "psycopg2", "psycopg", "psycopg-binary", + "opensearch-dsl==2.1.0", + "opensearch-py==2.6.0", ] qdrant = [ "qdrant-client" ] @@ -72,6 +74,7 @@ pgvector = [ "psycopg", "psycopg-binary", "pgvector" ] pgvecto_rs = [ "psycopg2" ] redis = [ "redis" ] chromadb = [ "chromadb" ] +awsopensearch = [ "awsopensearch" ] zilliz_cloud = [] [project.urls] diff --git a/vectordb_bench/backend/clients/__init__.py b/vectordb_bench/backend/clients/__init__.py index 6a99661c2..2b45b71b9 100644 --- a/vectordb_bench/backend/clients/__init__.py +++ b/vectordb_bench/backend/clients/__init__.py @@ -32,6 +32,7 @@ class DB(Enum): PgVectoRS = "PgVectoRS" Redis = "Redis" Chroma = "Chroma" + AWSOpenSearch = "OpenSearch" Test = "test" @@ -78,6 +79,10 @@ def init_cls(self) -> Type[VectorDB]: from .chroma.chroma import ChromaClient return ChromaClient + if self == DB.AWSOpenSearch: + from .aws_opensearch.aws_opensearch import AWSOpenSearch + return AWSOpenSearch + @property def config_cls(self) -> Type[DBConfig]: """Import while in use""" @@ -121,6 +126,10 @@ def config_cls(self) -> Type[DBConfig]: from .chroma.config import ChromaConfig return ChromaConfig + if self == DB.AWSOpenSearch: + from .aws_opensearch.config import AWSOpenSearchConfig + return AWSOpenSearchConfig + def case_config_cls(self, index_type: IndexType | None = None) -> Type[DBCaseConfig]: if self == DB.Milvus: from .milvus.config import _milvus_case_config @@ -150,6 +159,10 @@ def case_config_cls(self, index_type: IndexType | None = None) -> Type[DBCaseCon from .pgvecto_rs.config import _pgvecto_rs_case_config return _pgvecto_rs_case_config.get(index_type) + if self == DB.AWSOpenSearch: + from .aws_opensearch.config import AWSOpenSearchIndexConfig + return AWSOpenSearchIndexConfig + # DB.Pinecone, DB.Chroma, DB.Redis return EmptyDBCaseConfig diff --git a/vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py b/vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py new file mode 100644 index 000000000..5b0728ac8 --- /dev/null +++ b/vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py @@ -0,0 +1,159 @@ +import logging +from contextlib import contextmanager +import time +from typing import Iterable, Type +from ..api import VectorDB, DBCaseConfig, DBConfig, IndexType +from .config import AWSOpenSearchConfig, AWSOpenSearchIndexConfig +from opensearchpy import OpenSearch +from opensearchpy.helpers import bulk + +log = logging.getLogger(__name__) + + +class AWSOpenSearch(VectorDB): + def __init__( + self, + dim: int, + db_config: dict, + db_case_config: AWSOpenSearchIndexConfig, + index_name: str = "vdb_bench_index", # must be lowercase + id_col_name: str = "id", + vector_col_name: str = "embedding", + drop_old: bool = False, + **kwargs, + ): + self.dim = dim + self.db_config = db_config + self.case_config = db_case_config + self.index_name = index_name + self.id_col_name = id_col_name + self.category_col_names = [ + f"scalar-{categoryCount}" for categoryCount in [2, 5, 10, 100, 1000] + ] + self.vector_col_name = vector_col_name + + log.info(f"AWS_OpenSearch client config: {self.db_config}") + client = OpenSearch(**self.db_config) + if drop_old: + log.info(f"AWS_OpenSearch client drop old index: {self.index_name}") + is_existed = client.indices.exists(index=self.index_name) + if is_existed: + client.indices.delete(index=self.index_name) + self._create_index(client) + + @classmethod + def config_cls(cls) -> AWSOpenSearchConfig: + return AWSOpenSearchConfig + + @classmethod + def case_config_cls( + cls, index_type: IndexType | None = None + ) -> AWSOpenSearchIndexConfig: + return AWSOpenSearchIndexConfig + + def _create_index(self, client: OpenSearch): + settings = { + "index": { + "knn": True, + # "number_of_shards": 5, + # "refresh_interval": "600s", + } + } + mappings = { + "properties": { + self.id_col_name: {"type": "integer"}, + **{ + categoryCol: {"type": "keyword"} + for categoryCol in self.category_col_names + }, + self.vector_col_name: { + "type": "knn_vector", + "dimension": self.dim, + "method": self.case_config.index_param(), + }, + } + } + try: + client.indices.create( + index=self.index_name, body=dict(settings=settings, mappings=mappings) + ) + except Exception as e: + log.warning(f"Failed to create index: {self.index_name} error: {str(e)}") + raise e from None + + @contextmanager + def init(self) -> None: + """connect to elasticsearch""" + self.client = OpenSearch(**self.db_config) + + yield + # self.client.transport.close() + self.client = None + del self.client + + def insert_embeddings( + self, + embeddings: Iterable[list[float]], + metadata: list[int], + **kwargs, + ) -> tuple[int, Exception]: + """Insert the embeddings to the elasticsearch.""" + assert self.client is not None, "should self.init() first" + + insert_data = [] + for i in range(len(embeddings)): + insert_data.append({"index": {"_index": self.index_name, "_id": metadata[i]}}) + insert_data.append({self.vector_col_name: embeddings[i]}) + try: + resp = self.client.bulk(insert_data) + log.info(f"AWS_OpenSearch adding documents: {len(resp['items'])}") + resp = self.client.indices.stats(self.index_name) + log.info(f"Total document count in index: {resp['_all']['primaries']['indexing']['index_total']}") + return (len(embeddings), None) + except Exception as e: + log.warning(f"Failed to insert data: {self.index_name} error: {str(e)}") + time.sleep(10) + return self.insert_embeddings(embeddings, metadata) + + def search_embedding( + self, + query: list[float], + k: int = 100, + filters: dict | None = None, + ) -> list[int]: + """Get k most similar embeddings to query vector. + + Args: + query(list[float]): query embedding to look up documents similar to. + k(int): Number of most similar embeddings to return. Defaults to 100. + filters(dict, optional): filtering expression to filter the data while searching. + + Returns: + list[tuple[int, float]]: list of k most similar embeddings in (id, score) tuple to the query embedding. + """ + assert self.client is not None, "should self.init() first" + + body = { + "size": k, + "query": {"knn": {self.vector_col_name: {"vector": query, "k": k}}}, + } + try: + resp = self.client.search(index=self.index_name, body=body) + log.info(f'Search took: {resp["took"]}') + log.info(f'Search shards: {resp["_shards"]}') + log.info(f'Search hits total: {resp["hits"]["total"]}') + result = [int(d["_id"]) for d in resp["hits"]["hits"]] + # log.info(f'success! length={len(res)}') + + return result + except Exception as e: + log.warning(f"Failed to search: {self.index_name} error: {str(e)}") + raise e from None + + def optimize(self): + """optimize will be called between insertion and search in performance cases.""" + pass + + def ready_to_load(self): + """ready_to_load will be called before load in load cases.""" + pass diff --git a/vectordb_bench/backend/clients/aws_opensearch/cli.py b/vectordb_bench/backend/clients/aws_opensearch/cli.py new file mode 100644 index 000000000..5cb4ebbe1 --- /dev/null +++ b/vectordb_bench/backend/clients/aws_opensearch/cli.py @@ -0,0 +1,44 @@ +from typing import Annotated, TypedDict, Unpack + +import click +from pydantic import SecretStr + +from ....cli.cli import ( + CommonTypedDict, + HNSWFlavor2, + cli, + click_parameter_decorators_from_typed_dict, + run, +) +from .. import DB + + +class AWSOpenSearchTypedDict(TypedDict): + host: Annotated[ + str, click.option("--host", type=str, help="Db host", required=True) + ] + port: Annotated[int, click.option("--port", type=int, default=443, help="Db Port")] + user: Annotated[str, click.option("--user", type=str, default="admin", help="Db User")] + password: Annotated[str, click.option("--password", type=str, help="Db password")] + + +class AWSOpenSearchHNSWTypedDict(CommonTypedDict, AWSOpenSearchTypedDict, HNSWFlavor2): + ... + + +@cli.command() +@click_parameter_decorators_from_typed_dict(AWSOpenSearchHNSWTypedDict) +def AWSOpenSearch(**parameters: Unpack[AWSOpenSearchHNSWTypedDict]): + from .config import AWSOpenSearchConfig, AWSOpenSearchIndexConfig + run( + db=DB.AWSOpenSearch, + db_config=AWSOpenSearchConfig( + host=parameters["host"], + port=parameters["port"], + user=parameters["user"], + password=SecretStr(parameters["password"]), + ), + db_case_config=AWSOpenSearchIndexConfig( + ), + **parameters, + ) diff --git a/vectordb_bench/backend/clients/aws_opensearch/config.py b/vectordb_bench/backend/clients/aws_opensearch/config.py new file mode 100644 index 000000000..bc82380b7 --- /dev/null +++ b/vectordb_bench/backend/clients/aws_opensearch/config.py @@ -0,0 +1,58 @@ +from enum import Enum +from pydantic import SecretStr, BaseModel + +from ..api import DBConfig, DBCaseConfig, MetricType, IndexType + + +class AWSOpenSearchConfig(DBConfig, BaseModel): + host: str = "" + port: int = 443 + user: str = "" + password: SecretStr = "" + + def to_dict(self) -> dict: + return { + "hosts": [{'host': self.host, 'port': self.port}], + "http_auth": (self.user, self.password.get_secret_value()), + "use_ssl": True, + "http_compress": True, + "verify_certs": True, + "ssl_assert_hostname": False, + "ssl_show_warn": False, + "timeout": 600, + } + + +class AWSOS_Engine(Enum): + nmslib = "nmslib" + faiss = "faiss" + lucene = "Lucene" + + +class AWSOpenSearchIndexConfig(BaseModel, DBCaseConfig): + metric_type: MetricType = MetricType.L2 + engine: AWSOS_Engine = AWSOS_Engine.nmslib + efConstruction: int = 360 + M: int = 30 + + def parse_metric(self) -> str: + if self.metric_type == MetricType.IP: + return "innerproduct" # only support faiss / nmslib, not for Lucene. + elif self.metric_type == MetricType.COSINE: + return "cosinesimil" + return "l2" + + def index_param(self) -> dict: + params = { + "name": "hnsw", + "space_type": self.parse_metric(), + "engine": self.engine.value, + "parameters": { + "ef_construction": self.efConstruction, + "m": self.M + } + } + return params + + def search_param(self) -> dict: + return {} diff --git a/vectordb_bench/backend/clients/aws_opensearch/run.py b/vectordb_bench/backend/clients/aws_opensearch/run.py new file mode 100644 index 000000000..3924cbd75 --- /dev/null +++ b/vectordb_bench/backend/clients/aws_opensearch/run.py @@ -0,0 +1,125 @@ +import time, random +from opensearchpy import OpenSearch +from opensearch_dsl import Search, Document, Text, Keyword + +_HOST = 'xxxxxx.us-west-2.es.amazonaws.com' +_PORT = 443 +_AUTH = ('admin', 'xxxxxx') # For testing only. Don't store credentials in code. + +_INDEX_NAME = 'my-dsl-index' +_BATCH = 100 +_ROWS = 100 +_DIM = 128 +_TOPK = 10 + + +def create_client(): + client = OpenSearch( + hosts=[{'host': _HOST, 'port': _PORT}], + http_compress=True, # enables gzip compression for request bodies + http_auth=_AUTH, + use_ssl=True, + verify_certs=True, + ssl_assert_hostname=False, + ssl_show_warn=False, + ) + return client + + +def create_index(client, index_name): + settings = { + "index": { + "knn": True, + "number_of_shards": 1, + "refresh_interval": "5s", + } + } + mappings = { + "properties": { + "embedding": { + "type": "knn_vector", + "dimension": _DIM, + "method": { + "engine": "nmslib", + "name": "hnsw", + "space_type": "l2", + "parameters": { + "ef_construction": 128, + "m": 24, + } + } + } + } + } + + response = client.indices.create(index=index_name, body=dict(settings=settings, mappings=mappings)) + print('\nCreating index:') + print(response) + + +def delete_index(client, index_name): + response = client.indices.delete(index=index_name) + print('\nDeleting index:') + print(response) + + +def bulk_insert(client, index_name): + # Perform bulk operations + ids = [i for i in range(_ROWS)] + vec = [[random.random() for _ in range(_DIM)] for _ in range(_ROWS)] + + docs = [] + for i in range(0, _ROWS, _BATCH): + docs.clear() + for j in range(0, _BATCH): + docs.append({"index": {"_index": index_name, "_id": ids[i+j]}}) + docs.append({"embedding": vec[i+j]}) + response = client.bulk(docs) + print('\nAdding documents:', len(response['items']), response['errors']) + response = client.indices.stats(index_name) + print('\nTotal document count in index:', response['_all']['primaries']['indexing']['index_total']) + + +def search(client, index_name): + # Search for the document. + search_body = { + "size": _TOPK, + "query": { + "knn": { + "embedding": { + "vector": [random.random() for _ in range(_DIM)], + "k": _TOPK, + } + } + } + } + while True: + response = client.search(index=index_name, body=search_body) + print(f'\nSearch took: {response["took"]}') + print(f'\nSearch shards: {response["_shards"]}') + print(f'\nSearch hits total: {response["hits"]["total"]}') + result = response["hits"]["hits"] + if len(result) != 0: + print('\nSearch results:') + for hit in response["hits"]["hits"]: + print(hit["_id"], hit["_score"]) + break + else: + print('\nSearch not ready, sleep 1s') + time.sleep(1) + + +def main(): + client = create_client() + try: + create_index(client, _INDEX_NAME) + bulk_insert(client, _INDEX_NAME) + search(client, _INDEX_NAME) + delete_index(client, _INDEX_NAME) + except Exception as e: + print(e) + delete_index(client, _INDEX_NAME) + + +if __name__ == '__main__': + main() diff --git a/vectordb_bench/cli/vectordbbench.py b/vectordb_bench/cli/vectordbbench.py index 396909cd5..0b619bbec 100644 --- a/vectordb_bench/cli/vectordbbench.py +++ b/vectordb_bench/cli/vectordbbench.py @@ -4,6 +4,7 @@ from ..backend.clients.weaviate_cloud.cli import Weaviate from ..backend.clients.zilliz_cloud.cli import ZillizAutoIndex from ..backend.clients.milvus.cli import MilvusAutoIndex +from ..backend.clients.aws_opensearch.cli import AWSOpenSearch from .cli import cli @@ -14,6 +15,7 @@ cli.add_command(Test) cli.add_command(ZillizAutoIndex) cli.add_command(MilvusAutoIndex) +cli.add_command(AWSOpenSearch) if __name__ == "__main__": diff --git a/vectordb_bench/frontend/config/styles.py b/vectordb_bench/frontend/config/styles.py index 52d1017a9..3e0fdb112 100644 --- a/vectordb_bench/frontend/config/styles.py +++ b/vectordb_bench/frontend/config/styles.py @@ -46,6 +46,7 @@ def getPatternShape(i): DB.PgVectoRS: "https://assets.zilliz.com/PG_Vector_d464f2ef5f.png", DB.Redis: "https://assets.zilliz.com/Redis_Cloud_74b8bfef39.png", DB.Chroma: "https://assets.zilliz.com/chroma_ceb3f06ed7.png", + DB.AWSOpenSearch: "https://assets.zilliz.com/opensearch_1eee37584e.jpeg", } # RedisCloud color: #0D6EFD @@ -59,4 +60,5 @@ def getPatternShape(i): DB.WeaviateCloud.value: "#20C997", DB.PgVector.value: "#4C779A", DB.Redis.value: "#0D6EFD", + DB.AWSOpenSearch.value: "#0DCAF0", } From 29f5c086565bfd6f15bd83ae2661b154f68bbb3b Mon Sep 17 00:00:00 2001 From: "min.tian" Date: Wed, 24 Jul 2024 16:03:51 +0800 Subject: [PATCH 074/327] add new db_config for better labeling: version, note Signed-off-by: min.tian --- vectordb_bench/backend/clients/api.py | 21 +++++++- .../frontend/components/check_results/data.py | 19 ++++--- .../components/run_test/dbConfigSetting.py | 52 +++++++++++++------ .../frontend/components/run_test/initStyle.py | 4 +- vectordb_bench/models.py | 12 +++-- 5 files changed, 81 insertions(+), 27 deletions(-) diff --git a/vectordb_bench/backend/clients/api.py b/vectordb_bench/backend/clients/api.py index 0024bf600..d9ec5d83b 100644 --- a/vectordb_bench/backend/clients/api.py +++ b/vectordb_bench/backend/clients/api.py @@ -38,6 +38,22 @@ class DBConfig(ABC, BaseModel): """ db_label: str = "" + version: str = "" + note: str = "" + + @staticmethod + def common_short_configs() -> list[str]: + """ + short input, such as `db_label`, `version` + """ + return ["version", "db_label"] + + @staticmethod + def common_long_configs() -> list[str]: + """ + long input, such as `note` + """ + return ["note"] @abstractmethod def to_dict(self) -> dict: @@ -45,7 +61,10 @@ def to_dict(self) -> dict: @validator("*") def not_empty_field(cls, v, field): - if field.name == "db_label": + if ( + field.name in cls.common_short_configs() + or field.name in cls.common_long_configs() + ): return v if not v and isinstance(v, (str, SecretStr)): raise ValueError("Empty string!") diff --git a/vectordb_bench/frontend/components/check_results/data.py b/vectordb_bench/frontend/components/check_results/data.py index 1e6bba00e..b3cac21e1 100644 --- a/vectordb_bench/frontend/components/check_results/data.py +++ b/vectordb_bench/frontend/components/check_results/data.py @@ -24,7 +24,10 @@ def getFilterTasks( task for task in tasks if task.task_config.db_name in dbNames - and task.task_config.case_config.case_id.case_cls(task.task_config.case_config.custom_case).name in caseNames + and task.task_config.case_config.case_id.case_cls( + task.task_config.case_config.custom_case + ).name + in caseNames ] return filterTasks @@ -35,17 +38,20 @@ def mergeTasks(tasks: list[CaseResult]): db_name = task.task_config.db_name db = task.task_config.db.value db_label = task.task_config.db_config.db_label or "" - case = task.task_config.case_config.case_id.case_cls(task.task_config.case_config.custom_case) + version = task.task_config.db_config.version or "" + case = task.task_config.case_config.case_id.case_cls( + task.task_config.case_config.custom_case + ) dbCaseMetricsMap[db_name][case.name] = { "db": db, "db_label": db_label, + "version": version, "metrics": mergeMetrics( dbCaseMetricsMap[db_name][case.name].get("metrics", {}), asdict(task.metrics), ), "label": getBetterLabel( - dbCaseMetricsMap[db_name][case.name].get( - "label", ResultLabel.FAILED), + dbCaseMetricsMap[db_name][case.name].get("label", ResultLabel.FAILED), task.label, ), } @@ -57,6 +63,7 @@ def mergeTasks(tasks: list[CaseResult]): metrics = metricInfo["metrics"] db = metricInfo["db"] db_label = metricInfo["db_label"] + version = metricInfo["version"] label = metricInfo["label"] if label == ResultLabel.NORMAL: mergedTasks.append( @@ -64,6 +71,7 @@ def mergeTasks(tasks: list[CaseResult]): "db_name": db_name, "db": db, "db_label": db_label, + "version": version, "case_name": case_name, "metricsSet": set(metrics.keys()), **metrics, @@ -79,8 +87,7 @@ def mergeMetrics(metrics_1: dict, metrics_2: dict) -> dict: metrics = {**metrics_1} for key, value in metrics_2.items(): metrics[key] = ( - getBetterMetric( - key, value, metrics[key]) if key in metrics else value + getBetterMetric(key, value, metrics[key]) if key in metrics else value ) return metrics diff --git a/vectordb_bench/frontend/components/run_test/dbConfigSetting.py b/vectordb_bench/frontend/components/run_test/dbConfigSetting.py index 8f4f35c93..257608413 100644 --- a/vectordb_bench/frontend/components/run_test/dbConfigSetting.py +++ b/vectordb_bench/frontend/components/run_test/dbConfigSetting.py @@ -1,9 +1,10 @@ from pydantic import ValidationError -from vectordb_bench.frontend.config.styles import * +from vectordb_bench.backend.clients import DB +from vectordb_bench.frontend.config.styles import DB_CONFIG_SETTING_COLUMNS from vectordb_bench.frontend.utils import inputIsPassword -def dbConfigSettings(st, activedDbList): +def dbConfigSettings(st, activedDbList: list[DB]): expander = st.expander("Configurations for the selected databases", True) dbConfigs = {} @@ -27,7 +28,7 @@ def dbConfigSettings(st, activedDbList): return dbConfigs, isAllValid -def dbConfigSettingItem(st, activeDb): +def dbConfigSettingItem(st, activeDb: DB): st.markdown( f"
{activeDb.value}
", unsafe_allow_html=True, @@ -36,20 +37,41 @@ def dbConfigSettingItem(st, activeDb): dbConfigClass = activeDb.config_cls properties = dbConfigClass.schema().get("properties") - propertiesItems = list(properties.items()) - moveDBLabelToLast(propertiesItems) dbConfig = {} - for j, property in enumerate(propertiesItems): - column = columns[j % DB_CONFIG_SETTING_COLUMNS] - key, value = property + idx = 0 + + # db config (unique) + for key, property in properties.items(): + if ( + key not in dbConfigClass.common_short_configs() + and key not in dbConfigClass.common_long_configs() + ): + column = columns[idx % DB_CONFIG_SETTING_COLUMNS] + idx += 1 + dbConfig[key] = column.text_input( + key, + key="%s-%s" % (activeDb.name, key), + value=property.get("default", ""), + type="password" if inputIsPassword(key) else "default", + ) + # db config (common short labels) + for key in dbConfigClass.common_short_configs(): + column = columns[idx % DB_CONFIG_SETTING_COLUMNS] + idx += 1 dbConfig[key] = column.text_input( key, - key="%s-%s" % (activeDb, key), - value=value.get("default", ""), - type="password" if inputIsPassword(key) else "default", + key="%s-%s" % (activeDb.name, key), + value="", + type="default", + placeholder="optional, for labeling results", ) - return dbConfig - -def moveDBLabelToLast(propertiesItems): - propertiesItems.sort(key=lambda x: 1 if x[0] == "db_label" else 0) + # db config (common long text_input) + for key in dbConfigClass.common_long_configs(): + dbConfig[key] = st.text_area( + key, + key="%s-%s" % (activeDb.name, key), + value="", + placeholder="optional", + ) + return dbConfig diff --git a/vectordb_bench/frontend/components/run_test/initStyle.py b/vectordb_bench/frontend/components/run_test/initStyle.py index 59dd438e1..1e6af57ad 100644 --- a/vectordb_bench/frontend/components/run_test/initStyle.py +++ b/vectordb_bench/frontend/components/run_test/initStyle.py @@ -9,6 +9,8 @@ def initStyle(st): div[data-testid='stHorizontalBlock'] {gap: 8px;} /* check box */ .stCheckbox p { color: #000; font-size: 18px; font-weight: 600; } + /* db selector - db_name should not wrap */ + div[data-testid="stVerticalBlockBorderWrapper"] div[data-testid="stCheckbox"] div[data-testid="stWidgetLabel"] p { white-space: nowrap; } """, unsafe_allow_html=True, - ) \ No newline at end of file + ) diff --git a/vectordb_bench/models.py b/vectordb_bench/models.py index 56034796e..41be95b7c 100644 --- a/vectordb_bench/models.py +++ b/vectordb_bench/models.py @@ -2,7 +2,7 @@ import pathlib from datetime import date from enum import Enum, StrEnum, auto -from typing import List, Self, Sequence, Set +from typing import List, Self import ujson @@ -10,7 +10,6 @@ DB, DBConfig, DBCaseConfig, - IndexType, ) from .backend.cases import CaseType from .base import BaseModel @@ -128,9 +127,14 @@ class TaskConfig(BaseModel): @property def db_name(self): - db = self.db.value + db_name = f"{self.db.value}" db_label = self.db_config.db_label - return f"{db}-{db_label}" if db_label else db + if db_label: + db_name += f"-{db_label}" + version = self.db_config.version + if version: + db_name += f"-{version}" + return db_name class ResultLabel(Enum): From 4d0bdca95226cfcde6de6d2654216eeeaced6b00 Mon Sep 17 00:00:00 2001 From: baswanth09 <122412818+baswanth09@users.noreply.github.com> Date: Tue, 30 Jul 2024 21:41:26 -0400 Subject: [PATCH 075/327] Support for MemoryDB client (#339) * Add support for MemoryDB * Add support for MemoryDB * Update vectordb_bench/backend/clients/memorydb/cli.py Co-authored-by: Jonathan S. Katz * Update vectordb_bench/backend/clients/memorydb/cli.py Co-authored-by: Jonathan S. Katz --------- Co-authored-by: Baswanth Vegunta Co-authored-by: Jonathan S. Katz --- .gitignore | 1 + README.md | 1 + pyproject.toml | 1 + vectordb_bench/backend/clients/__init__.py | 9 + .../backend/clients/memorydb/cli.py | 88 ++++++ .../backend/clients/memorydb/config.py | 54 ++++ .../backend/clients/memorydb/memorydb.py | 254 ++++++++++++++++++ vectordb_bench/cli/vectordbbench.py | 2 + 8 files changed, 410 insertions(+) create mode 100644 vectordb_bench/backend/clients/memorydb/cli.py create mode 100644 vectordb_bench/backend/clients/memorydb/config.py create mode 100644 vectordb_bench/backend/clients/memorydb/memorydb.py diff --git a/.gitignore b/.gitignore index 004524444..f8602c11a 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ __MACOSX build/ venv/ .idea/ +results/ \ No newline at end of file diff --git a/README.md b/README.md index 566b6deb0..11aaed3b7 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,7 @@ All the database client supported | pgvector | `pip install vectordb-bench[pgvector]` | | pgvecto.rs | `pip install vectordb-bench[pgvecto_rs]` | | redis | `pip install vectordb-bench[redis]` | +| memorydb | `pip install vectordb-bench[memorydb]` | | chromadb | `pip install vectordb-bench[chromadb]` | | awsopensearch | `pip install vectordb-bench[awsopensearch]` | diff --git a/pyproject.toml b/pyproject.toml index 2a812179c..330867d56 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,6 +73,7 @@ elastic = [ "elasticsearch" ] pgvector = [ "psycopg", "psycopg-binary", "pgvector" ] pgvecto_rs = [ "psycopg2" ] redis = [ "redis" ] +memorydb = [ "memorydb" ] chromadb = [ "chromadb" ] awsopensearch = [ "awsopensearch" ] zilliz_cloud = [] diff --git a/vectordb_bench/backend/clients/__init__.py b/vectordb_bench/backend/clients/__init__.py index 2b45b71b9..c638208d9 100644 --- a/vectordb_bench/backend/clients/__init__.py +++ b/vectordb_bench/backend/clients/__init__.py @@ -31,6 +31,7 @@ class DB(Enum): PgVector = "PgVector" PgVectoRS = "PgVectoRS" Redis = "Redis" + MemoryDB = "MemoryDB" Chroma = "Chroma" AWSOpenSearch = "OpenSearch" Test = "test" @@ -74,6 +75,10 @@ def init_cls(self) -> Type[VectorDB]: if self == DB.Redis: from .redis.redis import Redis return Redis + + if self == DB.MemoryDB: + from .memorydb.memorydb import MemoryDB + return MemoryDB if self == DB.Chroma: from .chroma.chroma import ChromaClient @@ -121,6 +126,10 @@ def config_cls(self) -> Type[DBConfig]: if self == DB.Redis: from .redis.config import RedisConfig return RedisConfig + + if self == DB.MemoryDB: + from .memorydb.config import MemoryDBConfig + return MemoryDBConfig if self == DB.Chroma: from .chroma.config import ChromaConfig diff --git a/vectordb_bench/backend/clients/memorydb/cli.py b/vectordb_bench/backend/clients/memorydb/cli.py new file mode 100644 index 000000000..50b5f89ba --- /dev/null +++ b/vectordb_bench/backend/clients/memorydb/cli.py @@ -0,0 +1,88 @@ +from typing import Annotated, TypedDict, Unpack + +import click +from pydantic import SecretStr + +from ....cli.cli import ( + CommonTypedDict, + HNSWFlavor2, + cli, + click_parameter_decorators_from_typed_dict, + run, +) +from .. import DB + + +class MemoryDBTypedDict(TypedDict): + host: Annotated[ + str, click.option("--host", type=str, help="Db host", required=True) + ] + password: Annotated[str, click.option("--password", type=str, help="Db password")] + port: Annotated[int, click.option("--port", type=int, default=6379, help="Db Port")] + ssl: Annotated[ + bool, + click.option( + "--ssl/--no-ssl", + is_flag=True, + show_default=True, + default=True, + help="Enable or disable SSL for MemoryDB", + ), + ] + ssl_ca_certs: Annotated[ + str, + click.option( + "--ssl-ca-certs", + show_default=True, + help="Path to certificate authority file to use for SSL", + ), + ] + cmd: Annotated[ + bool, + click.option( + "--cmd", + is_flag=True, + show_default=True, + default=False, + help="Cluster Mode Disabled (CMD), use this flag when testing locally on a single node instance. In production, MemoryDB only supports cluster mode (CME)", + ), + ] + insert_batch_size: Annotated[ + int, + click.option( + "--insert-batch-size", + type=int, + default=10, + help="Batch size for inserting data. Adjust this as needed, but don't make it too big", + ), + ] + + +class MemoryDBHNSWTypedDict(CommonTypedDict, MemoryDBTypedDict, HNSWFlavor2): + ... + + +@cli.command() +@click_parameter_decorators_from_typed_dict(MemoryDBHNSWTypedDict) +def MemoryDB(**parameters: Unpack[MemoryDBHNSWTypedDict]): + from .config import MemoryDBConfig, MemoryDBHNSWConfig + + run( + db=DB.MemoryDB, + db_config=MemoryDBConfig( + db_label=parameters["db_label"], + password=SecretStr(parameters["password"]) if parameters["password"] else None, + host=SecretStr(parameters["host"]), + port=parameters["port"], + ssl=parameters["ssl"], + ssl_ca_certs=parameters["ssl_ca_certs"], + cmd=parameters["cmd"], + ), + db_case_config=MemoryDBHNSWConfig( + M=parameters["m"], + ef_construction=parameters["ef_construction"], + ef_runtime=parameters["ef_runtime"], + insert_batch_size=parameters["insert_batch_size"] + ), + **parameters, + ) \ No newline at end of file diff --git a/vectordb_bench/backend/clients/memorydb/config.py b/vectordb_bench/backend/clients/memorydb/config.py new file mode 100644 index 000000000..1284d3449 --- /dev/null +++ b/vectordb_bench/backend/clients/memorydb/config.py @@ -0,0 +1,54 @@ +from pydantic import BaseModel, SecretStr + +from ..api import DBCaseConfig, DBConfig, IndexType, MetricType + + +class MemoryDBConfig(DBConfig): + host: SecretStr + password: SecretStr | None = None + port: int | None = None + ssl: bool | None = None + cmd: bool | None = None + ssl_ca_certs: str | None = None + + def to_dict(self) -> dict: + return { + "host": self.host.get_secret_value(), + "port": self.port, + "password": self.password.get_secret_value() if self.password else None, + "ssl": self.ssl, + "cmd": self.cmd, + "ssl_ca_certs": self.ssl_ca_certs, + } + + +class MemoryDBIndexConfig(BaseModel, DBCaseConfig): + metric_type: MetricType | None = None + insert_batch_size: int | None = None + + def parse_metric(self) -> str: + if self.metric_type == MetricType.L2: + return "l2" + elif self.metric_type == MetricType.IP: + return "ip" + return "cosine" + + +class MemoryDBHNSWConfig(MemoryDBIndexConfig): + M: int | None = 16 + ef_construction: int | None = 64 + ef_runtime: int | None = 10 + index: IndexType = IndexType.HNSW + + def index_param(self) -> dict: + return { + "metric": self.parse_metric(), + "index_type": self.index.value, + "m": self.M, + "ef_construction": self.ef_construction, + } + + def search_param(self) -> dict: + return { + "ef_runtime": self.ef_runtime, + } \ No newline at end of file diff --git a/vectordb_bench/backend/clients/memorydb/memorydb.py b/vectordb_bench/backend/clients/memorydb/memorydb.py new file mode 100644 index 000000000..c5f80eb2a --- /dev/null +++ b/vectordb_bench/backend/clients/memorydb/memorydb.py @@ -0,0 +1,254 @@ +import logging, time +from contextlib import contextmanager +from typing import Any, Generator, Optional, Tuple, Type +from ..api import VectorDB, DBCaseConfig, IndexType +from .config import MemoryDBIndexConfig +import redis +from redis import Redis +from redis.cluster import RedisCluster +from redis.commands.search.field import TagField, VectorField, NumericField +from redis.commands.search.indexDefinition import IndexDefinition, IndexType +from redis.commands.search.query import Query +import numpy as np + + +log = logging.getLogger(__name__) +INDEX_NAME = "index" # Vector Index Name + +class MemoryDB(VectorDB): + def __init__( + self, + dim: int, + db_config: dict, + db_case_config: MemoryDBIndexConfig, + drop_old: bool = False, + **kwargs + ): + + self.db_config = db_config + self.case_config = db_case_config + self.collection_name = INDEX_NAME + self.target_nodes = RedisCluster.RANDOM if not self.db_config["cmd"] else None + self.insert_batch_size = db_case_config.insert_batch_size + self.dbsize = kwargs.get("num_rows") + + # Create a MemoryDB connection, if db has password configured, add it to the connection here and in init(): + log.info(f"Establishing connection to: {self.db_config}") + conn = self.get_client(primary=True) + log.info(f"Connection established: {conn}") + log.info(conn.execute_command("INFO server")) + + if drop_old: + try: + log.info(f"MemoryDB client getting info for: {INDEX_NAME}") + info = conn.ft(INDEX_NAME).info() + log.info(f"Index info: {info}") + except redis.exceptions.ResponseError as e: + log.error(e) + drop_old = False + log.info(f"MemoryDB client drop_old collection: {self.collection_name}") + + log.info("Executing FLUSHALL") + conn.flushall() + + # Since the default behaviour of FLUSHALL is asynchronous, wait for db to be empty + self.wait_until(self.wait_for_empty_db, 3, "", conn) + if not self.db_config["cmd"]: + replica_clients = self.get_client(replicas=True) + for rc, host in replica_clients: + self.wait_until(self.wait_for_empty_db, 3, "", rc) + log.debug(f"Flushall done in the host: {host}") + rc.close() + + self.make_index(dim, conn) + conn.close() + conn = None + + def make_index(self, vector_dimensions: int, conn: redis.Redis): + try: + # check to see if index exists + conn.ft(INDEX_NAME).info() + except Exception as e: + log.warn(f"Error getting info for index '{INDEX_NAME}': {e}") + index_param = self.case_config.index_param() + search_param = self.case_config.search_param() + vector_parameters = { # Vector Index Type: FLAT or HNSW + "TYPE": "FLOAT32", + "DIM": vector_dimensions, # Number of Vector Dimensions + "DISTANCE_METRIC": index_param["metric"], # Vector Search Distance Metric + } + if index_param["m"]: + vector_parameters["M"] = index_param["m"] + if index_param["ef_construction"]: + vector_parameters["EF_CONSTRUCTION"] = index_param["ef_construction"] + if search_param["ef_runtime"]: + vector_parameters["EF_RUNTIME"] = search_param["ef_runtime"] + + schema = ( + TagField("id"), + NumericField("metadata"), + VectorField("vector", # Vector Field Name + "HNSW", vector_parameters + ), + ) + + definition = IndexDefinition(index_type=IndexType.HASH) + rs = conn.ft(INDEX_NAME) + rs.create_index(schema, definition=definition) + + def get_client(self, **kwargs): + """ + Gets either cluster connection or normal connection based on `cmd` flag. + CMD stands for Cluster Mode Disabled and is a "mode". + """ + if not self.db_config["cmd"]: + # Cluster mode enabled + + client = RedisCluster( + host=self.db_config["host"], + port=self.db_config["port"], + ssl=self.db_config["ssl"], + password=self.db_config["password"], + ssl_ca_certs=self.db_config["ssl_ca_certs"], + ssl_cert_reqs=None, + ) + + # Ping all nodes to create a connection + client.execute_command("PING", target_nodes=RedisCluster.ALL_NODES) + replicas = client.get_replicas() + + if len(replicas) > 0: + # FT.SEARCH is a keyless command, use READONLY for replica connections + client.execute_command("READONLY", target_nodes=RedisCluster.REPLICAS) + + if kwargs.get("primary", False): + client = client.get_primaries()[0].redis_connection + + if kwargs.get("replicas", False): + # Return client and host name for each replica + return [(c.redis_connection, c.host) for c in replicas] + + else: + client = Redis( + host=self.db_config["host"], + port=self.db_config["port"], + db=0, + ssl=self.db_config["ssl"], + password=self.db_config["password"], + ssl_ca_certs=self.db_config["ssl_ca_certs"], + ssl_cert_reqs=None, + ) + client.execute_command("PING") + return client + + @contextmanager + def init(self) -> Generator[None, None, None]: + """ create and destory connections to database. + + Examples: + >>> with self.init(): + >>> self.insert_embeddings() + """ + self.conn = self.get_client() + search_param = self.case_config.search_param() + if search_param["ef_runtime"]: + self.ef_runtime_str = f'EF_RUNTIME {search_param["ef_runtime"]}' + else: + self.ef_runtime_str = "" + yield + self.conn.close() + self.conn = None + + def ready_to_load(self) -> bool: + pass + + def optimize(self) -> None: + self._post_insert() + + def insert_embeddings( + self, + embeddings: list[list[float]], + metadata: list[int], + **kwargs: Any, + ) -> Tuple[int, Optional[Exception]]: + """Insert embeddings into the database. + Should call self.init() first. + """ + + try: + with self.conn.pipeline(transaction=False) as pipe: + for i, embedding in enumerate(embeddings): + embedding = np.array(embedding).astype(np.float32) + pipe.hset(metadata[i], mapping = { + "id": str(metadata[i]), + "metadata": metadata[i], + "vector": embedding.tobytes(), + }) + # Execute the pipe so we don't keep too much in memory at once + if (i + 1) % self.insert_batch_size == 0: + pipe.execute() + + pipe.execute() + result_len = i + 1 + except Exception as e: + return 0, e + + return result_len, None + + def _post_insert(self): + """Wait for indexing to finish""" + client = self.get_client(primary=True) + log.info("Waiting for background indexing to finish") + args = (self.wait_for_no_activity, 5, "", client) + self.wait_until(*args) + if not self.db_config["cmd"]: + replica_clients = self.get_client(replicas=True) + for rc, host_name in replica_clients: + args = (self.wait_for_no_activity, 5, "", rc) + self.wait_until(*args) + log.debug(f"Background indexing completed in the host: {host_name}") + rc.close() + + def wait_until( + self, condition, interval=5, message="Operation took too long", *args + ): + while not condition(*args): + time.sleep(interval) + + def wait_for_no_activity(self, client: redis.RedisCluster | redis.Redis): + return ( + client.info("search")["search_background_indexing_status"] == "NO_ACTIVITY" + ) + + def wait_for_empty_db(self, client: redis.RedisCluster | redis.Redis): + return client.execute_command("DBSIZE") == 0 + + def search_embedding( + self, + query: list[float], + k: int = 10, + filters: dict | None = None, + timeout: int | None = None, + **kwargs: Any, + ) -> (list[int]): + assert self.conn is not None + + query_vector = np.array(query).astype(np.float32).tobytes() + query_obj = Query(f"*=>[KNN {k} @vector $vec]").return_fields("id").paging(0, k) + query_params = {"vec": query_vector} + + if filters: + # benchmark test filters of format: {'metadata': '>=10000', 'id': 10000} + # gets exact match for id, and range for metadata if they exist in filters + id_value = filters.get("id") + # Removing '>=' from the id_value: '>=10000' + metadata_value = filters.get("metadata")[2:] + if id_value and metadata_value: + query_obj = Query(f"(@metadata:[{metadata_value} +inf] @id:{ {id_value} })=>[KNN {k} @vector $vec]").return_fields("id").paging(0, k) + elif id_value: + #gets exact match for id + query_obj = Query(f"@id:{ {id_value} }=>[KNN {k} @vector $vec]").return_fields("id").paging(0, k) + else: #metadata only case, greater than or equal to metadata value + query_obj = Query(f"@metadata:[{metadata_value} +inf]=>[KNN {k} @vector $vec]").return_fields("id").paging(0, k) + res = self.conn.ft(INDEX_NAME).search(query_obj, query_params) + return [int(doc["id"]) for doc in res.docs] \ No newline at end of file diff --git a/vectordb_bench/cli/vectordbbench.py b/vectordb_bench/cli/vectordbbench.py index 0b619bbec..4645176fa 100644 --- a/vectordb_bench/cli/vectordbbench.py +++ b/vectordb_bench/cli/vectordbbench.py @@ -1,5 +1,6 @@ from ..backend.clients.pgvector.cli import PgVectorHNSW from ..backend.clients.redis.cli import Redis +from ..backend.clients.memorydb.cli import MemoryDB from ..backend.clients.test.cli import Test from ..backend.clients.weaviate_cloud.cli import Weaviate from ..backend.clients.zilliz_cloud.cli import ZillizAutoIndex @@ -11,6 +12,7 @@ cli.add_command(PgVectorHNSW) cli.add_command(Redis) +cli.add_command(MemoryDB) cli.add_command(Weaviate) cli.add_command(Test) cli.add_command(ZillizAutoIndex) From 36c74eed59884d1b1af9af58f847b5ef91d2fb9f Mon Sep 17 00:00:00 2001 From: cutecutecat Date: Thu, 1 Aug 2024 11:11:13 +0800 Subject: [PATCH 076/327] refactor: migrate to new pgvecto_rs sdk (#353) Signed-off-by: cutecutecat --- install/requirements_py3.11.txt | 1 + pyproject.toml | 4 +- .../backend/clients/pgvecto_rs/cli.py | 154 +++++++++++++ .../backend/clients/pgvecto_rs/config.py | 181 +++++++++------ .../backend/clients/pgvecto_rs/pgvecto_rs.py | 218 +++++++++++++----- vectordb_bench/cli/vectordbbench.py | 3 + .../frontend/config/dbCaseConfigs.py | 59 ++++- vectordb_bench/models.py | 4 +- 8 files changed, 479 insertions(+), 145 deletions(-) create mode 100644 vectordb_bench/backend/clients/pgvecto_rs/cli.py diff --git a/install/requirements_py3.11.txt b/install/requirements_py3.11.txt index c55601934..c3a3bbbda 100644 --- a/install/requirements_py3.11.txt +++ b/install/requirements_py3.11.txt @@ -5,6 +5,7 @@ pinecone-client weaviate-client elasticsearch pgvector +pgvecto_rs[psycopg3]>=0.2.1 sqlalchemy redis chromadb diff --git a/pyproject.toml b/pyproject.toml index 330867d56..de60cc349 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,10 +56,10 @@ all = [ "weaviate-client", "elasticsearch", "pgvector", + "pgvecto_rs[psycopg3]>=0.2.1", "sqlalchemy", "redis", "chromadb", - "psycopg2", "psycopg", "psycopg-binary", "opensearch-dsl==2.1.0", @@ -71,7 +71,7 @@ pinecone = [ "pinecone-client" ] weaviate = [ "weaviate-client" ] elastic = [ "elasticsearch" ] pgvector = [ "psycopg", "psycopg-binary", "pgvector" ] -pgvecto_rs = [ "psycopg2" ] +pgvecto_rs = [ "pgvecto_rs[psycopg3]>=0.2.1" ] redis = [ "redis" ] memorydb = [ "memorydb" ] chromadb = [ "chromadb" ] diff --git a/vectordb_bench/backend/clients/pgvecto_rs/cli.py b/vectordb_bench/backend/clients/pgvecto_rs/cli.py new file mode 100644 index 000000000..10dbff556 --- /dev/null +++ b/vectordb_bench/backend/clients/pgvecto_rs/cli.py @@ -0,0 +1,154 @@ +from typing import Annotated, Optional, Unpack + +import click +import os +from pydantic import SecretStr + +from ....cli.cli import ( + CommonTypedDict, + HNSWFlavor1, + IVFFlatTypedDict, + cli, + click_parameter_decorators_from_typed_dict, + run, +) +from vectordb_bench.backend.clients import DB + + +class PgVectoRSTypedDict(CommonTypedDict): + user_name: Annotated[ + str, click.option("--user-name", type=str, help="Db username", required=True) + ] + password: Annotated[ + str, + click.option( + "--password", + type=str, + help="Postgres database password", + default=lambda: os.environ.get("POSTGRES_PASSWORD", ""), + show_default="$POSTGRES_PASSWORD", + ), + ] + + host: Annotated[ + str, click.option("--host", type=str, help="Db host", required=True) + ] + db_name: Annotated[ + str, click.option("--db-name", type=str, help="Db name", required=True) + ] + max_parallel_workers: Annotated[ + Optional[int], + click.option( + "--max-parallel-workers", + type=int, + help="Sets the maximum number of parallel processes per maintenance operation (index creation)", + required=False, + ), + ] + quantization_type: Annotated[ + str, + click.option( + "--quantization-type", + type=click.Choice(["trivial", "scalar", "product"]), + help="quantization type for vectors", + required=False, + ), + ] + quantization_ratio: Annotated[ + str, + click.option( + "--quantization-ratio", + type=click.Choice(["x4", "x8", "x16", "x32", "x64"]), + help="quantization ratio(for product quantization)", + required=False, + ), + ] + + +class PgVectoRSFlatTypedDict(PgVectoRSTypedDict, IVFFlatTypedDict): ... + + +@cli.command() +@click_parameter_decorators_from_typed_dict(PgVectoRSFlatTypedDict) +def PgVectoRSFlat( + **parameters: Unpack[PgVectoRSFlatTypedDict], +): + from .config import PgVectoRSConfig, PgVectoRSFLATConfig + + run( + db=DB.PgVectoRS, + db_config=PgVectoRSConfig( + db_label=parameters["db_label"], + user_name=SecretStr(parameters["user_name"]), + password=SecretStr(parameters["password"]), + host=parameters["host"], + db_name=parameters["db_name"], + ), + db_case_config=PgVectoRSFLATConfig( + max_parallel_workers=parameters["max_parallel_workers"], + quantization_type=parameters["quantization_type"], + quantization_ratio=parameters["quantization_ratio"], + ), + **parameters, + ) + + +class PgVectoRSIVFFlatTypedDict(PgVectoRSTypedDict, IVFFlatTypedDict): ... + + +@cli.command() +@click_parameter_decorators_from_typed_dict(PgVectoRSIVFFlatTypedDict) +def PgVectoRSIVFFlat( + **parameters: Unpack[PgVectoRSIVFFlatTypedDict], +): + from .config import PgVectoRSConfig, PgVectoRSIVFFlatConfig + + run( + db=DB.PgVectoRS, + db_config=PgVectoRSConfig( + db_label=parameters["db_label"], + user_name=SecretStr(parameters["user_name"]), + password=SecretStr(parameters["password"]), + host=parameters["host"], + db_name=parameters["db_name"], + ), + db_case_config=PgVectoRSIVFFlatConfig( + max_parallel_workers=parameters["max_parallel_workers"], + quantization_type=parameters["quantization_type"], + quantization_ratio=parameters["quantization_ratio"], + probes=parameters["probes"], + lists=parameters["lists"], + ), + **parameters, + ) + + +class PgVectoRSHNSWTypedDict(PgVectoRSTypedDict, HNSWFlavor1): ... + + +@cli.command() +@click_parameter_decorators_from_typed_dict(PgVectoRSHNSWTypedDict) +def PgVectoRSHNSW( + **parameters: Unpack[PgVectoRSHNSWTypedDict], +): + from .config import PgVectoRSConfig, PgVectoRSHNSWConfig + + run( + db=DB.PgVectoRS, + db_config=PgVectoRSConfig( + db_label=parameters["db_label"], + user_name=SecretStr(parameters["user_name"]), + password=SecretStr(parameters["password"]), + host=parameters["host"], + db_name=parameters["db_name"], + ), + db_case_config=PgVectoRSHNSWConfig( + max_parallel_workers=parameters["max_parallel_workers"], + quantization_type=parameters["quantization_type"], + quantization_ratio=parameters["quantization_ratio"], + m=parameters["m"], + ef_construction=parameters["ef_construction"], + ef_search=parameters["ef_search"], + ), + **parameters, + ) diff --git a/vectordb_bench/backend/clients/pgvecto_rs/config.py b/vectordb_bench/backend/clients/pgvecto_rs/config.py index 73c41c239..c671a236c 100644 --- a/vectordb_bench/backend/clients/pgvecto_rs/config.py +++ b/vectordb_bench/backend/clients/pgvecto_rs/config.py @@ -1,30 +1,53 @@ -from typing import Literal +from abc import abstractmethod +from typing import TypedDict + from pydantic import BaseModel, SecretStr -from ..api import DBConfig, DBCaseConfig, MetricType, IndexType +from pgvecto_rs.types import IndexOption, Ivf, Hnsw, Flat, Quantization +from pgvecto_rs.types.index import QuantizationType, QuantizationRatio + +from ..api import DBConfig, DBCaseConfig, IndexType, MetricType POSTGRE_URL_PLACEHOLDER = "postgresql://%s:%s@%s/%s" +class PgVectorRSConfigDict(TypedDict): + """These keys will be directly used as kwargs in psycopg connection string, + so the names must match exactly psycopg API""" + + user: str + password: str + host: str + port: int + dbname: str + + class PgVectoRSConfig(DBConfig): - user_name: SecretStr = "postgres" + user_name: str = "postgres" password: SecretStr host: str = "localhost" port: int = 5432 db_name: str def to_dict(self) -> dict: - user_str = self.user_name.get_secret_value() + user_str = self.user_name pwd_str = self.password.get_secret_value() return { "host": self.host, "port": self.port, "dbname": self.db_name, "user": user_str, - "password": pwd_str + "password": pwd_str, } + class PgVectoRSIndexConfig(BaseModel, DBCaseConfig): metric_type: MetricType | None = None + create_index_before_load: bool = False + create_index_after_load: bool = True + + max_parallel_workers: int | None = None + quantization_type: QuantizationType | None = None + quantization_ratio: QuantizationRatio | None = None def parse_metric(self) -> str: if self.metric_type == MetricType.L2: @@ -40,88 +63,100 @@ def parse_metric_fun_op(self) -> str: return "<#>" return "<=>" -class PgVectoRSQuantConfig(PgVectoRSIndexConfig): - quantizationType: Literal["trivial", "scalar", "product"] - quantizationRatio: None | Literal["x4", "x8", "x16", "x32", "x64"] - - def parse_quantization(self) -> str: - if self.quantizationType == "trivial": - return "quantization = { trivial = { } }" - elif self.quantizationType == "scalar": - return "quantization = { scalar = { } }" - else: - return f'quantization = {{ product = {{ ratio = "{self.quantizationRatio}" }} }}' - + def search_param(self) -> dict: + return { + "metric_fun_op": self.parse_metric_fun_op(), + } -class HNSWConfig(PgVectoRSQuantConfig): - M: int - efConstruction: int - index: IndexType = IndexType.HNSW + @abstractmethod + def index_param(self) -> dict[str, str]: ... - def index_param(self) -> dict: - options = f""" -[indexing.hnsw] -m = {self.M} -ef_construction = {self.efConstruction} -{self.parse_quantization()} -""" - return {"options": options, "metric": self.parse_metric()} + @abstractmethod + def session_param(self) -> dict[str, str | int]: ... - def search_param(self) -> dict: - return {"metrics_op": self.parse_metric_fun_op()} +class PgVectoRSHNSWConfig(PgVectoRSIndexConfig): + index: IndexType = IndexType.HNSW + m: int | None = None + ef_search: int | None + ef_construction: int | None = None -class IVFFlatConfig(PgVectoRSQuantConfig): - nlist: int - nprobe: int | None = None + def index_param(self) -> dict[str, str]: + if self.quantization_type is None: + quantization = None + else: + quantization = Quantization( + typ=self.quantization_type, ratio=self.quantization_ratio + ) + + option = IndexOption( + index=Hnsw( + m=self.m, + ef_construction=self.ef_construction, + quantization=quantization, + ), + threads=self.max_parallel_workers, + ) + return {"options": option.dumps(), "metric": self.parse_metric()} + + def session_param(self) -> dict[str, str | int]: + session_parameters = {} + if self.ef_search is not None: + session_parameters["vectors.hnsw_ef_search"] = str(self.ef_search) + return session_parameters + + +class PgVectoRSIVFFlatConfig(PgVectoRSIndexConfig): index: IndexType = IndexType.IVFFlat + probes: int | None + lists: int | None + + def index_param(self) -> dict[str, str]: + if self.quantization_type is None: + quantization = None + else: + quantization = Quantization( + typ=self.quantization_type, ratio=self.quantization_ratio + ) - def index_param(self) -> dict: - options = f""" -[indexing.ivf] -nlist = {self.nlist} -nsample = {self.nprobe if self.nprobe else 10} -{self.parse_quantization()} -""" - return {"options": options, "metric": self.parse_metric()} + option = IndexOption( + index=Ivf(nlist=self.lists, quantization=quantization), + threads=self.max_parallel_workers, + ) + return {"options": option.dumps(), "metric": self.parse_metric()} - def search_param(self) -> dict: - return {"metrics_op": self.parse_metric_fun_op()} - -class IVFFlatSQ8Config(PgVectoRSIndexConfig): - nlist: int - nprobe: int | None = None - index: IndexType = IndexType.IVFSQ8 - - def index_param(self) -> dict: - options = f""" -[indexing.ivf] -nlist = {self.nlist} -nsample = {self.nprobe if self.nprobe else 10} -quantization = {{ scalar = {{ }} }} -""" - return {"options": options, "metric": self.parse_metric()} + def session_param(self) -> dict[str, str | int]: + session_parameters = {} + if self.probes is not None: + session_parameters["vectors.ivf_nprobe"] = str(self.probes) + return session_parameters - def search_param(self) -> dict: - return {"metrics_op": self.parse_metric_fun_op()} -class FLATConfig(PgVectoRSQuantConfig): +class PgVectoRSFLATConfig(PgVectoRSIndexConfig): index: IndexType = IndexType.Flat - def index_param(self) -> dict: - options = f""" -[indexing.flat] -{self.parse_quantization()} -""" - return {"options": options, "metric": self.parse_metric()} + def index_param(self) -> dict[str, str]: + if self.quantization_type is None: + quantization = None + else: + quantization = Quantization( + typ=self.quantization_type, ratio=self.quantization_ratio + ) - def search_param(self) -> dict: - return {"metrics_op": self.parse_metric_fun_op()} + option = IndexOption( + index=Flat( + quantization=quantization, + ), + threads=self.max_parallel_workers, + ) + return {"options": option.dumps(), "metric": self.parse_metric()} + + def session_param(self) -> dict[str, str | int]: + return {} _pgvecto_rs_case_config = { - IndexType.HNSW: HNSWConfig, - IndexType.IVFFlat: IVFFlatConfig, - IndexType.IVFSQ8: IVFFlatSQ8Config, - IndexType.Flat: FLATConfig, + IndexType.HNSW: PgVectoRSHNSWConfig, + IndexType.IVFFlat: PgVectoRSIVFFlatConfig, + IndexType.Flat: PgVectoRSFLATConfig, } diff --git a/vectordb_bench/backend/clients/pgvecto_rs/pgvecto_rs.py b/vectordb_bench/backend/clients/pgvecto_rs/pgvecto_rs.py index 22caa43e6..bc042cc57 100644 --- a/vectordb_bench/backend/clients/pgvecto_rs/pgvecto_rs.py +++ b/vectordb_bench/backend/clients/pgvecto_rs/pgvecto_rs.py @@ -1,73 +1,138 @@ """Wrapper around the Pgvecto.rs vector database over VectorDB""" -import io import logging +import pprint from contextlib import contextmanager -from typing import Any -import pandas as pd -import psycopg2 -import psycopg2.extras +from typing import Any, Generator, Optional, Tuple -from ..api import VectorDB, DBCaseConfig +import numpy as np +import psycopg +from psycopg import Connection, Cursor, sql +from pgvecto_rs.psycopg import register_vector + +from ..api import VectorDB +from .config import PgVectoRSConfig, PgVectoRSIndexConfig log = logging.getLogger(__name__) + class PgVectoRS(VectorDB): - """Use SQLAlchemy instructions""" + """Use psycopg instructions""" + + conn: psycopg.Connection[Any] | None = None + cursor: psycopg.Cursor[Any] | None = None + _unfiltered_search: sql.Composed + _filtered_search: sql.Composed def __init__( self, dim: int, - db_config: dict, - db_case_config: DBCaseConfig, - collection_name: str = "PgVectorCollection", + db_config: PgVectoRSConfig, + db_case_config: PgVectoRSIndexConfig, + collection_name: str = "PgVectoRSCollection", drop_old: bool = False, **kwargs, ): + + self.name = "PgVectorRS" self.db_config = db_config self.case_config = db_case_config self.table_name = collection_name self.dim = dim - self._index_name = "pqvector_index" + self._index_name = "pgvectors_index" self._primary_field = "id" self._vector_field = "embedding" # construct basic units - self.conn = psycopg2.connect(**self.db_config) - self.conn.autocommit = False - self.cursor = self.conn.cursor() + self.conn, self.cursor = self._create_connection(**self.db_config) - # create vector extension - self.cursor.execute("CREATE EXTENSION IF NOT EXISTS vectors") - self.conn.commit() + log.info(f"{self.name} config values: {self.db_config}\n{self.case_config}") + if not any( + ( + self.case_config.create_index_before_load, + self.case_config.create_index_after_load, + ) + ): + err = f"{self.name} config must create an index using create_index_before_load or create_index_after_load" + log.error(err) + raise RuntimeError( + f"{err}\n{pprint.pformat(self.db_config)}\n{pprint.pformat(self.case_config)}" + ) if drop_old: log.info(f"Pgvecto.rs client drop table : {self.table_name}") self._drop_index() self._drop_table() self._create_table(dim) - self._create_index() + if self.case_config.create_index_before_load: + self._create_index() self.cursor.close() self.conn.close() self.cursor = None self.conn = None + @staticmethod + def _create_connection(**kwargs) -> Tuple[Connection, Cursor]: + conn = psycopg.connect(**kwargs) + + # create vector extension + conn.execute("CREATE EXTENSION IF NOT EXISTS vectors") + conn.commit() + register_vector(conn) + + conn.autocommit = False + cursor = conn.cursor() + + assert conn is not None, "Connection is not initialized" + assert cursor is not None, "Cursor is not initialized" + + return conn, cursor + @contextmanager - def init(self) -> None: + def init(self) -> Generator[None, None, None]: """ Examples: >>> with self.init(): >>> self.insert_embeddings() >>> self.search_embedding() """ - self.conn = psycopg2.connect(**self.db_config) - self.conn.autocommit = False - self.cursor = self.conn.cursor() - self.cursor.execute('SET search_path = "$user", public, vectors') + + self.conn, self.cursor = self._create_connection(**self.db_config) + + # index configuration may have commands defined that we should set during each client session + session_options = self.case_config.session_param() + + for key, val in session_options.items(): + command = sql.SQL("SET {setting_name} " + "= {val};").format( + setting_name=sql.Identifier(key), + val=val, + ) + log.debug(command.as_string(self.cursor)) + self.cursor.execute(command) self.conn.commit() + self._filtered_search = sql.Composed( + [ + sql.SQL( + "SELECT id FROM public.{table_name} WHERE id >= %s ORDER BY embedding " + ).format(table_name=sql.Identifier(self.table_name)), + sql.SQL(self.case_config.search_param()["metric_fun_op"]), + sql.SQL(" %s::vector LIMIT %s::int"), + ] + ) + + self._unfiltered_search = sql.Composed( + [ + sql.SQL( + "SELECT id FROM public.{table_name} ORDER BY embedding " + ).format(table_name=sql.Identifier(self.table_name)), + sql.SQL(self.case_config.search_param()["metric_fun_op"]), + sql.SQL(" %s::vector LIMIT %s::int"), + ] + ) + try: yield finally: @@ -79,42 +144,65 @@ def init(self) -> None: def _drop_table(self): assert self.conn is not None, "Connection is not initialized" assert self.cursor is not None, "Cursor is not initialized" + log.info(f"{self.name} client drop table : {self.table_name}") - self.cursor.execute(f'DROP TABLE IF EXISTS public."{self.table_name}"') + self.cursor.execute( + sql.SQL("DROP TABLE IF EXISTS public.{table_name}").format( + table_name=sql.Identifier(self.table_name) + ) + ) self.conn.commit() def ready_to_load(self): pass def optimize(self): - pass + self._post_insert() - def ready_to_search(self): - pass + def _post_insert(self): + log.info(f"{self.name} post insert before optimize") + if self.case_config.create_index_after_load: + self._drop_index() + self._create_index() def _drop_index(self): assert self.conn is not None, "Connection is not initialized" assert self.cursor is not None, "Cursor is not initialized" + log.info(f"{self.name} client drop index : {self._index_name}") - self.cursor.execute(f'DROP INDEX IF EXISTS "{self._index_name}"') + drop_index_sql = sql.SQL("DROP INDEX IF EXISTS {index_name}").format( + index_name=sql.Identifier(self._index_name) + ) + log.debug(drop_index_sql.as_string(self.cursor)) + self.cursor.execute(drop_index_sql) self.conn.commit() def _create_index(self): assert self.conn is not None, "Connection is not initialized" assert self.cursor is not None, "Cursor is not initialized" + log.info(f"{self.name} client create index : {self._index_name}") index_param = self.case_config.index_param() + index_create_sql = sql.SQL( + """ + CREATE INDEX IF NOT EXISTS {index_name} ON public.{table_name} + USING vectors (embedding {embedding_metric}) WITH (options = {index_options}) + """ + ).format( + index_name=sql.Identifier(self._index_name), + table_name=sql.Identifier(self.table_name), + embedding_metric=sql.Identifier(index_param["metric"]), + index_options=index_param["options"], + ) try: - # create table - self.cursor.execute( - f'CREATE INDEX IF NOT EXISTS {self._index_name} ON public."{self.table_name}" \ - USING vectors (embedding {index_param["metric"]}) WITH (options = $${index_param["options"]}$$);' - ) + log.debug(index_create_sql.as_string(self.cursor)) + self.cursor.execute(index_create_sql) self.conn.commit() except Exception as e: log.warning( - f"Failed to create pgvecto.rs table: {self.table_name} error: {e}" + f"Failed to create pgvecto.rs index {self._index_name} \ + at table {self.table_name} error: {e}" ) raise e from None @@ -122,12 +210,18 @@ def _create_table(self, dim: int): assert self.conn is not None, "Connection is not initialized" assert self.cursor is not None, "Cursor is not initialized" + table_create_sql = sql.SQL( + """ + CREATE TABLE IF NOT EXISTS public.{table_name} + (id BIGINT PRIMARY KEY, embedding vector({dim})) + """ + ).format( + table_name=sql.Identifier(self.table_name), + dim=dim, + ) try: # create table - self.cursor.execute( - f'CREATE TABLE IF NOT EXISTS public."{self.table_name}" \ - (id Integer PRIMARY KEY, embedding vector({dim}));' - ) + self.cursor.execute(table_create_sql) self.conn.commit() except Exception as e: log.warning( @@ -140,7 +234,7 @@ def insert_embeddings( embeddings: list[list[float]], metadata: list[int], **kwargs: Any, - ) -> (int, Exception): + ) -> Tuple[int, Optional[Exception]]: assert self.conn is not None, "Connection is not initialized" assert self.cursor is not None, "Cursor is not initialized" @@ -148,19 +242,27 @@ def insert_embeddings( assert self.cursor is not None, "Cursor is not initialized" try: - items = { - "id": metadata, - "embedding": embeddings - } - df = pd.DataFrame(items) - csv_buffer = io.StringIO() - df.to_csv(csv_buffer, index=False, header=False) - csv_buffer.seek(0) - self.cursor.copy_expert(f"COPY public.\"{self.table_name}\" FROM STDIN WITH (FORMAT CSV)", csv_buffer) + metadata_arr = np.array(metadata) + embeddings_arr = np.array(embeddings) + + with self.cursor.copy( + sql.SQL("COPY public.{table_name} FROM STDIN (FORMAT BINARY)").format( + table_name=sql.Identifier(self.table_name) + ) + ) as copy: + copy.set_types(["bigint", "vector"]) + for i, row in enumerate(metadata_arr): + copy.write_row((row, embeddings_arr[i])) self.conn.commit() + + if kwargs.get("last_batch"): + self._post_insert() + return len(metadata), None except Exception as e: - log.warning(f"Failed to insert data into pgvecto.rs table ({self.table_name}), error: {e}") + log.warning( + f"Failed to insert data into pgvecto.rs table ({self.table_name}), error: {e}" + ) return 0, e def search_embedding( @@ -173,20 +275,18 @@ def search_embedding( assert self.conn is not None, "Connection is not initialized" assert self.cursor is not None, "Cursor is not initialized" - search_param = self.case_config.search_param() + q = np.asarray(query) if filters: + log.debug(self._filtered_search.as_string(self.cursor)) gt = filters.get("id") - self.cursor.execute( - f"SELECT id FROM (SELECT * FROM public.\"{self.table_name}\" ORDER BY embedding \ - {search_param['metrics_op']} '{query}' LIMIT {k}) AS X WHERE id > {gt} ;" + result = self.cursor.execute( + self._filtered_search, (gt, q, k), prepare=True, binary=True ) else: - self.cursor.execute( - f"SELECT id FROM public.\"{self.table_name}\" ORDER BY embedding \ - {search_param['metrics_op']} '{query}' LIMIT {k};" + log.debug(self._unfiltered_search.as_string(self.cursor)) + result = self.cursor.execute( + self._unfiltered_search, (q, k), prepare=True, binary=True ) - self.conn.commit() - result = self.cursor.fetchall() - return [int(i[0]) for i in result] + return [int(i[0]) for i in result.fetchall()] diff --git a/vectordb_bench/cli/vectordbbench.py b/vectordb_bench/cli/vectordbbench.py index 4645176fa..ab7eff44c 100644 --- a/vectordb_bench/cli/vectordbbench.py +++ b/vectordb_bench/cli/vectordbbench.py @@ -1,4 +1,5 @@ from ..backend.clients.pgvector.cli import PgVectorHNSW +from ..backend.clients.pgvecto_rs.cli import PgVectoRSHNSW, PgVectoRSIVFFlat from ..backend.clients.redis.cli import Redis from ..backend.clients.memorydb.cli import MemoryDB from ..backend.clients.test.cli import Test @@ -11,6 +12,8 @@ from .cli import cli cli.add_command(PgVectorHNSW) +cli.add_command(PgVectoRSHNSW) +cli.add_command(PgVectoRSIVFFlat) cli.add_command(Redis) cli.add_command(MemoryDB) cli.add_command(Weaviate) diff --git a/vectordb_bench/frontend/config/dbCaseConfigs.py b/vectordb_bench/frontend/config/dbCaseConfigs.py index ce8a3a4ae..687f1efbf 100644 --- a/vectordb_bench/frontend/config/dbCaseConfigs.py +++ b/vectordb_bench/frontend/config/dbCaseConfigs.py @@ -190,6 +190,19 @@ class CaseConfigInput(BaseModel): }, ) +CaseConfigParamInput_IndexType_PgVectoRS = CaseConfigInput( + label=CaseConfigParamType.IndexType, + inputHelp="Select Index Type", + inputType=InputType.Option, + inputConfig={ + "options": [ + IndexType.HNSW.value, + IndexType.IVFFlat.value, + IndexType.Flat.value, + ], + }, +) + CaseConfigParamInput_M = CaseConfigInput( label=CaseConfigParamType.M, inputType=InputType.Number, @@ -272,14 +285,26 @@ class CaseConfigInput(BaseModel): CaseConfigParamInput_EFConstruction_PgVectoRS = CaseConfigInput( - label=CaseConfigParamType.EFConstruction, + label=CaseConfigParamType.ef_construction, inputType=InputType.Number, inputConfig={ - "min": 8, - "max": 512, - "value": 360, + "min": 10, + "max": 2000, + "value": 300, }, - isDisplayed=lambda config: config[CaseConfigParamType.IndexType] + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) + == IndexType.HNSW.value, +) + +CaseConfigParamInput_EFSearch_PgVectoRS = CaseConfigInput( + label=CaseConfigParamType.ef_search, + inputType=InputType.Number, + inputConfig={ + "min": 1, + "max": 65535, + "value": 100, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.HNSW.value, ) @@ -598,6 +623,7 @@ class CaseConfigInput(BaseModel): == IndexType.HNSW.value, ) + CaseConfigParamInput_QuantizationType_PgVectoRS = CaseConfigInput( label=CaseConfigParamType.quantizationType, inputType=InputType.Option, @@ -626,6 +652,18 @@ class CaseConfigInput(BaseModel): ], ) +CaseConfigParamInput_max_parallel_workers_PgVectorRS = CaseConfigInput( + label=CaseConfigParamType.max_parallel_workers, + displayLabel="Max parallel workers", + inputHelp="Recommended value: (cpu cores - 1). This will set the parameters: [optimizing.optimizing_threads]", + inputType=InputType.Number, + inputConfig={ + "min": 0, + "max": 1024, + "value": 16, + }, +) + CaseConfigParamInput_ZillizLevel = CaseConfigInput( label=CaseConfigParamType.level, inputType=InputType.Number, @@ -707,22 +745,25 @@ class CaseConfigInput(BaseModel): ] PgVectoRSLoadingConfig = [ - CaseConfigParamInput_IndexType, - CaseConfigParamInput_M, + CaseConfigParamInput_IndexType_PgVectoRS, + CaseConfigParamInput_m, CaseConfigParamInput_EFConstruction_PgVectoRS, CaseConfigParamInput_Nlist, CaseConfigParamInput_QuantizationType_PgVectoRS, CaseConfigParamInput_QuantizationRatio_PgVectoRS, + CaseConfigParamInput_max_parallel_workers_PgVectorRS, ] PgVectoRSPerformanceConfig = [ - CaseConfigParamInput_IndexType, - CaseConfigParamInput_M, + CaseConfigParamInput_IndexType_PgVectoRS, + CaseConfigParamInput_m, CaseConfigParamInput_EFConstruction_PgVectoRS, + CaseConfigParamInput_EFSearch_PgVectoRS, CaseConfigParamInput_Nlist, CaseConfigParamInput_Nprobe, CaseConfigParamInput_QuantizationType_PgVectoRS, CaseConfigParamInput_QuantizationRatio_PgVectoRS, + CaseConfigParamInput_max_parallel_workers_PgVectorRS, ] ZillizCloudPerformanceConfig = [ diff --git a/vectordb_bench/models.py b/vectordb_bench/models.py index 41be95b7c..bc99e8595 100644 --- a/vectordb_bench/models.py +++ b/vectordb_bench/models.py @@ -45,8 +45,8 @@ class CaseConfigParamType(Enum): numCandidates = "num_candidates" lists = "lists" probes = "probes" - quantizationType = "quantizationType" - quantizationRatio = "quantizationRatio" + quantizationType = "quantization_type" + quantizationRatio = "quantization_ratio" m = "m" nbits = "nbits" intermediate_graph_degree = "intermediate_graph_degree" From 3ff12b4bc9876c1c9871ef7412b7b7adf0c1199a Mon Sep 17 00:00:00 2001 From: Sheharyar Ahmad Date: Thu, 1 Aug 2024 13:00:09 +0500 Subject: [PATCH 077/327] Added pgvectorscale client (#355) * pgvectorscale client added * added pgvectorscale dependencies to enable independent client installation * Bug fix vector type not found in the database. --- README.md | 1 + pyproject.toml | 1 + vectordb_bench/backend/clients/__init__.py | 13 + vectordb_bench/backend/clients/api.py | 1 + .../backend/clients/pgvectorscale/config.py | 111 +++++++ .../clients/pgvectorscale/pgvectorscale.py | 272 ++++++++++++++++++ .../components/run_test/caseSelector.py | 10 + .../frontend/config/dbCaseConfigs.py | 123 ++++++++ vectordb_bench/models.py | 8 + 9 files changed, 540 insertions(+) create mode 100644 vectordb_bench/backend/clients/pgvectorscale/config.py create mode 100644 vectordb_bench/backend/clients/pgvectorscale/pgvectorscale.py diff --git a/README.md b/README.md index 11aaed3b7..1ce4564ef 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,7 @@ All the database client supported | elastic | `pip install vectordb-bench[elastic]` | | pgvector | `pip install vectordb-bench[pgvector]` | | pgvecto.rs | `pip install vectordb-bench[pgvecto_rs]` | +| pgvectorscale | `pip install vectordb-bench[pgvectorscale]` | | redis | `pip install vectordb-bench[redis]` | | memorydb | `pip install vectordb-bench[memorydb]` | | chromadb | `pip install vectordb-bench[chromadb]` | diff --git a/pyproject.toml b/pyproject.toml index de60cc349..4da9ce2cc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,6 +71,7 @@ pinecone = [ "pinecone-client" ] weaviate = [ "weaviate-client" ] elastic = [ "elasticsearch" ] pgvector = [ "psycopg", "psycopg-binary", "pgvector" ] +pgvectorscale = [ "psycopg", "psycopg-binary", "pgvector" ] pgvecto_rs = [ "pgvecto_rs[psycopg3]>=0.2.1" ] redis = [ "redis" ] memorydb = [ "memorydb" ] diff --git a/vectordb_bench/backend/clients/__init__.py b/vectordb_bench/backend/clients/__init__.py index c638208d9..3e87e1fbe 100644 --- a/vectordb_bench/backend/clients/__init__.py +++ b/vectordb_bench/backend/clients/__init__.py @@ -30,6 +30,7 @@ class DB(Enum): WeaviateCloud = "WeaviateCloud" PgVector = "PgVector" PgVectoRS = "PgVectoRS" + PgVectorScale = "PgVectorScale" Redis = "Redis" MemoryDB = "MemoryDB" Chroma = "Chroma" @@ -71,6 +72,10 @@ def init_cls(self) -> Type[VectorDB]: if self == DB.PgVectoRS: from .pgvecto_rs.pgvecto_rs import PgVectoRS return PgVectoRS + + if self == DB.PgVectorScale: + from .pgvectorscale.pgvectorscale import PgVectorScale + return PgVectorScale if self == DB.Redis: from .redis.redis import Redis @@ -123,6 +128,10 @@ def config_cls(self) -> Type[DBConfig]: from .pgvecto_rs.config import PgVectoRSConfig return PgVectoRSConfig + if self == DB.PgVectorScale: + from .pgvectorscale.config import PgVectorScaleConfig + return PgVectorScaleConfig + if self == DB.Redis: from .redis.config import RedisConfig return RedisConfig @@ -172,6 +181,10 @@ def case_config_cls(self, index_type: IndexType | None = None) -> Type[DBCaseCon from .aws_opensearch.config import AWSOpenSearchIndexConfig return AWSOpenSearchIndexConfig + if self == DB.PgVectorScale: + from .pgvectorscale.config import _pgvectorscale_case_config + return _pgvectorscale_case_config.get(index_type) + # DB.Pinecone, DB.Chroma, DB.Redis return EmptyDBCaseConfig diff --git a/vectordb_bench/backend/clients/api.py b/vectordb_bench/backend/clients/api.py index d9ec5d83b..faa36712d 100644 --- a/vectordb_bench/backend/clients/api.py +++ b/vectordb_bench/backend/clients/api.py @@ -15,6 +15,7 @@ class MetricType(str, Enum): class IndexType(str, Enum): HNSW = "HNSW" DISKANN = "DISKANN" + STREAMING_DISKANN = "DISKANN" IVFFlat = "IVF_FLAT" IVFSQ8 = "IVF_SQ8" Flat = "FLAT" diff --git a/vectordb_bench/backend/clients/pgvectorscale/config.py b/vectordb_bench/backend/clients/pgvectorscale/config.py new file mode 100644 index 000000000..bd9f6106b --- /dev/null +++ b/vectordb_bench/backend/clients/pgvectorscale/config.py @@ -0,0 +1,111 @@ +from abc import abstractmethod +from typing import TypedDict +from pydantic import BaseModel, SecretStr +from typing_extensions import LiteralString +from ..api import DBCaseConfig, DBConfig, IndexType, MetricType + +POSTGRE_URL_PLACEHOLDER = "postgresql://%s:%s@%s/%s" + + +class PgVectorScaleConfigDict(TypedDict): + """These keys will be directly used as kwargs in psycopg connection string, + so the names must match exactly psycopg API""" + + user: str + password: str + host: str + port: int + dbname: str + + +class PgVectorScaleConfig(DBConfig): + user_name: SecretStr = SecretStr("postgres") + password: SecretStr + host: str = "localhost" + port: int = 5432 + db_name: str + + def to_dict(self) -> PgVectorScaleConfigDict: + user_str = self.user_name.get_secret_value() + pwd_str = self.password.get_secret_value() + return { + "host": self.host, + "port": self.port, + "dbname": self.db_name, + "user": user_str, + "password": pwd_str, + } + + +class PgVectorScaleIndexConfig(BaseModel, DBCaseConfig): + metric_type: MetricType | None = None + create_index_before_load: bool = False + create_index_after_load: bool = True + + def parse_metric(self) -> str: + if self.metric_type == MetricType.COSINE: + return "vector_cosine_ops" + return "" + + def parse_metric_fun_op(self) -> LiteralString: + if self.metric_type == MetricType.COSINE: + return "<=>" + return "" + + def parse_metric_fun_str(self) -> str: + if self.metric_type == MetricType.COSINE: + return "cosine_distance" + return "" + + @abstractmethod + def index_param(self) -> dict: + ... + + @abstractmethod + def search_param(self) -> dict: + ... + + @abstractmethod + def session_param(self) -> dict: + ... + + +class PgVectorScaleStreamingDiskANNConfig(PgVectorScaleIndexConfig): + index: IndexType = IndexType.STREAMING_DISKANN + storage_layout: str | None + num_neighbors: int | None + search_list_size: int | None + max_alpha: float | None + num_dimensions: int | None + num_bits_per_dimension: int | None + query_search_list_size: int | None + query_rescore: int | None + + def index_param(self) -> dict: + return { + "metric": self.parse_metric(), + "index_type": self.index.value, + "options": { + "storage_layout": self.storage_layout, + "num_neighbors": self.num_neighbors, + "search_list_size": self.search_list_size, + "max_alpha": self.max_alpha, + "num_dimensions": self.num_dimensions, + }, + } + + def search_param(self) -> dict: + return { + "metric": self.parse_metric(), + "metric_fun_op": self.parse_metric_fun_op(), + } + + def session_param(self) -> dict: + return { + "diskann.query_search_list_size": self.query_search_list_size, + "diskann.query_rescore": self.query_rescore, + } + +_pgvectorscale_case_config = { + IndexType.STREAMING_DISKANN: PgVectorScaleStreamingDiskANNConfig, +} diff --git a/vectordb_bench/backend/clients/pgvectorscale/pgvectorscale.py b/vectordb_bench/backend/clients/pgvectorscale/pgvectorscale.py new file mode 100644 index 000000000..7c8c314c2 --- /dev/null +++ b/vectordb_bench/backend/clients/pgvectorscale/pgvectorscale.py @@ -0,0 +1,272 @@ +"""Wrapper around the Pgvectorscale vector database over VectorDB""" + +import logging +import pprint +from contextlib import contextmanager +from typing import Any, Generator, Optional, Tuple + +import numpy as np +import psycopg +from pgvector.psycopg import register_vector +from psycopg import Connection, Cursor, sql + +from ..api import VectorDB +from .config import PgVectorScaleConfigDict, PgVectorScaleIndexConfig + +log = logging.getLogger(__name__) + + +class PgVectorScale(VectorDB): + """Use psycopg instructions""" + + conn: psycopg.Connection[Any] | None = None + coursor: psycopg.Cursor[Any] | None = None + + def __init__( + self, + dim: int, + db_config: PgVectorScaleConfigDict, + db_case_config: PgVectorScaleIndexConfig, + collection_name: str = "pg_vectorscale_collection", + drop_old: bool = False, + **kwargs, + ): + self.name = "PgVectorScale" + self.db_config = db_config + self.case_config = db_case_config + self.table_name = collection_name + self.dim = dim + + self._index_name = "pgvectorscale_index" + self._primary_field = "id" + self._vector_field = "embedding" + + self.conn, self.cursor = self._create_connection(**self.db_config) + + log.info(f"{self.name} config values: {self.db_config}\n{self.case_config}") + if not any( + ( + self.case_config.create_index_before_load, + self.case_config.create_index_after_load, + ) + ): + err = f"{self.name} config must create an index using create_index_before_load or create_index_after_load" + log.error(err) + raise RuntimeError( + f"{err}\n{pprint.pformat(self.db_config)}\n{pprint.pformat(self.case_config)}" + ) + + if drop_old: + self._drop_index() + self._drop_table() + self._create_table(dim) + if self.case_config.create_index_before_load: + self._create_index() + + self.cursor.close() + self.conn.close() + self.cursor = None + self.conn = None + + @staticmethod + def _create_connection(**kwargs) -> Tuple[Connection, Cursor]: + conn = psycopg.connect(**kwargs) + conn.cursor().execute("CREATE EXTENSION IF NOT EXISTS vectorscale CASCADE") + conn.commit() + register_vector(conn) + conn.autocommit = False + cursor = conn.cursor() + + assert conn is not None, "Connection is not initialized" + assert cursor is not None, "Cursor is not initialized" + + return conn, cursor + + @contextmanager + def init(self) -> Generator[None, None, None]: + self.conn, self.cursor = self._create_connection(**self.db_config) + + # index configuration may have commands defined that we should set during each client session + session_options: dict[str, Any] = self.case_config.session_param() + + if len(session_options) > 0: + for setting_name, setting_val in session_options.items(): + command = sql.SQL("SET {setting_name} " + "= {setting_val};").format( + setting_name=sql.Identifier(setting_name), + setting_val=sql.Identifier(str(setting_val)), + ) + log.debug(command.as_string(self.cursor)) + self.cursor.execute(command) + self.conn.commit() + + self._unfiltered_search = sql.Composed( + [ + sql.SQL("SELECT id FROM public.{} ORDER BY embedding ").format( + sql.Identifier(self.table_name) + ), + sql.SQL(self.case_config.search_param()["metric_fun_op"]), + sql.SQL(" %s::vector LIMIT %s::int"), + ] + ) + + try: + yield + finally: + self.cursor.close() + self.conn.close() + self.cursor = None + self.conn = None + + def _drop_table(self): + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + log.info(f"{self.name} client drop table : {self.table_name}") + + self.cursor.execute( + sql.SQL("DROP TABLE IF EXISTS public.{table_name}").format( + table_name=sql.Identifier(self.table_name) + ) + ) + self.conn.commit() + + def ready_to_load(self): + pass + + def optimize(self): + self._post_insert() + + def _post_insert(self): + log.info(f"{self.name} post insert before optimize") + if self.case_config.create_index_after_load: + self._drop_index() + self._create_index() + + def _drop_index(self): + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + log.info(f"{self.name} client drop index : {self._index_name}") + + drop_index_sql = sql.SQL("DROP INDEX IF EXISTS {index_name}").format( + index_name=sql.Identifier(self._index_name) + ) + log.debug(drop_index_sql.as_string(self.cursor)) + self.cursor.execute(drop_index_sql) + self.conn.commit() + + def _create_index(self): + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + log.info(f"{self.name} client create index : {self._index_name}") + + index_param: dict[str, Any] = self.case_config.index_param() + + options = [] + for option_name, option_val in index_param["options"].items(): + if option_val is not None: + options.append( + sql.SQL("{option_name} = {val}").format( + option_name=sql.Identifier(option_name), + val=sql.Identifier(str(option_val)), + ) + ) + + num_bits_per_dimension = "2" if self.dim < 900 else "1" + options.append( + sql.SQL("{option_name} = {val}").format( + option_name=sql.Identifier("num_bits_per_dimension"), + val=sql.Identifier(num_bits_per_dimension), + ) + ) + + if any(options): + with_clause = sql.SQL("WITH ({});").format(sql.SQL(", ").join(options)) + else: + with_clause = sql.Composed(()) + + index_create_sql = sql.SQL( + """ + CREATE INDEX IF NOT EXISTS {index_name} ON public.{table_name} + USING {index_type} (embedding {embedding_metric}) + """ + ).format( + index_name=sql.Identifier(self._index_name), + table_name=sql.Identifier(self.table_name), + index_type=sql.Identifier(index_param["index_type"].lower()), + embedding_metric=sql.Identifier(index_param["metric"]), + ) + index_create_sql_with_with_clause = ( + index_create_sql + with_clause + ).join(" ") + log.debug(index_create_sql_with_with_clause.as_string(self.cursor)) + self.cursor.execute(index_create_sql_with_with_clause) + self.conn.commit() + + def _create_table(self, dim: int): + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + try: + log.info(f"{self.name} client create table : {self.table_name}") + + self.cursor.execute( + sql.SQL( + "CREATE TABLE IF NOT EXISTS public.{table_name} (id BIGINT PRIMARY KEY, embedding vector({dim}));" + ).format(table_name=sql.Identifier(self.table_name), dim=dim) + ) + self.conn.commit() + except Exception as e: + log.warning( + f"Failed to create pgvectorscale table: {self.table_name} error: {e}" + ) + raise e from None + + def insert_embeddings( + self, + embeddings: list[list[float]], + metadata: list[int], + **kwargs: Any, + ) -> Tuple[int, Optional[Exception]]: + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + try: + metadata_arr = np.array(metadata) + embeddings_arr = np.array(embeddings) + + with self.cursor.copy( + sql.SQL("COPY public.{table_name} FROM STDIN (FORMAT BINARY)").format( + table_name=sql.Identifier(self.table_name) + ) + ) as copy: + copy.set_types(["bigint", "vector"]) + for i, row in enumerate(metadata_arr): + copy.write_row((row, embeddings_arr[i])) + self.conn.commit() + + if kwargs.get("last_batch"): + self._post_insert() + + return len(metadata), None + except Exception as e: + log.warning( + f"Failed to insert data into pgvector table ({self.table_name}), error: {e}" + ) + return 0, e + + def search_embedding( + self, + query: list[float], + k: int = 100, + filters: dict | None = None, + timeout: int | None = None, + ) -> list[int]: + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + q = np.asarray(query) + # TODO add filters support + result = self.cursor.execute( + self._unfiltered_search, (q, k), prepare=True, binary=True + ) + + return [int(i[0]) for i in result.fetchall()] diff --git a/vectordb_bench/frontend/components/run_test/caseSelector.py b/vectordb_bench/frontend/components/run_test/caseSelector.py index 58799deff..5597bbc61 100644 --- a/vectordb_bench/frontend/components/run_test/caseSelector.py +++ b/vectordb_bench/frontend/components/run_test/caseSelector.py @@ -100,6 +100,16 @@ def caseConfigSetting(st, dbToCaseClusterConfigs, uiCaseItem: UICaseItem, active value=config.inputConfig["value"], help=config.inputHelp, ) + elif config.inputType == InputType.Float: + caseConfig[config.label] = column.number_input( + config.displayLabel if config.displayLabel else config.label.value, + step=config.inputConfig.get("step", 0.1), + min_value=config.inputConfig["min"], + max_value=config.inputConfig["max"], + key=key, + value=config.inputConfig["value"], + help=config.inputHelp, + ) k += 1 if k == 0: columns[1].write("Auto") diff --git a/vectordb_bench/frontend/config/dbCaseConfigs.py b/vectordb_bench/frontend/config/dbCaseConfigs.py index 687f1efbf..13634eca5 100644 --- a/vectordb_bench/frontend/config/dbCaseConfigs.py +++ b/vectordb_bench/frontend/config/dbCaseConfigs.py @@ -148,6 +148,7 @@ class InputType(IntEnum): Text = 20001 Number = 20002 Option = 20003 + Float = 20004 class CaseConfigInput(BaseModel): @@ -169,6 +170,7 @@ class CaseConfigInput(BaseModel): IndexType.IVFFlat.value, IndexType.IVFSQ8.value, IndexType.DISKANN.value, + IndexType.STREAMING_DISKANN.value, IndexType.Flat.value, IndexType.AUTOINDEX.value, IndexType.GPU_IVF_FLAT.value, @@ -178,6 +180,104 @@ class CaseConfigInput(BaseModel): }, ) + +CaseConfigParamInput_IndexType_PgVectorScale = CaseConfigInput( + label=CaseConfigParamType.IndexType, + inputHelp="Select Index Type", + inputType=InputType.Option, + inputConfig={ + "options": [ + IndexType.STREAMING_DISKANN.value, + ], + }, +) + + +CaseConfigParamInput_storage_layout = CaseConfigInput( + label=CaseConfigParamType.storage_layout, + inputHelp="Select Storage Layout", + inputType=InputType.Option, + inputConfig={ + "options": [ + "memory_optimized", + "plain", + ], + }, +) + +CaseConfigParamInput_num_neighbors = CaseConfigInput( + label=CaseConfigParamType.num_neighbors, + inputType=InputType.Number, + inputConfig={ + "min": 10, + "max": 300, + "value": 50, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) + == IndexType.STREAMING_DISKANN.value, +) + +CaseConfigParamInput_search_list_size = CaseConfigInput( + label=CaseConfigParamType.search_list_size, + inputType=InputType.Number, + inputConfig={ + "min": 10, + "max": 300, + "value": 100, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) + == IndexType.STREAMING_DISKANN.value, +) + +CaseConfigParamInput_max_alpha = CaseConfigInput( + label=CaseConfigParamType.max_alpha, + inputType=InputType.Float, + inputConfig={ + "min": 0.1, + "max": 2.0, + "value": 1.2, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) + == IndexType.STREAMING_DISKANN.value, +) + +CaseConfigParamInput_num_dimensions = CaseConfigInput( + label=CaseConfigParamType.num_dimensions, + inputType=InputType.Number, + inputConfig={ + "min": 0, + "max": 2000, + "value": 0, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) + == IndexType.STREAMING_DISKANN.value, +) + +CaseConfigParamInput_query_search_list_size = CaseConfigInput( + label=CaseConfigParamType.query_search_list_size, + inputType=InputType.Number, + inputConfig={ + "min": 50, + "max": 150, + "value": 100, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) + == IndexType.STREAMING_DISKANN.value, +) + + +CaseConfigParamInput_query_rescore = CaseConfigInput( + label=CaseConfigParamType.query_rescore, + inputType=InputType.Number, + inputConfig={ + "min": 0, + "max": 150, + "value": 50, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) + == IndexType.STREAMING_DISKANN.value, +) + CaseConfigParamInput_IndexType_PgVector = CaseConfigInput( label=CaseConfigParamType.IndexType, inputHelp="Select Index Type", @@ -427,6 +527,7 @@ class CaseConfigInput(BaseModel): in [IndexType.GPU_IVF_PQ.value], ) + CaseConfigParamInput_Nbits_PQ = CaseConfigInput( label=CaseConfigParamType.nbits, inputType=InputType.Number, @@ -770,6 +871,24 @@ class CaseConfigInput(BaseModel): CaseConfigParamInput_ZillizLevel, ] +PgVectorScaleLoadingConfig = [ + CaseConfigParamInput_IndexType_PgVectorScale, + CaseConfigParamInput_num_neighbors, + CaseConfigParamInput_storage_layout, + CaseConfigParamInput_search_list_size, + CaseConfigParamInput_max_alpha, +] + +PgVectorScalePerformanceConfig = [ + CaseConfigParamInput_IndexType_PgVectorScale, + CaseConfigParamInput_num_neighbors, + CaseConfigParamInput_storage_layout, + CaseConfigParamInput_search_list_size, + CaseConfigParamInput_max_alpha, + CaseConfigParamInput_query_rescore, + CaseConfigParamInput_query_search_list_size, +] + CASE_CONFIG_MAP = { DB.Milvus: { CaseLabel.Load: MilvusLoadConfig, @@ -794,4 +913,8 @@ class CaseConfigInput(BaseModel): CaseLabel.Load: PgVectoRSLoadingConfig, CaseLabel.Performance: PgVectoRSPerformanceConfig, }, + DB.PgVectorScale: { + CaseLabel.Load: PgVectorScaleLoadingConfig, + CaseLabel.Performance: PgVectorScalePerformanceConfig, + }, } diff --git a/vectordb_bench/models.py b/vectordb_bench/models.py index bc99e8595..75b427091 100644 --- a/vectordb_bench/models.py +++ b/vectordb_bench/models.py @@ -62,6 +62,14 @@ class CaseConfigParamType(Enum): level = "level" maintenance_work_mem = "maintenance_work_mem" max_parallel_workers = "max_parallel_workers" + storage_layout = "storage_layout" + num_neighbors = "num_neighbors" + search_list_size = "search_list_size" + max_alpha = "max_alpha" + num_dimensions = "num_dimensions" + num_bits_per_dimension = "num_bits_per_dimension" + query_search_list_size = "query_search_list_size" + query_rescore = "query_rescore" class CustomizedCase(BaseModel): From ea4e96d6aa182fec014c4e5d9ee22d23b265f997 Mon Sep 17 00:00:00 2001 From: kailiu wang Date: Fri, 16 Aug 2024 15:14:11 +0800 Subject: [PATCH 078/327] pr (#358) --- vectordb_bench/backend/clients/redis/cli.py | 8 ++++ .../backend/clients/redis/config.py | 43 ++++++++++++++++--- 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/vectordb_bench/backend/clients/redis/cli.py b/vectordb_bench/backend/clients/redis/cli.py index 172b2b52a..eb86b4c00 100644 --- a/vectordb_bench/backend/clients/redis/cli.py +++ b/vectordb_bench/backend/clients/redis/cli.py @@ -3,6 +3,9 @@ import click from pydantic import SecretStr +from .config import RedisHNSWConfig + + from ....cli.cli import ( CommonTypedDict, HNSWFlavor2, @@ -69,6 +72,11 @@ def Redis(**parameters: Unpack[RedisHNSWTypedDict]): ssl=parameters["ssl"], ssl_ca_certs=parameters["ssl_ca_certs"], cmd=parameters["cmd"], + ), + db_case_config=RedisHNSWConfig( + M=parameters["m"], + efConstruction=parameters["ef_construction"], + ef=parameters["ef_runtime"], ), **parameters, ) diff --git a/vectordb_bench/backend/clients/redis/config.py b/vectordb_bench/backend/clients/redis/config.py index 133521e97..55a7a4159 100644 --- a/vectordb_bench/backend/clients/redis/config.py +++ b/vectordb_bench/backend/clients/redis/config.py @@ -1,14 +1,45 @@ -from pydantic import SecretStr -from ..api import DBConfig +from pydantic import SecretStr, BaseModel +from ..api import DBConfig, DBCaseConfig, MetricType, IndexType class RedisConfig(DBConfig): - password: SecretStr + password: SecretStr | None = None host: SecretStr - port: int = None + port: int | None = None def to_dict(self) -> dict: return { "host": self.host.get_secret_value(), "port": self.port, - "password": self.password.get_secret_value(), - } \ No newline at end of file + "password": self.password.get_secret_value() if self.password is not None else None, + } + + + +class RedisIndexConfig(BaseModel): + """Base config for milvus""" + + metric_type: MetricType | None = None + + def parse_metric(self) -> str: + if not self.metric_type: + return "" + return self.metric_type.value + +class RedisHNSWConfig(RedisIndexConfig, DBCaseConfig): + M: int + efConstruction: int + ef: int | None = None + index: IndexType = IndexType.HNSW + + def index_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "index_type": self.index.value, + "params": {"M": self.M, "efConstruction": self.efConstruction}, + } + + def search_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "params": {"ef": self.ef}, + } From c532de896b89b57ec625710842104b969a4d7ce0 Mon Sep 17 00:00:00 2001 From: Wahaj Ali <894764+wahajali@users.noreply.github.com> Date: Tue, 27 Aug 2024 06:49:24 +0500 Subject: [PATCH 079/327] Add support for filtered search in pgvector (#362) --- .../backend/clients/pgvector/pgvector.py | 25 +++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/vectordb_bench/backend/clients/pgvector/pgvector.py b/vectordb_bench/backend/clients/pgvector/pgvector.py index 1ae661041..102481d8d 100644 --- a/vectordb_bench/backend/clients/pgvector/pgvector.py +++ b/vectordb_bench/backend/clients/pgvector/pgvector.py @@ -22,7 +22,7 @@ class PgVector(VectorDB): conn: psycopg.Connection[Any] | None = None cursor: psycopg.Cursor[Any] | None = None - # TODO add filters support + _filtered_search: sql.Composed _unfiltered_search: sql.Composed def __init__( @@ -112,6 +112,16 @@ def init(self) -> Generator[None, None, None]: self.cursor.execute(command) self.conn.commit() + self._filtered_search = sql.Composed( + [ + sql.SQL( + "SELECT id FROM public.{table_name} WHERE id >= %s ORDER BY embedding " + ).format(table_name=sql.Identifier(self.table_name)), + sql.SQL(self.case_config.search_param()["metric_fun_op"]), + sql.SQL(" %s::vector LIMIT %s::int"), + ] + ) + self._unfiltered_search = sql.Composed( [ sql.SQL("SELECT id FROM public.{} ORDER BY embedding ").format( @@ -342,9 +352,14 @@ def search_embedding( assert self.cursor is not None, "Cursor is not initialized" q = np.asarray(query) - # TODO add filters support - result = self.cursor.execute( - self._unfiltered_search, (q, k), prepare=True, binary=True - ) + if filters: + gt = filters.get("id") + result = self.cursor.execute( + self._filtered_search, (gt, q, k), prepare=True, binary=True + ) + else: + result = self.cursor.execute( + self._unfiltered_search, (q, k), prepare=True, binary=True + ) return [int(i[0]) for i in result.fetchall()] From 3fabfab449048b2b7e3d612b1c4105e19385b9a0 Mon Sep 17 00:00:00 2001 From: Sheharyar Ahmad Date: Thu, 29 Aug 2024 15:30:11 +0500 Subject: [PATCH 080/327] Added cli support for pgvectorscale --- .../backend/clients/pgvectorscale/cli.py | 108 ++++++++++++++++++ vectordb_bench/cli/vectordbbench.py | 3 +- 2 files changed, 110 insertions(+), 1 deletion(-) create mode 100644 vectordb_bench/backend/clients/pgvectorscale/cli.py diff --git a/vectordb_bench/backend/clients/pgvectorscale/cli.py b/vectordb_bench/backend/clients/pgvectorscale/cli.py new file mode 100644 index 000000000..e5a161c6b --- /dev/null +++ b/vectordb_bench/backend/clients/pgvectorscale/cli.py @@ -0,0 +1,108 @@ +import click +import os +from pydantic import SecretStr + +from ....cli.cli import ( + CommonTypedDict, + cli, + click_parameter_decorators_from_typed_dict, + run, +) +from typing import Annotated, Unpack +from vectordb_bench.backend.clients import DB + + +class PgVectorScaleTypedDict(CommonTypedDict): + user_name: Annotated[ + str, click.option("--user-name", type=str, help="Db username", required=True) + ] + password: Annotated[ + str, + click.option("--password", + type=str, + help="Postgres database password", + default=lambda: os.environ.get("POSTGRES_PASSWORD", ""), + show_default="$POSTGRES_PASSWORD", + ), + ] + + host: Annotated[ + str, click.option("--host", type=str, help="Db host", required=True) + ] + db_name: Annotated[ + str, click.option("--db-name", type=str, help="Db name", required=True) + ] + + +class PgVectorScaleDiskAnnTypedDict(PgVectorScaleTypedDict): + storage_layout: Annotated[ + str, + click.option( + "--storage-layout", type=str, help="Streaming DiskANN storage layout", + ), + ] + num_neighbors: Annotated[ + int, + click.option( + "--num-neighbors", type=int, help="Streaming DiskANN num neighbors", + ), + ] + search_list_size: Annotated[ + int, + click.option( + "--search-list-size", type=int, help="Streaming DiskANN search list size", + ), + ] + max_alpha: Annotated[ + float, + click.option( + "--max-alpha", type=float, help="Streaming DiskANN max alpha", + ), + ] + num_dimensions: Annotated[ + int, + click.option( + "--num-dimensions", type=int, help="Streaming DiskANN num dimensions", + ), + ] + query_search_list_size: Annotated[ + int, + click.option( + "--query-search-list-size", type=int, help="Streaming DiskANN query search list size", + ), + ] + query_rescore: Annotated[ + int, + click.option( + "--query-rescore", type=int, help="Streaming DiskANN query rescore", + ), + ] + + +@cli.command() +@click_parameter_decorators_from_typed_dict(PgVectorScaleDiskAnnTypedDict) +def PgVectorScaleDiskAnn( + **parameters: Unpack[PgVectorScaleDiskAnnTypedDict], +): + from .config import PgVectorScaleConfig, PgVectorScaleStreamingDiskANNConfig + + run( + db=DB.PgVectorScale, + db_config=PgVectorScaleConfig( + db_label=parameters["db_label"], + user_name=SecretStr(parameters["user_name"]), + password=SecretStr(parameters["password"]), + host=parameters["host"], + db_name=parameters["db_name"], + ), + db_case_config=PgVectorScaleStreamingDiskANNConfig( + storage_layout=parameters["storage_layout"], + num_neighbors=parameters["num_neighbors"], + search_list_size=parameters["search_list_size"], + max_alpha=parameters["max_alpha"], + num_dimensions=parameters["num_dimensions"], + query_search_list_size=parameters["query_search_list_size"], + query_rescore=parameters["query_rescore"], + ), + **parameters, + ) \ No newline at end of file diff --git a/vectordb_bench/cli/vectordbbench.py b/vectordb_bench/cli/vectordbbench.py index ab7eff44c..e62c25a3d 100644 --- a/vectordb_bench/cli/vectordbbench.py +++ b/vectordb_bench/cli/vectordbbench.py @@ -1,5 +1,6 @@ from ..backend.clients.pgvector.cli import PgVectorHNSW from ..backend.clients.pgvecto_rs.cli import PgVectoRSHNSW, PgVectoRSIVFFlat +from ..backend.clients.pgvectorscale.cli import PgVectorScaleDiskAnn from ..backend.clients.redis.cli import Redis from ..backend.clients.memorydb.cli import MemoryDB from ..backend.clients.test.cli import Test @@ -8,7 +9,6 @@ from ..backend.clients.milvus.cli import MilvusAutoIndex from ..backend.clients.aws_opensearch.cli import AWSOpenSearch - from .cli import cli cli.add_command(PgVectorHNSW) @@ -21,6 +21,7 @@ cli.add_command(ZillizAutoIndex) cli.add_command(MilvusAutoIndex) cli.add_command(AWSOpenSearch) +cli.add_command(PgVectorScaleDiskAnn) if __name__ == "__main__": From 0889c80693746d90133fb66d712c53905ac401ca Mon Sep 17 00:00:00 2001 From: Sheharyar Ahmad Date: Mon, 2 Sep 2024 07:16:38 +0500 Subject: [PATCH 081/327] Add support for filtered search in pgvectorscale (#364) * Add support for filtered search in pgvectorscale * Fixed filtered query result not assigned. --- .../clients/pgvectorscale/pgvectorscale.py | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/vectordb_bench/backend/clients/pgvectorscale/pgvectorscale.py b/vectordb_bench/backend/clients/pgvectorscale/pgvectorscale.py index 7c8c314c2..d8f26394c 100644 --- a/vectordb_bench/backend/clients/pgvectorscale/pgvectorscale.py +++ b/vectordb_bench/backend/clients/pgvectorscale/pgvectorscale.py @@ -22,6 +22,9 @@ class PgVectorScale(VectorDB): conn: psycopg.Connection[Any] | None = None coursor: psycopg.Cursor[Any] | None = None + _unfiltered_search: sql.Composed + _filtered_search: sql.Composed + def __init__( self, dim: int, @@ -99,6 +102,16 @@ def init(self) -> Generator[None, None, None]: self.cursor.execute(command) self.conn.commit() + self._filtered_search = sql.Composed( + [ + sql.SQL("SELECT id FROM public.{} WHERE id >= %s ORDER BY embedding ").format( + sql.Identifier(self.table_name), + ), + sql.SQL(self.case_config.search_param()["metric_fun_op"]), + sql.SQL(" %s::vector LIMIT %s::int") + ] + ) + self._unfiltered_search = sql.Composed( [ sql.SQL("SELECT id FROM public.{} ORDER BY embedding ").format( @@ -264,9 +277,14 @@ def search_embedding( assert self.cursor is not None, "Cursor is not initialized" q = np.asarray(query) - # TODO add filters support - result = self.cursor.execute( - self._unfiltered_search, (q, k), prepare=True, binary=True - ) + if filters: + gt = filters.get("id") + result = self.cursor.execute( + self._filtered_search, (gt, q, k), prepare=True, binary=True + ) + else: + result = self.cursor.execute( + self._unfiltered_search, (q, k), prepare=True, binary=True + ) return [int(i[0]) for i in result.fetchall()] From 5b1248d8bedc0cfa5a5a7984552f1dc5aa746170 Mon Sep 17 00:00:00 2001 From: Luca Giacchino Date: Fri, 9 Aug 2024 13:29:03 -0700 Subject: [PATCH 082/327] Add quantization option for pgvector with support for halfvec --- README.md | 2 + .../backend/clients/pgvector/cli.py | 16 ++- .../backend/clients/pgvector/config.py | 25 +++- .../backend/clients/pgvector/pgvector.py | 113 +++++++++++++----- .../frontend/config/dbCaseConfigs.py | 15 +++ 5 files changed, 135 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 1ce4564ef..f76ac5c10 100644 --- a/README.md +++ b/README.md @@ -117,6 +117,8 @@ Options: --m INTEGER hnsw m --ef-construction INTEGER hnsw ef-construction --ef-search INTEGER hnsw ef-search + --quantization-type [none|halfvec] + quantization type for vectors --help Show this message and exit. ``` #### Using a configuration file. diff --git a/vectordb_bench/backend/clients/pgvector/cli.py b/vectordb_bench/backend/clients/pgvector/cli.py index 31b268231..4e0922694 100644 --- a/vectordb_bench/backend/clients/pgvector/cli.py +++ b/vectordb_bench/backend/clients/pgvector/cli.py @@ -56,7 +56,15 @@ class PgVectorTypedDict(CommonTypedDict): required=False, ), ] - + quantization_type: Annotated[ + Optional[str], + click.option( + "--quantization-type", + type=click.Choice(["none", "halfvec"]), + help="quantization type for vectors", + required=False, + ), + ] class PgVectorIVFFlatTypedDict(PgVectorTypedDict, IVFFlatTypedDict): ... @@ -79,7 +87,10 @@ def PgVectorIVFFlat( db_name=parameters["db_name"], ), db_case_config=PgVectorIVFFlatConfig( - metric_type=None, lists=parameters["lists"], probes=parameters["probes"] + metric_type=None, + lists=parameters["lists"], + probes=parameters["probes"], + quantization_type=parameters["quantization_type"], ), **parameters, ) @@ -111,6 +122,7 @@ def PgVectorHNSW( ef_search=parameters["ef_search"], maintenance_work_mem=parameters["maintenance_work_mem"], max_parallel_workers=parameters["max_parallel_workers"], + quantization_type=parameters["quantization_type"], ), **parameters, ) diff --git a/vectordb_bench/backend/clients/pgvector/config.py b/vectordb_bench/backend/clients/pgvector/config.py index 496a3b440..857211234 100644 --- a/vectordb_bench/backend/clients/pgvector/config.py +++ b/vectordb_bench/backend/clients/pgvector/config.py @@ -59,11 +59,18 @@ class PgVectorIndexConfig(BaseModel, DBCaseConfig): create_index_after_load: bool = True def parse_metric(self) -> str: - if self.metric_type == MetricType.L2: - return "vector_l2_ops" - elif self.metric_type == MetricType.IP: - return "vector_ip_ops" - return "vector_cosine_ops" + if self.quantization_type == "halfvec": + if self.metric_type == MetricType.L2: + return "halfvec_l2_ops" + elif self.metric_type == MetricType.IP: + return "halfvec_ip_ops" + return "halfvec_cosine_ops" + else: + if self.metric_type == MetricType.L2: + return "vector_l2_ops" + elif self.metric_type == MetricType.IP: + return "vector_ip_ops" + return "vector_cosine_ops" def parse_metric_fun_op(self) -> LiteralString: if self.metric_type == MetricType.L2: @@ -143,9 +150,12 @@ class PgVectorIVFFlatConfig(PgVectorIndexConfig): index: IndexType = IndexType.ES_IVFFlat maintenance_work_mem: Optional[str] = None max_parallel_workers: Optional[int] = None + quantization_type: Optional[str] = None def index_param(self) -> PgVectorIndexParam: index_parameters = {"lists": self.lists} + if self.quantization_type == "none": + self.quantization_type = None return { "metric": self.parse_metric(), "index_type": self.index.value, @@ -154,6 +164,7 @@ def index_param(self) -> PgVectorIndexParam: ), "maintenance_work_mem": self.maintenance_work_mem, "max_parallel_workers": self.max_parallel_workers, + "quantization_type": self.quantization_type, } def search_param(self) -> PgVectorSearchParam: @@ -183,9 +194,12 @@ class PgVectorHNSWConfig(PgVectorIndexConfig): index: IndexType = IndexType.ES_HNSW maintenance_work_mem: Optional[str] = None max_parallel_workers: Optional[int] = None + quantization_type: Optional[str] = None def index_param(self) -> PgVectorIndexParam: index_parameters = {"m": self.m, "ef_construction": self.ef_construction} + if self.quantization_type == "none": + self.quantization_type = None return { "metric": self.parse_metric(), "index_type": self.index.value, @@ -194,6 +208,7 @@ def index_param(self) -> PgVectorIndexParam: ), "maintenance_work_mem": self.maintenance_work_mem, "max_parallel_workers": self.max_parallel_workers, + "quantization_type": self.quantization_type, } def search_param(self) -> PgVectorSearchParam: diff --git a/vectordb_bench/backend/clients/pgvector/pgvector.py b/vectordb_bench/backend/clients/pgvector/pgvector.py index 102481d8d..8123acf18 100644 --- a/vectordb_bench/backend/clients/pgvector/pgvector.py +++ b/vectordb_bench/backend/clients/pgvector/pgvector.py @@ -112,25 +112,63 @@ def init(self) -> Generator[None, None, None]: self.cursor.execute(command) self.conn.commit() - self._filtered_search = sql.Composed( - [ - sql.SQL( - "SELECT id FROM public.{table_name} WHERE id >= %s ORDER BY embedding " - ).format(table_name=sql.Identifier(self.table_name)), - sql.SQL(self.case_config.search_param()["metric_fun_op"]), - sql.SQL(" %s::vector LIMIT %s::int"), - ] - ) + index_param = self.case_config.index_param() + # The following sections assume that the quantization_type value matches the quantization function name + if index_param["quantization_type"] != None: + self._filtered_search = sql.Composed( + [ + sql.SQL( + "SELECT id FROM public.{table_name} WHERE id >= %s ORDER BY embedding::{quantization_type}({dim}) " + ).format( + table_name=sql.Identifier(self.table_name), + quantization_type=sql.SQL(index_param["quantization_type"]), + dim=sql.Literal(self.dim), + ), + sql.SQL(self.case_config.search_param()["metric_fun_op"]), + sql.SQL(" %s::{quantization_type}({dim}) LIMIT %s::int").format( + quantization_type=sql.SQL(index_param["quantization_type"]), + dim=sql.Literal(self.dim), + ), + ] + ) + else: + self._filtered_search = sql.Composed( + [ + sql.SQL( + "SELECT id FROM public.{table_name} WHERE id >= %s ORDER BY embedding " + ).format(table_name=sql.Identifier(self.table_name)), + sql.SQL(self.case_config.search_param()["metric_fun_op"]), + sql.SQL(" %s::vector LIMIT %s::int"), + ] + ) - self._unfiltered_search = sql.Composed( - [ - sql.SQL("SELECT id FROM public.{} ORDER BY embedding ").format( - sql.Identifier(self.table_name) - ), - sql.SQL(self.case_config.search_param()["metric_fun_op"]), - sql.SQL(" %s::vector LIMIT %s::int"), - ] - ) + if index_param["quantization_type"] != None: + self._unfiltered_search = sql.Composed( + [ + sql.SQL( + "SELECT id FROM public.{table_name} ORDER BY embedding::{quantization_type}({dim}) " + ).format( + table_name=sql.Identifier(self.table_name), + quantization_type=sql.SQL(index_param["quantization_type"]), + dim=sql.Literal(self.dim), + ), + sql.SQL(self.case_config.search_param()["metric_fun_op"]), + sql.SQL(" %s::{quantization_type}({dim}) LIMIT %s::int").format( + quantization_type=sql.SQL(index_param["quantization_type"]), + dim=sql.Literal(self.dim), + ), + ] + ) + else: + self._unfiltered_search = sql.Composed( + [ + sql.SQL("SELECT id FROM public.{} ORDER BY embedding ").format( + sql.Identifier(self.table_name) + ), + sql.SQL(self.case_config.search_param()["metric_fun_op"]), + sql.SQL(" %s::vector LIMIT %s::int"), + ] + ) try: yield @@ -265,17 +303,34 @@ def _create_index(self): else: with_clause = sql.Composed(()) - index_create_sql = sql.SQL( - """ - CREATE INDEX IF NOT EXISTS {index_name} ON public.{table_name} - USING {index_type} (embedding {embedding_metric}) - """ - ).format( - index_name=sql.Identifier(self._index_name), - table_name=sql.Identifier(self.table_name), - index_type=sql.Identifier(index_param["index_type"]), - embedding_metric=sql.Identifier(index_param["metric"]), - ) + if index_param["quantization_type"] != None: + index_create_sql = sql.SQL( + """ + CREATE INDEX IF NOT EXISTS {index_name} ON public.{table_name} + USING {index_type} ((embedding::{quantization_type}({dim})) {embedding_metric}) + """ + ).format( + index_name=sql.Identifier(self._index_name), + table_name=sql.Identifier(self.table_name), + index_type=sql.Identifier(index_param["index_type"]), + # This assumes that the quantization_type value matches the quantization function name + quantization_type=sql.SQL(index_param["quantization_type"]), + dim=self.dim, + embedding_metric=sql.Identifier(index_param["metric"]), + ) + else: + index_create_sql = sql.SQL( + """ + CREATE INDEX IF NOT EXISTS {index_name} ON public.{table_name} + USING {index_type} (embedding {embedding_metric}) + """ + ).format( + index_name=sql.Identifier(self._index_name), + table_name=sql.Identifier(self.table_name), + index_type=sql.Identifier(index_param["index_type"]), + embedding_metric=sql.Identifier(index_param["metric"]), + ) + index_create_sql_with_with_clause = ( index_create_sql + with_clause ).join(" ") diff --git a/vectordb_bench/frontend/config/dbCaseConfigs.py b/vectordb_bench/frontend/config/dbCaseConfigs.py index 13634eca5..78d1936d7 100644 --- a/vectordb_bench/frontend/config/dbCaseConfigs.py +++ b/vectordb_bench/frontend/config/dbCaseConfigs.py @@ -738,6 +738,19 @@ class CaseConfigInput(BaseModel): ], ) +CaseConfigParamInput_QuantizationType_PgVector = CaseConfigInput( + label=CaseConfigParamType.quantizationType, + inputType=InputType.Option, + inputConfig={ + "options": ["none", "halfvec"], + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) + in [ + IndexType.HNSW.value, + IndexType.IVFFlat.value, + ], +) + CaseConfigParamInput_QuantizationRatio_PgVectoRS = CaseConfigInput( label=CaseConfigParamType.quantizationRatio, inputType=InputType.Option, @@ -831,6 +844,7 @@ class CaseConfigInput(BaseModel): CaseConfigParamInput_Lists_PgVector, CaseConfigParamInput_m, CaseConfigParamInput_EFConstruction_PgVector, + CaseConfigParamInput_QuantizationType_PgVector, CaseConfigParamInput_maintenance_work_mem_PgVector, CaseConfigParamInput_max_parallel_workers_PgVector, ] @@ -841,6 +855,7 @@ class CaseConfigInput(BaseModel): CaseConfigParamInput_EFSearch_PgVector, CaseConfigParamInput_Lists_PgVector, CaseConfigParamInput_Probes_PgVector, + CaseConfigParamInput_QuantizationType_PgVector, CaseConfigParamInput_maintenance_work_mem_PgVector, CaseConfigParamInput_max_parallel_workers_PgVector, ] From 167807765ec8d7f12ef894f134c12a8b4780b397 Mon Sep 17 00:00:00 2001 From: Sheharyar Ahmad Date: Wed, 2 Oct 2024 23:23:32 +0500 Subject: [PATCH 083/327] Add cli support for running benchmark with custom dataset in pgvector hnsw and ivvfflat. --- .../backend/clients/pgvector/cli.py | 3 + vectordb_bench/cli/cli.py | 137 ++++++++++++++++++ 2 files changed, 140 insertions(+) diff --git a/vectordb_bench/backend/clients/pgvector/cli.py b/vectordb_bench/backend/clients/pgvector/cli.py index 4e0922694..d5779caf6 100644 --- a/vectordb_bench/backend/clients/pgvector/cli.py +++ b/vectordb_bench/backend/clients/pgvector/cli.py @@ -10,6 +10,7 @@ IVFFlatTypedDict, cli, click_parameter_decorators_from_typed_dict, + get_custom_case_config, run, ) from vectordb_bench.backend.clients import DB @@ -77,6 +78,7 @@ def PgVectorIVFFlat( ): from .config import PgVectorConfig, PgVectorIVFFlatConfig + parameters["custom_case"] = get_custom_case_config(parameters) run( db=DB.PgVector, db_config=PgVectorConfig( @@ -107,6 +109,7 @@ def PgVectorHNSW( ): from .config import PgVectorConfig, PgVectorHNSWConfig + parameters["custom_case"] = get_custom_case_config(parameters) run( db=DB.PgVector, db_config=PgVectorConfig( diff --git a/vectordb_bench/cli/cli.py b/vectordb_bench/cli/cli.py index 00910261b..950bf730b 100644 --- a/vectordb_bench/cli/cli.py +++ b/vectordb_bench/cli/cli.py @@ -17,6 +17,8 @@ Any, ) import click + +from vectordb_bench.backend.clients.api import MetricType from .. import config from ..backend.clients import DB from ..interface import benchMarkRunner, global_result_future @@ -147,6 +149,37 @@ def parse_task_stages( return stages +def check_custom_case_parameters(ctx, param, value): + if ctx.params.get("case_type") == "PerformanceCustomDataset": + if value is None: + raise click.BadParameter("Custom case parameters\ + \n--custom-case-name\n--custom-dataset-name\n--custom-dataset-dir\n--custom-dataset-size \ + \n--custom-dataset-dim\n--custom-dataset-file-count\n are required") + return value + + +def get_custom_case_config(parameters: dict) -> dict: + custom_case_config = {} + if parameters["case_type"] == "PerformanceCustomDataset": + custom_case_config = { + "name": parameters["custom_case_name"], + "description": parameters["custom_case_description"], + "load_timeout": parameters["custom_case_load_timeout"], + "optimize_timeout": parameters["custom_case_optimize_timeout"], + "dataset_config": { + "name": parameters["custom_dataset_name"], + "dir": parameters["custom_dataset_dir"], + "size": parameters["custom_dataset_size"], + "dim": parameters["custom_dataset_dim"], + "metric_type": parameters["custom_dataset_metric_type"], + "file_count": parameters["custom_dataset_file_count"], + "use_shuffled": parameters["custom_dataset_use_shuffled"], + "with_gt": parameters["custom_dataset_with_gt"], + } + } + return custom_case_config + + log = logging.getLogger(__name__) @@ -205,6 +238,7 @@ class CommonTypedDict(TypedDict): click.option( "--case-type", type=click.Choice([ct.name for ct in CaseType if ct.name != "Custom"]), + is_eager=True, default="Performance1536D50K", help="Case type", ), @@ -258,6 +292,108 @@ class CommonTypedDict(TypedDict): callback=lambda *args: list(map(int, click_arg_split(*args))), ), ] + custom_case_name: Annotated[ + str, + click.option( + "--custom-case-name", + help="Custom dataset case name", + callback=check_custom_case_parameters, + ) + ] + custom_case_description: Annotated[ + str, + click.option( + "--custom-case-description", + help="Custom dataset case description", + default="This is a customized dataset.", + show_default=True, + ) + ] + custom_case_load_timeout: Annotated[ + int, + click.option( + "--custom-case-load-timeout", + help="Custom dataset case load timeout", + default=36000, + show_default=True, + ) + ] + custom_case_optimize_timeout: Annotated[ + int, + click.option( + "--custom-case-optimize-timeout", + help="Custom dataset case optimize timeout", + default=36000, + show_default=True, + ) + ] + custom_dataset_name: Annotated[ + str, + click.option( + "--custom-dataset-name", + help="Custom dataset name", + callback=check_custom_case_parameters, + ), + ] + custom_dataset_dir: Annotated[ + str, + click.option( + "--custom-dataset-dir", + help="Custom dataset directory", + callback=check_custom_case_parameters, + ), + ] + custom_dataset_size: Annotated[ + int, + click.option( + "--custom-dataset-size", + help="Custom dataset size", + callback=check_custom_case_parameters, + ), + ] + custom_dataset_dim: Annotated[ + int, + click.option( + "--custom-dataset-dim", + help="Custom dataset dimension", + callback=check_custom_case_parameters, + ), + ] + custom_dataset_metric_type: Annotated[ + str, + click.option( + "--custom-dataset-metric-type", + help="Custom dataset metric type", + default=MetricType.COSINE.name, + show_default=True, + ), + ] + custom_dataset_file_count: Annotated[ + int, + click.option( + "--custom-dataset-file-count", + help="Custom dataset file count", + callback=check_custom_case_parameters, + ), + ] + custom_dataset_use_shuffled: Annotated[ + bool, + click.option( + "--custom-dataset-use-shuffled/--no-custom-dataset-use-shuffled", + help="Custom dataset use shuffled", + default=False, + show_default=True, + ), + ] + custom_dataset_with_gt: Annotated[ + bool, + click.option( + "--custom-dataset-with-gt/--no-custom-dataset-with-gt", + help="Custom dataset with ground truth", + default=True, + show_default=True, + ), + ] class HNSWBaseTypedDict(TypedDict): @@ -343,6 +479,7 @@ def run( concurrency_duration=parameters["concurrency_duration"], num_concurrency=[int(s) for s in parameters["num_concurrency"]], ), + custom_case=parameters["custom_case"], ), stages=parse_task_stages( ( From cba70433635bac00b00dd0c4ef66c774af03caad Mon Sep 17 00:00:00 2001 From: cutecutecat Date: Tue, 8 Oct 2024 10:04:00 +0800 Subject: [PATCH 084/327] feat: upgrade pgvecto.rs sdk to v0.2.2 Signed-off-by: cutecutecat --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 4da9ce2cc..015ed8c3f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,7 +56,7 @@ all = [ "weaviate-client", "elasticsearch", "pgvector", - "pgvecto_rs[psycopg3]>=0.2.1", + "pgvecto_rs[psycopg3]>=0.2.2", "sqlalchemy", "redis", "chromadb", @@ -72,7 +72,7 @@ weaviate = [ "weaviate-client" ] elastic = [ "elasticsearch" ] pgvector = [ "psycopg", "psycopg-binary", "pgvector" ] pgvectorscale = [ "psycopg", "psycopg-binary", "pgvector" ] -pgvecto_rs = [ "pgvecto_rs[psycopg3]>=0.2.1" ] +pgvecto_rs = [ "pgvecto_rs[psycopg3]>=0.2.2" ] redis = [ "redis" ] memorydb = [ "memorydb" ] chromadb = [ "chromadb" ] From d3238217c4ee10f11af84f5198d6848d4706ce34 Mon Sep 17 00:00:00 2001 From: Sheharyar Ahmad Date: Mon, 14 Oct 2024 20:28:18 +0500 Subject: [PATCH 085/327] Randomly pick start idx of test dataset in concurrency search. --- vectordb_bench/backend/runner/mp_runner.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vectordb_bench/backend/runner/mp_runner.py b/vectordb_bench/backend/runner/mp_runner.py index 3b99cfe6e..6a9f7c979 100644 --- a/vectordb_bench/backend/runner/mp_runner.py +++ b/vectordb_bench/backend/runner/mp_runner.py @@ -2,6 +2,7 @@ import traceback import concurrent import multiprocessing as mp +import random import logging from typing import Iterable import numpy as np @@ -46,7 +47,7 @@ def search(self, test_data: list[list[float]], q: mp.Queue, cond: mp.Condition) cond.wait() with self.db.init(): - num, idx = len(test_data), 0 + num, idx = len(test_data), random.randint(0, len(test_data) - 1) start_time = time.perf_counter() count = 0 From 7f32779b3d693650c59411bc22a2d4a2e217289a Mon Sep 17 00:00:00 2001 From: Navneet Verma Date: Wed, 16 Oct 2024 14:24:52 -0700 Subject: [PATCH 086/327] Added optimzation for Opensearch Signed-off-by: Navneet Verma --- .../clients/aws_opensearch/aws_opensearch.py | 53 ++++++++++++++++--- .../backend/clients/aws_opensearch/config.py | 18 ++++--- .../backend/clients/aws_opensearch/run.py | 37 +++++++++++-- .../frontend/config/dbCaseConfigs.py | 42 +++++++++++++++ 4 files changed, 135 insertions(+), 15 deletions(-) diff --git a/vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py b/vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py index 5b0728ac8..a27eb01fc 100644 --- a/vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py +++ b/vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py @@ -3,7 +3,7 @@ import time from typing import Iterable, Type from ..api import VectorDB, DBCaseConfig, DBConfig, IndexType -from .config import AWSOpenSearchConfig, AWSOpenSearchIndexConfig +from .config import AWSOpenSearchConfig, AWSOpenSearchIndexConfig, AWSOS_Engine from opensearchpy import OpenSearch from opensearchpy.helpers import bulk @@ -83,7 +83,7 @@ def _create_index(self, client: OpenSearch): @contextmanager def init(self) -> None: - """connect to elasticsearch""" + """connect to opensearch""" self.client = OpenSearch(**self.db_config) yield @@ -97,7 +97,7 @@ def insert_embeddings( metadata: list[int], **kwargs, ) -> tuple[int, Exception]: - """Insert the embeddings to the elasticsearch.""" + """Insert the embeddings to the opensearch.""" assert self.client is not None, "should self.init() first" insert_data = [] @@ -136,13 +136,15 @@ def search_embedding( body = { "size": k, "query": {"knn": {self.vector_col_name: {"vector": query, "k": k}}}, + **({"filter": {"range": {self.id_col_name: {"gt": filters["id"]}}}} if filters else {}) } try: - resp = self.client.search(index=self.index_name, body=body) + resp = self.client.search(index=self.index_name, body=body,size=k,_source=False,docvalue_fields=[self.id_col_name],stored_fields="_none_",filter_path=[f"hits.hits.fields.{self.id_col_name}"],) log.info(f'Search took: {resp["took"]}') log.info(f'Search shards: {resp["_shards"]}') log.info(f'Search hits total: {resp["hits"]["total"]}') - result = [int(d["_id"]) for d in resp["hits"]["hits"]] + result = [h["fields"][self.id_col_name][0] for h in resp["hits"]["hits"]] + #result = [int(d["_id"]) for d in resp["hits"]["hits"]] # log.info(f'success! length={len(res)}') return result @@ -152,7 +154,46 @@ def search_embedding( def optimize(self): """optimize will be called between insertion and search in performance cases.""" - pass + # Call refresh first to ensure that all segments are created + self._refresh_index() + self._do_force_merge() + # Call refresh again to ensure that the index is ready after force merge. + self._refresh_index() + # ensure that all graphs are loaded in memory and ready for search + self._load_graphs_to_memory() + + def _refresh_index(self): + log.debug(f"Starting refresh for index {self.index_name}") + SECONDS_WAITING_FOR_REFRESH_API_CALL_SEC = 30 + while True: + try: + log.info(f"Starting the Refresh Index..") + self.client.indices.refresh(index=self.index_name) + break + except Exception as e: + log.info( + f"Refresh errored out. Sleeping for {SECONDS_WAITING_FOR_REFRESH_API_CALL_SEC} sec and then Retrying : {e}") + time.sleep(SECONDS_WAITING_FOR_REFRESH_API_CALL_SEC) + continue + log.debug(f"Completed refresh for index {self.index_name}") + + def _do_force_merge(self): + log.debug(f"Starting force merge for index {self.index_name}") + force_merge_endpoint = f'/{self.index_name}/_forcemerge?max_num_segments=1&wait_for_completion=false' + force_merge_task_id = self.client.transport.perform_request('POST', force_merge_endpoint)['task'] + SECONDS_WAITING_FOR_FORCE_MERGE_API_CALL_SEC = 30 + while True: + time.sleep(SECONDS_WAITING_FOR_FORCE_MERGE_API_CALL_SEC) + task_status = self.client.tasks.get(task_id=force_merge_task_id) + if task_status['completed']: + break + log.debug(f"Completed force merge for index {self.index_name}") + + def _load_graphs_to_memory(self): + if self.case_config.engine != AWSOS_Engine.lucene: + log.info("Calling warmup API to load graphs into memory") + warmup_endpoint = f'/_plugins/_knn/warmup/{self.index_name}' + self.client.transport.perform_request('GET', warmup_endpoint) def ready_to_load(self): """ready_to_load will be called before load in load cases.""" diff --git a/vectordb_bench/backend/clients/aws_opensearch/config.py b/vectordb_bench/backend/clients/aws_opensearch/config.py index bc82380b7..15cd4ead8 100644 --- a/vectordb_bench/backend/clients/aws_opensearch/config.py +++ b/vectordb_bench/backend/clients/aws_opensearch/config.py @@ -1,9 +1,10 @@ +import logging from enum import Enum from pydantic import SecretStr, BaseModel from ..api import DBConfig, DBCaseConfig, MetricType, IndexType - +log = logging.getLogger(__name__) class AWSOpenSearchConfig(DBConfig, BaseModel): host: str = "" port: int = 443 @@ -31,14 +32,18 @@ class AWSOS_Engine(Enum): class AWSOpenSearchIndexConfig(BaseModel, DBCaseConfig): metric_type: MetricType = MetricType.L2 - engine: AWSOS_Engine = AWSOS_Engine.nmslib - efConstruction: int = 360 - M: int = 30 + engine: AWSOS_Engine = AWSOS_Engine.faiss + efConstruction: int = 256 + efSearch: int = 256 + M: int = 16 def parse_metric(self) -> str: if self.metric_type == MetricType.IP: - return "innerproduct" # only support faiss / nmslib, not for Lucene. + return "innerproduct" elif self.metric_type == MetricType.COSINE: + if self.engine == AWSOS_Engine.faiss: + log.info(f"Using metric type as innerproduct because faiss doesn't support cosine as metric type for Opensearch") + return "innerproduct" return "cosinesimil" return "l2" @@ -49,7 +54,8 @@ def index_param(self) -> dict: "engine": self.engine.value, "parameters": { "ef_construction": self.efConstruction, - "m": self.M + "m": self.M, + "ef_search": self.efSearch } } return params diff --git a/vectordb_bench/backend/clients/aws_opensearch/run.py b/vectordb_bench/backend/clients/aws_opensearch/run.py index 3924cbd75..d2698d139 100644 --- a/vectordb_bench/backend/clients/aws_opensearch/run.py +++ b/vectordb_bench/backend/clients/aws_opensearch/run.py @@ -40,12 +40,12 @@ def create_index(client, index_name): "type": "knn_vector", "dimension": _DIM, "method": { - "engine": "nmslib", + "engine": "faiss", "name": "hnsw", "space_type": "l2", "parameters": { - "ef_construction": 128, - "m": 24, + "ef_construction": 256, + "m": 16, } } } @@ -108,12 +108,43 @@ def search(client, index_name): print('\nSearch not ready, sleep 1s') time.sleep(1) +def optimize_index(client, index_name): + print(f"Starting force merge for index {index_name}") + force_merge_endpoint = f'/{index_name}/_forcemerge?max_num_segments=1&wait_for_completion=false' + force_merge_task_id = client.transport.perform_request('POST', force_merge_endpoint)['task'] + SECONDS_WAITING_FOR_FORCE_MERGE_API_CALL_SEC = 30 + while True: + time.sleep(SECONDS_WAITING_FOR_FORCE_MERGE_API_CALL_SEC) + task_status = client.tasks.get(task_id=force_merge_task_id) + if task_status['completed']: + break + print(f"Completed force merge for index {index_name}") + + +def refresh_index(client, index_name): + print(f"Starting refresh for index {index_name}") + SECONDS_WAITING_FOR_REFRESH_API_CALL_SEC = 30 + while True: + try: + print(f"Starting the Refresh Index..") + client.indices.refresh(index=index_name) + break + except Exception as e: + print( + f"Refresh errored out. Sleeping for {SECONDS_WAITING_FOR_REFRESH_API_CALL_SEC} sec and then Retrying : {e}") + time.sleep(SECONDS_WAITING_FOR_REFRESH_API_CALL_SEC) + continue + print(f"Completed refresh for index {index_name}") + + def main(): client = create_client() try: create_index(client, _INDEX_NAME) bulk_insert(client, _INDEX_NAME) + optimize_index(client, _INDEX_NAME) + refresh_index(client, _INDEX_NAME) search(client, _INDEX_NAME) delete_index(client, _INDEX_NAME) except Exception as e: diff --git a/vectordb_bench/frontend/config/dbCaseConfigs.py b/vectordb_bench/frontend/config/dbCaseConfigs.py index 78d1936d7..68bf83f19 100644 --- a/vectordb_bench/frontend/config/dbCaseConfigs.py +++ b/vectordb_bench/frontend/config/dbCaseConfigs.py @@ -360,6 +360,37 @@ class CaseConfigInput(BaseModel): }, ) +CaseConfigParamInput_EFConstruction_AWSOpensearch = CaseConfigInput( + label=CaseConfigParamType.EFConstruction, + inputType=InputType.Number, + inputConfig={ + "min": 100, + "max": 1024, + "value": 256, + }, +) + +CaseConfigParamInput_M_AWSOpensearch = CaseConfigInput( + label=CaseConfigParamType.M, + inputType=InputType.Number, + inputConfig={ + "min": 4, + "max": 64, + "value": 16, + }, +) + +CaseConfigParamInput_EF_SEARCH_AWSOpensearch = CaseConfigInput( + label=CaseConfigParamType.ef_search, + inputType=InputType.Number, + inputConfig={ + "min": 100, + "max": 1024, + "value": 256, + }, +) + + CaseConfigParamInput_maintenance_work_mem_PgVector = CaseConfigInput( label=CaseConfigParamType.maintenance_work_mem, inputHelp="Recommended value: 1.33x the index size, not to exceed the available free memory." @@ -839,6 +870,13 @@ class CaseConfigInput(BaseModel): CaseConfigParamInput_NumCandidates_ES, ] +AWSOpensearchLoadingConfig = [CaseConfigParamInput_EFConstruction_AWSOpensearch, CaseConfigParamInput_M_AWSOpensearch] +AWSOpenSearchPerformanceConfig = [ + CaseConfigParamInput_EFConstruction_AWSOpensearch, + CaseConfigParamInput_M_AWSOpensearch, + CaseConfigParamInput_EF_SEARCH_AWSOpensearch, +] + PgVectorLoadingConfig = [ CaseConfigParamInput_IndexType_PgVector, CaseConfigParamInput_Lists_PgVector, @@ -920,6 +958,10 @@ class CaseConfigInput(BaseModel): CaseLabel.Load: ESLoadingConfig, CaseLabel.Performance: ESPerformanceConfig, }, + DB.AWSOpenSearch: { + CaseLabel.Load: AWSOpensearchLoadingConfig, + CaseLabel.Performance: AWSOpenSearchPerformanceConfig, + }, DB.PgVector: { CaseLabel.Load: PgVectorLoadingConfig, CaseLabel.Performance: PgVectorPerformanceConfig, From 467b00aac779b326673f0b273249cbd2cc18a99a Mon Sep 17 00:00:00 2001 From: "min.tian" Date: Fri, 18 Oct 2024 11:02:23 +0800 Subject: [PATCH 087/327] record the test time; add version / note info for milvus and zillizcloud results; Signed-off-by: min.tian --- vectordb_bench/frontend/vdb_benchmark.py | 14 +- vectordb_bench/models.py | 8 +- .../result_20230727_standard_milvus.json | 54 ++++++- .../result_20230808_standard_milvus.json | 48 ++++++ .../result_20230727_standard_zillizcloud.json | 30 +++- .../result_20230808_standard_zillizcloud.json | 24 +++ ..._20240105_standard_202401_zillizcloud.json | 147 ++++++++++++------ vectordb_bench/results/getLeaderboardData.py | 24 ++- vectordb_bench/results/leaderboard.json | 2 +- 9 files changed, 287 insertions(+), 64 deletions(-) diff --git a/vectordb_bench/frontend/vdb_benchmark.py b/vectordb_bench/frontend/vdb_benchmark.py index b859c68b8..c76a2f3de 100644 --- a/vectordb_bench/frontend/vdb_benchmark.py +++ b/vectordb_bench/frontend/vdb_benchmark.py @@ -1,8 +1,13 @@ import streamlit as st from vectordb_bench.frontend.components.check_results.footer import footer -from vectordb_bench.frontend.components.check_results.stPageConfig import initResultsPageConfig +from vectordb_bench.frontend.components.check_results.stPageConfig import ( + initResultsPageConfig, +) from vectordb_bench.frontend.components.check_results.headerIcon import drawHeaderIcon -from vectordb_bench.frontend.components.check_results.nav import NavToQuriesPerDollar, NavToRunTest +from vectordb_bench.frontend.components.check_results.nav import ( + NavToQuriesPerDollar, + NavToRunTest, +) from vectordb_bench.frontend.components.check_results.charts import drawCharts from vectordb_bench.frontend.components.check_results.filters import getshownData from vectordb_bench.frontend.components.get_results.saveAsImage import getResults @@ -20,7 +25,10 @@ def main(): allResults = benchMarkRunner.get_results() st.title("Vector Database Benchmark") - st.caption("Note that all testing was completed in July 2023, except for the times already noted.") + st.caption( + "Except for zillizcloud-v2024.1, which was tested in _January 2024_, all other tests were completed before _August 2023_." + ) + st.caption("All tested milvus are in _standalone_ mode.") # results selector and filter resultSelectorContainer = st.sidebar.container() diff --git a/vectordb_bench/models.py b/vectordb_bench/models.py index 75b427091..e4c361855 100644 --- a/vectordb_bench/models.py +++ b/vectordb_bench/models.py @@ -163,16 +163,20 @@ class TestResult(BaseModel): results: list[CaseResult] file_fmt: str = "result_{}_{}_{}.json" # result_20230718_statndard_milvus.json + timestamp: float = 0.0 def flush(self): db2case = self.get_db_results() - + timestamp = date.today().timestamp() result_root = config.RESULTS_LOCAL_DIR for db, result in db2case.items(): self.write_db_file( result_dir=result_root.joinpath(db.value), partial=TestResult( - run_id=self.run_id, task_label=self.task_label, results=result + run_id=self.run_id, + task_label=self.task_label, + results=result, + timestamp=timestamp, ), db=db.value.lower(), ) diff --git a/vectordb_bench/results/Milvus/result_20230727_standard_milvus.json b/vectordb_bench/results/Milvus/result_20230727_standard_milvus.json index cc9944ef3..47b0d4138 100644 --- a/vectordb_bench/results/Milvus/result_20230727_standard_milvus.json +++ b/vectordb_bench/results/Milvus/result_20230727_standard_milvus.json @@ -14,6 +14,8 @@ "db": "Milvus", "db_config": { "db_label": "2c8g-disk", + "version": "v2.2.12", + "note": "standalone mode", "uri": "**********" }, "db_case_config": { @@ -40,6 +42,8 @@ "db": "Milvus", "db_config": { "db_label": "2c8g-disk", + "version": "v2.2.12", + "note": "standalone mode", "uri": "**********" }, "db_case_config": { @@ -66,6 +70,8 @@ "db": "Milvus", "db_config": { "db_label": "2c8g-disk", + "version": "v2.2.12", + "note": "standalone mode", "uri": "**********" }, "db_case_config": { @@ -92,6 +98,8 @@ "db": "Milvus", "db_config": { "db_label": "2c8g-disk", + "version": "v2.2.12", + "note": "standalone mode", "uri": "**********" }, "db_case_config": { @@ -118,6 +126,8 @@ "db": "Milvus", "db_config": { "db_label": "2c8g-disk", + "version": "v2.2.12", + "note": "standalone mode", "uri": "**********" }, "db_case_config": { @@ -144,6 +154,8 @@ "db": "Milvus", "db_config": { "db_label": "2c8g-disk", + "version": "v2.2.12", + "note": "standalone mode", "uri": "**********" }, "db_case_config": { @@ -170,6 +182,8 @@ "db": "Milvus", "db_config": { "db_label": "4c16g-disk", + "version": "v2.2.12", + "note": "standalone mode", "uri": "**********" }, "db_case_config": { @@ -196,6 +210,8 @@ "db": "Milvus", "db_config": { "db_label": "4c16g-disk", + "version": "v2.2.12", + "note": "standalone mode", "uri": "**********" }, "db_case_config": { @@ -222,6 +238,8 @@ "db": "Milvus", "db_config": { "db_label": "4c16g-disk", + "version": "v2.2.12", + "note": "standalone mode", "uri": "**********" }, "db_case_config": { @@ -248,6 +266,8 @@ "db": "Milvus", "db_config": { "db_label": "4c16g-disk", + "version": "v2.2.12", + "note": "standalone mode", "uri": "**********" }, "db_case_config": { @@ -276,6 +296,8 @@ "db": "Milvus", "db_config": { "db_label": "4c16g-disk", + "version": "v2.2.12", + "note": "standalone mode", "uri": "**********" }, "db_case_config": { @@ -304,6 +326,8 @@ "db": "Milvus", "db_config": { "db_label": "4c16g-disk", + "version": "v2.2.12", + "note": "standalone mode", "uri": "**********" }, "db_case_config": { @@ -332,6 +356,8 @@ "db": "Milvus", "db_config": { "db_label": "2c8g-hnsw", + "version": "v2.2.12", + "note": "standalone mode", "uri": "**********" }, "db_case_config": { @@ -360,6 +386,8 @@ "db": "Milvus", "db_config": { "db_label": "2c8g-hnsw", + "version": "v2.2.12", + "note": "standalone mode", "uri": "**********" }, "db_case_config": { @@ -388,6 +416,8 @@ "db": "Milvus", "db_config": { "db_label": "2c8g-hnsw", + "version": "v2.2.12", + "note": "standalone mode", "uri": "**********" }, "db_case_config": { @@ -416,6 +446,8 @@ "db": "Milvus", "db_config": { "db_label": "2c8g-hnsw", + "version": "v2.2.12", + "note": "standalone mode", "uri": "**********" }, "db_case_config": { @@ -444,6 +476,8 @@ "db": "Milvus", "db_config": { "db_label": "2c8g-hnsw", + "version": "v2.2.12", + "note": "standalone mode", "uri": "**********" }, "db_case_config": { @@ -472,6 +506,8 @@ "db": "Milvus", "db_config": { "db_label": "2c8g-hnsw", + "version": "v2.2.12", + "note": "standalone mode", "uri": "**********" }, "db_case_config": { @@ -500,6 +536,8 @@ "db": "Milvus", "db_config": { "db_label": "2c8g-hnsw", + "version": "v2.2.12", + "note": "standalone mode", "uri": "**********" }, "db_case_config": { @@ -528,6 +566,8 @@ "db": "Milvus", "db_config": { "db_label": "2c8g-hnsw", + "version": "v2.2.12", + "note": "standalone mode", "uri": "**********" }, "db_case_config": { @@ -556,6 +596,8 @@ "db": "Milvus", "db_config": { "db_label": "16c64g-hnsw", + "version": "v2.2.12", + "note": "standalone mode", "uri": "**********" }, "db_case_config": { @@ -584,6 +626,8 @@ "db": "Milvus", "db_config": { "db_label": "16c64g-hnsw", + "version": "v2.2.12", + "note": "standalone mode", "uri": "**********" }, "db_case_config": { @@ -612,6 +656,8 @@ "db": "Milvus", "db_config": { "db_label": "16c64g-hnsw", + "version": "v2.2.12", + "note": "standalone mode", "uri": "**********" }, "db_case_config": { @@ -640,6 +686,8 @@ "db": "Milvus", "db_config": { "db_label": "16c64g-hnsw", + "version": "v2.2.12", + "note": "standalone mode", "uri": "**********" }, "db_case_config": { @@ -668,6 +716,8 @@ "db": "Milvus", "db_config": { "db_label": "16c64g-hnsw", + "version": "v2.2.12", + "note": "standalone mode", "uri": "**********" }, "db_case_config": { @@ -696,6 +746,8 @@ "db": "Milvus", "db_config": { "db_label": "16c64g-hnsw", + "version": "v2.2.12", + "note": "standalone mode", "uri": "**********" }, "db_case_config": { @@ -714,4 +766,4 @@ } ], "file_fmt": "result_{}_{}_{}.json" -} +} \ No newline at end of file diff --git a/vectordb_bench/results/Milvus/result_20230808_standard_milvus.json b/vectordb_bench/results/Milvus/result_20230808_standard_milvus.json index bfc48ac58..30c59c9ad 100644 --- a/vectordb_bench/results/Milvus/result_20230808_standard_milvus.json +++ b/vectordb_bench/results/Milvus/result_20230808_standard_milvus.json @@ -14,6 +14,8 @@ "db": "Milvus", "db_config": { "db_label": "2c8g-disk", + "version": "v2.2.12", + "note": "standalone mode", "uri": "**********" }, "db_case_config": { @@ -40,6 +42,8 @@ "db": "Milvus", "db_config": { "db_label": "2c8g-disk", + "version": "v2.2.12", + "note": "standalone mode", "uri": "**********" }, "db_case_config": { @@ -66,6 +70,8 @@ "db": "Milvus", "db_config": { "db_label": "2c8g-disk", + "version": "v2.2.12", + "note": "standalone mode", "uri": "**********" }, "db_case_config": { @@ -92,6 +98,8 @@ "db": "Milvus", "db_config": { "db_label": "2c8g-disk", + "version": "v2.2.12", + "note": "standalone mode", "uri": "**********" }, "db_case_config": { @@ -118,6 +126,8 @@ "db": "Milvus", "db_config": { "db_label": "2c8g-disk", + "version": "v2.2.12", + "note": "standalone mode", "uri": "**********" }, "db_case_config": { @@ -144,6 +154,8 @@ "db": "Milvus", "db_config": { "db_label": "2c8g-disk", + "version": "v2.2.12", + "note": "standalone mode", "uri": "**********" }, "db_case_config": { @@ -170,6 +182,8 @@ "db": "Milvus", "db_config": { "db_label": "4c16g-disk", + "version": "v2.2.12", + "note": "standalone mode", "uri": "**********" }, "db_case_config": { @@ -196,6 +210,8 @@ "db": "Milvus", "db_config": { "db_label": "4c16g-disk", + "version": "v2.2.12", + "note": "standalone mode", "uri": "**********" }, "db_case_config": { @@ -222,6 +238,8 @@ "db": "Milvus", "db_config": { "db_label": "4c16g-disk", + "version": "v2.2.12", + "note": "standalone mode", "uri": "**********" }, "db_case_config": { @@ -248,6 +266,8 @@ "db": "Milvus", "db_config": { "db_label": "4c16g-disk", + "version": "v2.2.12", + "note": "standalone mode", "uri": "**********" }, "db_case_config": { @@ -274,6 +294,8 @@ "db": "Milvus", "db_config": { "db_label": "4c16g-disk", + "version": "v2.2.12", + "note": "standalone mode", "uri": "**********" }, "db_case_config": { @@ -300,6 +322,8 @@ "db": "Milvus", "db_config": { "db_label": "4c16g-disk", + "version": "v2.2.12", + "note": "standalone mode", "uri": "**********" }, "db_case_config": { @@ -326,6 +350,8 @@ "db": "Milvus", "db_config": { "db_label": "2c8g-hnsw", + "version": "v2.2.12", + "note": "standalone mode", "uri": "**********" }, "db_case_config": { @@ -354,6 +380,8 @@ "db": "Milvus", "db_config": { "db_label": "2c8g-hnsw", + "version": "v2.2.12", + "note": "standalone mode", "uri": "**********" }, "db_case_config": { @@ -382,6 +410,8 @@ "db": "Milvus", "db_config": { "db_label": "2c8g-hnsw", + "version": "v2.2.12", + "note": "standalone mode", "uri": "**********" }, "db_case_config": { @@ -410,6 +440,8 @@ "db": "Milvus", "db_config": { "db_label": "2c8g-hnsw", + "version": "v2.2.12", + "note": "standalone mode", "uri": "**********" }, "db_case_config": { @@ -438,6 +470,8 @@ "db": "Milvus", "db_config": { "db_label": "2c8g-hnsw", + "version": "v2.2.12", + "note": "standalone mode", "uri": "**********" }, "db_case_config": { @@ -466,6 +500,8 @@ "db": "Milvus", "db_config": { "db_label": "2c8g-hnsw", + "version": "v2.2.12", + "note": "standalone mode", "uri": "**********" }, "db_case_config": { @@ -494,6 +530,8 @@ "db": "Milvus", "db_config": { "db_label": "16c64g-hnsw", + "version": "v2.2.12", + "note": "standalone mode", "uri": "**********" }, "db_case_config": { @@ -522,6 +560,8 @@ "db": "Milvus", "db_config": { "db_label": "16c64g-hnsw", + "version": "v2.2.12", + "note": "standalone mode", "uri": "**********" }, "db_case_config": { @@ -550,6 +590,8 @@ "db": "Milvus", "db_config": { "db_label": "16c64g-hnsw", + "version": "v2.2.12", + "note": "standalone mode", "uri": "**********" }, "db_case_config": { @@ -578,6 +620,8 @@ "db": "Milvus", "db_config": { "db_label": "16c64g-hnsw", + "version": "v2.2.12", + "note": "standalone mode", "uri": "**********" }, "db_case_config": { @@ -606,6 +650,8 @@ "db": "Milvus", "db_config": { "db_label": "16c64g-hnsw", + "version": "v2.2.12", + "note": "standalone mode", "uri": "**********" }, "db_case_config": { @@ -634,6 +680,8 @@ "db": "Milvus", "db_config": { "db_label": "16c64g-hnsw", + "version": "v2.2.12", + "note": "standalone mode", "uri": "**********" }, "db_case_config": { diff --git a/vectordb_bench/results/ZillizCloud/result_20230727_standard_zillizcloud.json b/vectordb_bench/results/ZillizCloud/result_20230727_standard_zillizcloud.json index 919333d77..e20a2eab6 100644 --- a/vectordb_bench/results/ZillizCloud/result_20230727_standard_zillizcloud.json +++ b/vectordb_bench/results/ZillizCloud/result_20230727_standard_zillizcloud.json @@ -14,6 +14,7 @@ "db": "ZillizCloud", "db_config": { "db_label": "1cu-cap", + "version": "v2023.6", "uri": "**********", "user": "root", "password": "**********" @@ -41,6 +42,7 @@ "db": "ZillizCloud", "db_config": { "db_label": "1cu-cap", + "version": "v2023.6", "uri": "**********", "user": "root", "password": "**********" @@ -68,6 +70,7 @@ "db": "ZillizCloud", "db_config": { "db_label": "1cu-cap", + "version": "v2023.6", "uri": "**********", "user": "root", "password": "**********" @@ -95,6 +98,7 @@ "db": "ZillizCloud", "db_config": { "db_label": "1cu-cap", + "version": "v2023.6", "uri": "**********", "user": "root", "password": "**********" @@ -122,6 +126,7 @@ "db": "ZillizCloud", "db_config": { "db_label": "1cu-cap", + "version": "v2023.6", "uri": "**********", "user": "root", "password": "**********" @@ -149,6 +154,7 @@ "db": "ZillizCloud", "db_config": { "db_label": "1cu-cap", + "version": "v2023.6", "uri": "**********", "user": "root", "password": "**********" @@ -176,6 +182,7 @@ "db": "ZillizCloud", "db_config": { "db_label": "1cu-cap", + "version": "v2023.6", "uri": "**********", "user": "root", "password": "**********" @@ -203,6 +210,7 @@ "db": "ZillizCloud", "db_config": { "db_label": "1cu-cap", + "version": "v2023.6", "uri": "**********", "user": "root", "password": "**********" @@ -230,6 +238,7 @@ "db": "ZillizCloud", "db_config": { "db_label": "2cu-cap", + "version": "v2023.6", "uri": "**********", "user": "root", "password": "**********" @@ -257,6 +266,7 @@ "db": "ZillizCloud", "db_config": { "db_label": "2cu-cap", + "version": "v2023.6", "uri": "**********", "user": "root", "password": "**********" @@ -284,6 +294,7 @@ "db": "ZillizCloud", "db_config": { "db_label": "2cu-cap", + "version": "v2023.6", "uri": "**********", "user": "root", "password": "**********" @@ -311,6 +322,7 @@ "db": "ZillizCloud", "db_config": { "db_label": "2cu-cap", + "version": "v2023.6", "uri": "**********", "user": "root", "password": "**********" @@ -338,6 +350,7 @@ "db": "ZillizCloud", "db_config": { "db_label": "2cu-cap", + "version": "v2023.6", "uri": "**********", "user": "root", "password": "**********" @@ -365,6 +378,7 @@ "db": "ZillizCloud", "db_config": { "db_label": "2cu-cap", + "version": "v2023.6", "uri": "**********", "user": "root", "password": "**********" @@ -392,6 +406,7 @@ "db": "ZillizCloud", "db_config": { "db_label": "1cu-perf", + "version": "v2023.6", "uri": "**********", "user": "root", "password": "**********" @@ -419,6 +434,7 @@ "db": "ZillizCloud", "db_config": { "db_label": "1cu-perf", + "version": "v2023.6", "uri": "**********", "user": "root", "password": "**********" @@ -446,6 +462,7 @@ "db": "ZillizCloud", "db_config": { "db_label": "1cu-perf", + "version": "v2023.6", "uri": "**********", "user": "root", "password": "**********" @@ -473,6 +490,7 @@ "db": "ZillizCloud", "db_config": { "db_label": "1cu-perf", + "version": "v2023.6", "uri": "**********", "user": "root", "password": "**********" @@ -500,6 +518,7 @@ "db": "ZillizCloud", "db_config": { "db_label": "1cu-perf", + "version": "v2023.6", "uri": "**********", "user": "root", "password": "**********" @@ -527,6 +546,7 @@ "db": "ZillizCloud", "db_config": { "db_label": "1cu-perf", + "version": "v2023.6", "uri": "**********", "user": "root", "password": "**********" @@ -554,6 +574,7 @@ "db": "ZillizCloud", "db_config": { "db_label": "1cu-perf", + "version": "v2023.6", "uri": "**********", "user": "root", "password": "**********" @@ -581,6 +602,7 @@ "db": "ZillizCloud", "db_config": { "db_label": "1cu-perf", + "version": "v2023.6", "uri": "**********", "user": "root", "password": "**********" @@ -608,6 +630,7 @@ "db": "ZillizCloud", "db_config": { "db_label": "8cu-perf", + "version": "v2023.6", "uri": "**********", "user": "root", "password": "**********" @@ -635,6 +658,7 @@ "db": "ZillizCloud", "db_config": { "db_label": "8cu-perf", + "version": "v2023.6", "uri": "**********", "user": "root", "password": "**********" @@ -662,6 +686,7 @@ "db": "ZillizCloud", "db_config": { "db_label": "8cu-perf", + "version": "v2023.6", "uri": "**********", "user": "root", "password": "**********" @@ -689,6 +714,7 @@ "db": "ZillizCloud", "db_config": { "db_label": "8cu-perf", + "version": "v2023.6", "uri": "**********", "user": "root", "password": "**********" @@ -716,6 +742,7 @@ "db": "ZillizCloud", "db_config": { "db_label": "8cu-perf", + "version": "v2023.6", "uri": "**********", "user": "root", "password": "**********" @@ -743,6 +770,7 @@ "db": "ZillizCloud", "db_config": { "db_label": "8cu-perf", + "version": "v2023.6", "uri": "**********", "user": "root", "password": "**********" @@ -760,4 +788,4 @@ } ], "file_fmt": "result_{}_{}_{}.json" -} +} \ No newline at end of file diff --git a/vectordb_bench/results/ZillizCloud/result_20230808_standard_zillizcloud.json b/vectordb_bench/results/ZillizCloud/result_20230808_standard_zillizcloud.json index 68460e649..a0047bc45 100644 --- a/vectordb_bench/results/ZillizCloud/result_20230808_standard_zillizcloud.json +++ b/vectordb_bench/results/ZillizCloud/result_20230808_standard_zillizcloud.json @@ -14,6 +14,7 @@ "db": "ZillizCloud", "db_config": { "db_label": "1cu-perf", + "version": "v2023.6", "uri": "**********", "user": "root", "password": "**********" @@ -41,6 +42,7 @@ "db": "ZillizCloud", "db_config": { "db_label": "1cu-perf", + "version": "v2023.6", "uri": "**********", "user": "root", "password": "**********" @@ -68,6 +70,7 @@ "db": "ZillizCloud", "db_config": { "db_label": "1cu-perf", + "version": "v", "uri": "**********", "user": "root", "password": "**********" @@ -95,6 +98,7 @@ "db": "ZillizCloud", "db_config": { "db_label": "1cu-perf", + "version": "v2023.6", "uri": "**********", "user": "root", "password": "**********" @@ -122,6 +126,7 @@ "db": "ZillizCloud", "db_config": { "db_label": "1cu-perf", + "version": "v2023.6", "uri": "**********", "user": "root", "password": "**********" @@ -149,6 +154,7 @@ "db": "ZillizCloud", "db_config": { "db_label": "1cu-perf", + "version": "v2023.6", "uri": "**********", "user": "root", "password": "**********" @@ -176,6 +182,7 @@ "db": "ZillizCloud", "db_config": { "db_label": "8cu-perf", + "version": "v2023.6", "uri": "**********", "user": "root", "password": "**********" @@ -203,6 +210,7 @@ "db": "ZillizCloud", "db_config": { "db_label": "8cu-perf", + "version": "v2023.6", "uri": "**********", "user": "root", "password": "**********" @@ -230,6 +238,7 @@ "db": "ZillizCloud", "db_config": { "db_label": "8cu-perf", + "version": "v2023.6", "uri": "**********", "user": "root", "password": "**********" @@ -257,6 +266,7 @@ "db": "ZillizCloud", "db_config": { "db_label": "8cu-perf", + "version": "v2023.6", "uri": "**********", "user": "root", "password": "**********" @@ -284,6 +294,7 @@ "db": "ZillizCloud", "db_config": { "db_label": "8cu-perf", + "version": "v2023.6", "uri": "**********", "user": "root", "password": "**********" @@ -311,6 +322,7 @@ "db": "ZillizCloud", "db_config": { "db_label": "8cu-perf", + "version": "v2023.6", "uri": "**********", "user": "root", "password": "**********" @@ -338,6 +350,7 @@ "db": "ZillizCloud", "db_config": { "db_label": "1cu-cap", + "version": "v2023.6", "uri": "**********", "user": "root", "password": "**********" @@ -365,6 +378,7 @@ "db": "ZillizCloud", "db_config": { "db_label": "1cu-cap", + "version": "v2023.6", "uri": "**********", "user": "root", "password": "**********" @@ -392,6 +406,7 @@ "db": "ZillizCloud", "db_config": { "db_label": "1cu-cap", + "version": "v2023.6", "uri": "**********", "user": "root", "password": "**********" @@ -419,6 +434,7 @@ "db": "ZillizCloud", "db_config": { "db_label": "1cu-cap", + "version": "v2023.6", "uri": "**********", "user": "root", "password": "**********" @@ -446,6 +462,7 @@ "db": "ZillizCloud", "db_config": { "db_label": "1cu-cap", + "version": "v2023.6", "uri": "**********", "user": "root", "password": "**********" @@ -473,6 +490,7 @@ "db": "ZillizCloud", "db_config": { "db_label": "1cu-cap", + "version": "v2023.6", "uri": "**********", "user": "root", "password": "**********" @@ -500,6 +518,7 @@ "db": "ZillizCloud", "db_config": { "db_label": "2cu-cap", + "version": "v2023.6", "uri": "**********", "user": "root", "password": "**********" @@ -527,6 +546,7 @@ "db": "ZillizCloud", "db_config": { "db_label": "2cu-cap", + "version": "v2023.6", "uri": "**********", "user": "root", "password": "**********" @@ -554,6 +574,7 @@ "db": "ZillizCloud", "db_config": { "db_label": "2cu-cap", + "version": "v2023.6", "uri": "**********", "user": "root", "password": "**********" @@ -581,6 +602,7 @@ "db": "ZillizCloud", "db_config": { "db_label": "2cu-cap", + "version": "v2023.6", "uri": "**********", "user": "root", "password": "**********" @@ -608,6 +630,7 @@ "db": "ZillizCloud", "db_config": { "db_label": "2cu-cap", + "version": "v2023.6", "uri": "**********", "user": "root", "password": "**********" @@ -635,6 +658,7 @@ "db": "ZillizCloud", "db_config": { "db_label": "2cu-cap", + "version": "v2023.6", "uri": "**********", "user": "root", "password": "**********" diff --git a/vectordb_bench/results/ZillizCloud/result_20240105_standard_202401_zillizcloud.json b/vectordb_bench/results/ZillizCloud/result_20240105_standard_202401_zillizcloud.json index f37ed02a6..5d9907754 100644 --- a/vectordb_bench/results/ZillizCloud/result_20240105_standard_202401_zillizcloud.json +++ b/vectordb_bench/results/ZillizCloud/result_20240105_standard_202401_zillizcloud.json @@ -13,7 +13,8 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "8cu-perf-(Jan-2024)", + "db_label": "8cu-perf", + "version": "v2024.1", "uri": "**********", "user": "db_admin", "password": "**********" @@ -40,7 +41,8 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "8cu-perf-(Jan-2024)", + "db_label": "8cu-perf", + "version": "v2024.1", "uri": "**********", "user": "db_admin", "password": "**********" @@ -67,7 +69,8 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "8cu-perf-(Jan-2024)", + "db_label": "8cu-perf", + "version": "v2024.1", "uri": "**********", "user": "db_admin", "password": "**********" @@ -94,7 +97,8 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "8cu-perf-(Jan-2024)", + "db_label": "8cu-perf", + "version": "v2024.1", "uri": "**********", "user": "db_admin", "password": "**********" @@ -121,7 +125,8 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "8cu-perf-(Jan-2024)", + "db_label": "8cu-perf", + "version": "v2024.1", "uri": "**********", "user": "db_admin", "password": "**********" @@ -148,7 +153,8 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "8cu-perf-(Jan-2024)", + "db_label": "8cu-perf", + "version": "v2024.1", "uri": "**********", "user": "db_admin", "password": "**********" @@ -175,7 +181,8 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "8cu-perf-(Jan-2024)", + "db_label": "8cu-perf", + "version": "v2024.1", "uri": "**********", "user": "db_admin", "password": "**********" @@ -202,7 +209,8 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "8cu-perf-(Jan-2024)", + "db_label": "8cu-perf", + "version": "v2024.1", "uri": "**********", "user": "db_admin", "password": "**********" @@ -229,7 +237,8 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "8cu-perf-(Jan-2024)", + "db_label": "8cu-perf", + "version": "v2024.1", "uri": "**********", "user": "db_admin", "password": "**********" @@ -256,7 +265,8 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "8cu-perf-(Jan-2024)", + "db_label": "8cu-perf", + "version": "v2024.1", "uri": "**********", "user": "db_admin", "password": "**********" @@ -283,7 +293,8 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "8cu-perf-(Jan-2024)", + "db_label": "8cu-perf", + "version": "v2024.1", "uri": "**********", "user": "db_admin", "password": "**********" @@ -310,7 +321,8 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "8cu-perf-(Jan-2024)", + "db_label": "8cu-perf", + "version": "v2024.1", "uri": "**********", "user": "db_admin", "password": "**********" @@ -337,7 +349,8 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-cap-(Jan-2024)", + "db_label": "1cu-cap", + "version": "v2024.1", "uri": "**********", "user": "db_admin", "password": "**********" @@ -364,7 +377,8 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-cap-(Jan-2024)", + "db_label": "1cu-cap", + "version": "v2024.1", "uri": "**********", "user": "db_admin", "password": "**********" @@ -391,7 +405,8 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-cap-(Jan-2024)", + "db_label": "1cu-cap", + "version": "v2024.1", "uri": "**********", "user": "db_admin", "password": "**********" @@ -418,7 +433,8 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-cap-(Jan-2024)", + "db_label": "1cu-cap", + "version": "v2024.1", "uri": "**********", "user": "db_admin", "password": "**********" @@ -445,7 +461,8 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-cap-(Jan-2024)", + "db_label": "1cu-cap", + "version": "v2024.1", "uri": "**********", "user": "db_admin", "password": "**********" @@ -472,7 +489,8 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-cap-(Jan-2024)", + "db_label": "1cu-cap", + "version": "v2024.1", "uri": "**********", "user": "db_admin", "password": "**********" @@ -499,7 +517,8 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "2cu-cap-(Jan-2024)", + "db_label": "2cu-cap", + "version": "v2024.1", "uri": "**********", "user": "db_admin", "password": "**********" @@ -526,7 +545,8 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "2cu-cap-(Jan-2024)", + "db_label": "2cu-cap", + "version": "v2024.1", "uri": "**********", "user": "db_admin", "password": "**********" @@ -553,7 +573,8 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "2cu-cap-(Jan-2024)", + "db_label": "2cu-cap", + "version": "v2024.1", "uri": "**********", "user": "db_admin", "password": "**********" @@ -580,7 +601,8 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "2cu-cap-(Jan-2024)", + "db_label": "2cu-cap", + "version": "v2024.1", "uri": "**********", "user": "db_admin", "password": "**********" @@ -607,7 +629,8 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "2cu-cap-(Jan-2024)", + "db_label": "2cu-cap", + "version": "v2024.1", "uri": "**********", "user": "db_admin", "password": "**********" @@ -634,7 +657,8 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "2cu-cap-(Jan-2024)", + "db_label": "2cu-cap", + "version": "v2024.1", "uri": "**********", "user": "db_admin", "password": "**********" @@ -661,7 +685,8 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "2cu-cap-(Jan-2024)", + "db_label": "2cu-cap", + "version": "v2024.1", "uri": "**********", "user": "db_admin", "password": "**********" @@ -688,7 +713,8 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "2cu-cap-(Jan-2024)", + "db_label": "2cu-cap", + "version": "v2024.1", "uri": "**********", "user": "db_admin", "password": "**********" @@ -715,7 +741,8 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "2cu-cap-(Jan-2024)", + "db_label": "2cu-cap", + "version": "v2024.1", "uri": "**********", "user": "db_admin", "password": "**********" @@ -742,7 +769,8 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "2cu-cap-(Jan-2024)", + "db_label": "2cu-cap", + "version": "v2024.1", "uri": "**********", "user": "db_admin", "password": "**********" @@ -769,7 +797,8 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "2cu-cap-(Jan-2024)", + "db_label": "2cu-cap", + "version": "v2024.1", "uri": "**********", "user": "db_admin", "password": "**********" @@ -796,7 +825,8 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "2cu-cap-(Jan-2024)", + "db_label": "2cu-cap", + "version": "v2024.1", "uri": "**********", "user": "db_admin", "password": "**********" @@ -823,7 +853,8 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-perf-(Jan-2024)", + "db_label": "1cu-perf", + "version": "v2024.1", "uri": "**********", "user": "db_admin", "password": "**********" @@ -850,7 +881,8 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-perf-(Jan-2024)", + "db_label": "1cu-perf", + "version": "v2024.1", "uri": "**********", "user": "db_admin", "password": "**********" @@ -877,7 +909,8 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-perf-(Jan-2024)", + "db_label": "1cu-perf", + "version": "v2024.1", "uri": "**********", "user": "db_admin", "password": "**********" @@ -904,7 +937,8 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-perf-(Jan-2024)", + "db_label": "1cu-perf", + "version": "v2024.1", "uri": "**********", "user": "db_admin", "password": "**********" @@ -931,7 +965,8 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-perf-(Jan-2024)", + "db_label": "1cu-perf", + "version": "v2024.1", "uri": "**********", "user": "db_admin", "password": "**********" @@ -958,7 +993,8 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-perf-(Jan-2024)", + "db_label": "1cu-perf", + "version": "v2024.1", "uri": "**********", "user": "db_admin", "password": "**********" @@ -985,7 +1021,8 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-perf-(Jan-2024)", + "db_label": "1cu-perf", + "version": "v2024.1", "uri": "**********", "user": "db_admin", "password": "**********" @@ -1012,7 +1049,8 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-perf-(Jan-2024)", + "db_label": "1cu-perf", + "version": "v2024.1", "uri": "**********", "user": "db_admin", "password": "**********" @@ -1039,7 +1077,8 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-perf-(Jan-2024)", + "db_label": "1cu-perf", + "version": "v2024.1", "uri": "**********", "user": "db_admin", "password": "**********" @@ -1066,7 +1105,8 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-perf-(Jan-2024)", + "db_label": "1cu-perf", + "version": "v2024.1", "uri": "**********", "user": "db_admin", "password": "**********" @@ -1093,7 +1133,8 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-perf-(Jan-2024)", + "db_label": "1cu-perf", + "version": "v2024.1", "uri": "**********", "user": "db_admin", "password": "**********" @@ -1120,7 +1161,8 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-perf-(Jan-2024)", + "db_label": "1cu-perf", + "version": "v2024.1", "uri": "**********", "user": "db_admin", "password": "**********" @@ -1147,7 +1189,8 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-cap-(Jan-2024)", + "db_label": "1cu-cap", + "version": "v2024.1", "uri": "**********", "user": "db_admin", "password": "**********" @@ -1174,7 +1217,8 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-cap-(Jan-2024)", + "db_label": "1cu-cap", + "version": "v2024.1", "uri": "**********", "user": "db_admin", "password": "**********" @@ -1201,7 +1245,8 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-cap-(Jan-2024)", + "db_label": "1cu-cap", + "version": "v2024.1", "uri": "**********", "user": "db_admin", "password": "**********" @@ -1228,7 +1273,8 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-cap-(Jan-2024)", + "db_label": "1cu-cap", + "version": "v2024.1", "uri": "**********", "user": "db_admin", "password": "**********" @@ -1255,7 +1301,8 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-cap-(Jan-2024)", + "db_label": "1cu-cap", + "version": "v2024.1", "uri": "**********", "user": "db_admin", "password": "**********" @@ -1282,7 +1329,8 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "1cu-cap-(Jan-2024)", + "db_label": "1cu-cap", + "version": "v2024.1", "uri": "**********", "user": "db_admin", "password": "**********" @@ -1299,5 +1347,6 @@ "label": "x" } ], - "file_fmt": "result_{}_{}_{}.json" + "file_fmt": "result_{}_{}_{}.json", + "timestamp": 1704038400.0 } \ No newline at end of file diff --git a/vectordb_bench/results/getLeaderboardData.py b/vectordb_bench/results/getLeaderboardData.py index c6484514d..222c568ab 100644 --- a/vectordb_bench/results/getLeaderboardData.py +++ b/vectordb_bench/results/getLeaderboardData.py @@ -4,7 +4,8 @@ from vectordb_bench.backend.cases import CaseType from vectordb_bench.frontend.config.dbPrices import DB_DBLABEL_TO_PRICE from vectordb_bench.interface import benchMarkRunner -from vectordb_bench.models import CaseResult, ResultLabel, TestResult +from vectordb_bench.models import ResultLabel, TestResult +from datetime import datetime taskLabelToCode = { ResultLabel.FAILED: -1, @@ -13,12 +14,16 @@ } +def format_time(ts: float) -> str: + default_standard_test_time = datetime(2023, 8, 1) + t = datetime.fromtimestamp(ts) + if t < default_standard_test_time: + t = default_standard_test_time + return t.strftime("%Y-%m") + + def main(): allResults: list[TestResult] = benchMarkRunner.get_results() - results: list[CaseResult] = [] - for result in allResults: - if "standard" in result.task_label: - results += result.results if allResults is not None: data = [ @@ -26,13 +31,18 @@ def main(): "db": d.task_config.db.value, "db_label": d.task_config.db_config.db_label, "db_name": d.task_config.db_name, - "case": d.task_config.case_config.case_id.case_name, + "case": d.task_config.case_config.case_id.case_name(), "qps": d.metrics.qps, "latency": d.metrics.serial_latency_p99, "recall": d.metrics.recall, "label": taskLabelToCode[d.label], + "note": d.task_config.db_config.note, + "version": d.task_config.db_config.version, + "test_time": format_time(test_result.timestamp), } - for d in results + for test_result in allResults + if "standard" in test_result.task_label + for d in test_result.results if d.task_config.case_config.case_id != CaseType.CapacityDim128 and d.task_config.case_config.case_id != CaseType.CapacityDim960 ] diff --git a/vectordb_bench/results/leaderboard.json b/vectordb_bench/results/leaderboard.json index 261a82c94..9f64ade18 100644 --- a/vectordb_bench/results/leaderboard.json +++ b/vectordb_bench/results/leaderboard.json @@ -1 +1 @@ -[{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":15.2269,"latency":861.8,"recall":0.9888,"label":1,"qp$":114368.53745044857},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":15.1749,"latency":774.3,"recall":0.989,"label":1,"qp$":113977.96786981013},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":27.6181,"latency":305.5,"recall":0.9999,"label":1,"qp$":207438.26413519715},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":11.2945,"latency":3611.2000000000003,"recall":0.996,"label":1,"qp$":84832.4640100146},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":17.3271,"latency":3774.7999999999997,"recall":0.9961,"label":1,"qp$":130143.04193615691},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":26.26,"latency":556.1,"recall":0.9999,"label":1,"qp$":197237.6382224077},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":100.6667,"latency":21.1,"recall":0.9909,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":101.1399,"latency":19.7,"recall":0.9907,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":52.2606,"latency":18.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":61.0661,"latency":49.8,"recall":0.9911,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":58.9326,"latency":44.6,"recall":0.9911,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":42.5977,"latency":54.9,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":536.0726,"latency":8.200000000000001,"recall":0.9728,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":467.179,"latency":7.0,"recall":0.9697,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":431.7512,"latency":8.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":274.5407,"latency":4.8999999999999995,"recall":0.9807,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":236.5672,"latency":10.3,"recall":0.981,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":309.4833,"latency":4.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":178.6585,"latency":13.700000000000001,"recall":0.9843,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":178.3732,"latency":15.0,"recall":0.9844,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":229.3526,"latency":12.5,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":1258.7043,"latency":4.8999999999999995,"recall":0.9799,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":1075.8776,"latency":5.3,"recall":0.98,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1494.8493,"latency":4.7,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":37.432,"latency":75.0,"recall":0.9975,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":37.0696,"latency":73.5,"recall":0.9976,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":81.1915,"latency":53.0,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":321.6048,"latency":13.4,"recall":0.989,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":22.1467,"latency":86.8,"recall":0.9972,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":303.2551,"latency":10.9,"recall":0.9876,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":21.5388,"latency":81.69999999999999,"recall":0.997,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":394.5418,"latency":8.3,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":37.878,"latency":45.0,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":180.2757,"latency":6.0,"recall":0.9942,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":179.0033,"latency":6.4,"recall":0.9943,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":526.8846,"latency":3.6,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":626.5243,"latency":6.2,"recall":0.9954,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":78.4227,"latency":25.3,"recall":0.9982,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":599.4213,"latency":6.6,"recall":0.9955,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":78.5351,"latency":26.3,"recall":0.9982,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":2098.2113,"latency":3.4,"recall":1.0,"label":1,"qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":275.6292,"latency":10.0,"recall":1.0,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":10.6271,"latency":730.7,"recall":0.8898,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":10.8507,"latency":733.1999999999999,"recall":0.8897,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":75.7055,"latency":121.2,"recall":0.9999,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":0.8836,"latency":2523.0,"recall":0.8528,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":0.8937,"latency":3720.2000000000003,"recall":0.8525,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":1.2145,"latency":3622.3999999999996,"recall":0.7487,"label":1,"qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":18.7634,"latency":153.70000000000002,"recall":0.8737,"label":1,"qp$":694226.515930113},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":18.3619,"latency":79.8,"recall":0.8741,"label":1,"qp$":679371.4285714285},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":25.2744,"latency":61.199999999999996,"recall":0.9979,"label":1,"qp$":935126.8242548818},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":261.798,"latency":23.099999999999998,"recall":0.9262,"label":1,"qp$":6455293.150684931},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":166.1851,"latency":23.900000000000002,"recall":0.9264,"label":1,"qp$":4097714.7945205485},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":121.7169,"latency":29.0,"recall":0.9693,"label":1,"qp$":3001238.6301369863},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":46.6189,"latency":43.1,"recall":0.8737,"label":1,"qp$":1724851.3874614593},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":42.4856,"latency":44.0,"recall":0.8741,"label":1,"qp$":1571923.5354573485},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":138.9479,"latency":26.200000000000003,"recall":0.9979,"label":1,"qp$":5140929.496402878},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":20.7437,"latency":75.80000000000001,"recall":0.9291,"label":1,"qp$":382960.6153846154},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":20.2993,"latency":76.5,"recall":0.9293,"label":1,"qp$":374756.30769230763},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":26.4719,"latency":67.0,"recall":1.0,"label":1,"qp$":488712.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":8.6675,"latency":180.2,"recall":0.8369,"label":1,"qp$":160015.38461538462},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":7.8121,"latency":167.7,"recall":0.8369,"label":1,"qp$":144223.3846153846},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":16.869,"latency":87.8,"recall":0.9814,"label":1,"qp$":311427.6923076923},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":365.0835,"latency":23.599999999999998,"recall":0.945,"label":1,"qp$":1125257.3630136987},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":325.5271,"latency":25.1,"recall":0.9452,"label":1,"qp$":1003336.9520547946},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":596.7942,"latency":24.2,"recall":0.9693,"label":1,"qp$":1839434.1780821919},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":303.2044,"latency":27.400000000000002,"recall":0.9246,"label":1,"qp$":934534.1095890411},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":136.0345,"latency":31.9,"recall":0.9244,"label":1,"qp$":419284.4178082192},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":66.7221,"latency":42.1,"recall":0.963,"label":1,"qp$":205650.30821917808},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":131.2549,"latency":30.200000000000003,"recall":0.9867,"label":1,"qp$":606569.4993581515},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":127.9337,"latency":30.099999999999998,"recall":0.9869,"label":1,"qp$":591221.2066752247},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":595.8462,"latency":23.400000000000002,"recall":1.0,"label":1,"qp$":2753589.627727856},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":228.4,"latency":22.2,"recall":0.9348,"label":1,"qp$":5631780.821917809},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":181.5,"latency":26.1,"recall":0.9345,"label":1,"qp$":4475342.465753425},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":205.7,"latency":24.2,"recall":0.9586,"label":1,"qp$":5072054.794520548},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":67.63,"latency":36.0,"recall":0.8064,"label":1,"qp$":2502240.4933196297},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":63.35,"latency":38.4,"recall":0.8065,"label":1,"qp$":2343884.892086331},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":176.7,"latency":27.599999999999998,"recall":1.0,"label":1,"qp$":6537718.396711202},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":15.33,"latency":84.9,"recall":0.8064,"label":1,"qp$":567194.2446043165},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":15.13,"latency":86.7,"recall":0.8065,"label":1,"qp$":559794.4501541625},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":17.41,"latency":74.30000000000001,"recall":1.0,"label":1,"qp$":644152.1068859198},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":16.34,"latency":88.7,"recall":0.879,"label":1,"qp$":301661.53846153844},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":10.45,"latency":126.8,"recall":0.8208,"label":1,"qp$":192923.0769230769},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":16.18,"latency":87.5,"recall":0.8793,"label":1,"qp$":298707.69230769225},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":9.8,"latency":130.89999999999998,"recall":0.8212,"label":1,"qp$":180923.0769230769},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":36.11,"latency":55.1,"recall":1.0,"label":1,"qp$":666646.1538461539},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":14.84,"latency":92.7,"recall":0.96,"label":1,"qp$":273969.23076923075},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":322.7,"latency":26.4,"recall":0.9478,"label":1,"qp$":994623.287671233},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":265.5,"latency":26.9,"recall":0.9332,"label":1,"qp$":818321.9178082192},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":303.8,"latency":27.3,"recall":0.9478,"label":1,"qp$":936369.8630136987},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":180.2,"latency":28.2,"recall":0.9335,"label":1,"qp$":555410.9589041097},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":730.7,"latency":24.6,"recall":0.9586,"label":1,"qp$":2252157.5342465756},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":104.3,"latency":31.7,"recall":0.9563,"label":1,"qp$":321472.602739726},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":147.7,"latency":35.3,"recall":0.9707,"label":1,"qp$":682567.3940949935},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":782.5,"latency":25.9,"recall":1.0,"label":1,"qp$":3616174.5827984596},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":537.4975,"latency":18.9,"recall":0.8903,"label":1,"qp$":1356936.1851332397},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":372.0466,"latency":17.8,"recall":0.8904,"label":1,"qp$":939248.0785413746},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1156.2898,"latency":14.4,"recall":0.9989,"label":1,"qp$":2919104.684431978},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":110.248,"latency":69.0,"recall":0.898,"label":1,"qp$":278325.94670406735},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":87.2601,"latency":27.799999999999997,"recall":0.898,"label":1,"qp$":220291.97755960727},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":125.7846,"latency":23.099999999999998,"recall":0.975,"label":1,"qp$":317548.77980364655},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":240.7209,"latency":17.4,"recall":0.8887,"label":1,"qp$":3038552.734922861},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":189.4399,"latency":17.5,"recall":0.8889,"label":1,"qp$":2391246.98457223},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":313.5116,"latency":16.1,"recall":0.9999,"label":1,"qp$":3957369.424964937},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":78.7196,"latency":49.4,"recall":0.9203,"label":1,"qp$":1707172.048192771},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":68.3111,"latency":35.5,"recall":0.9202,"label":1,"qp$":1481445.5421686745},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":210.2147,"latency":26.8,"recall":0.9996,"label":1,"qp$":4558873.012048192},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":188.6436,"latency":917.5,"recall":0.9175,"label":1,"qp$":2381195.5119214584},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":155.6991,"latency":917.1,"recall":0.9171,"label":1,"qp$":1965346.283309958},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":445.3289,"latency":14.1,"recall":0.9999,"label":1,"qp$":5621262.412342216},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":633.6033,"latency":24.6,"recall":0.919,"label":1,"qp$":1599559.5231416551},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":95.5682,"latency":58.7,"recall":0.9463,"label":1,"qp$":241266.14305750353},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":434.4062,"latency":17.4,"recall":0.9181,"label":1,"qp$":1096677.643758766},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":80.3192,"latency":26.599999999999998,"recall":0.9462,"label":1,"qp$":202769.3688639551},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":1509.3293,"latency":18.5,"recall":0.9995,"label":1,"qp$":3810368.4992987383},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":166.7252,"latency":19.599999999999998,"recall":0.9988,"label":1,"qp$":420905.1332398317},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":67.9121,"latency":179.5,"recall":0.9909,"label":1,"qp$":7499.495705521471},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.7636,"latency":1921.3,"recall":0.9908,"label":1,"qp$":84.32392638036809},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":32.0,"latency":124.5,"recall":1.0,"label":1,"qp$":3533.7423312883434},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":63.1365,"latency":145.7,"recall":0.991,"label":1,"qp$":22504.09900990099},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.7512,"latency":1983.8,"recall":0.9908,"label":1,"qp$":267.7544554455445},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":30.1358,"latency":129.8,"recall":1.0,"label":1,"qp$":10741.473267326734},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":46.8622,"latency":123.0,"recall":0.9957,"label":1,"qp$":16703.358415841583},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1.8388,"latency":1063.5,"recall":0.9957,"label":1,"qp$":655.4138613861386},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":45.0666,"latency":109.2,"recall":1.0,"label":1,"qp$":16063.342574257427},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":43.5017,"latency":228.7,"recall":0.9957,"label":1,"qp$":4803.8687116564415},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1.5668,"latency":1114.4,"recall":0.9957,"label":1,"qp$":173.02085889570552},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":41.5443,"latency":159.0,"recall":1.0,"label":1,"qp$":4587.714110429448},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":297.5,"latency":7.2,"recall":0.974,"label":1,"qp$":6735849.0566037735},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":228.3,"latency":10.6,"recall":0.994,"label":1,"qp$":5169056.603773585},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":584.0,"latency":4.6,"recall":1.0,"label":1,"qp$":13222641.509433962},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":1871.0,"latency":7.0,"recall":0.9602,"label":1,"qp$":5295283.018867925},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":556.7,"latency":6.7,"recall":0.9723,"label":1,"qp$":1575566.0377358492},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1583.0,"latency":6.8,"recall":0.9836,"label":1,"qp$":4480188.679245283},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":294.3,"latency":10.9,"recall":0.9939,"label":1,"qp$":832924.5283018869},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":2345.0,"latency":8.9,"recall":1.0,"label":1,"qp$":6636792.452830189},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":295.6,"latency":12.3,"recall":1.0,"label":1,"qp$":836603.7735849057},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":143.0,"latency":33.5,"recall":0.9818,"label":1,"qp$":3237735.8490566034},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":106.0,"latency":20.7,"recall":0.9887,"label":1,"qp$":2400000.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":189.0,"latency":11.6,"recall":1.0,"label":1,"qp$":4279245.2830188675},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":379.9721,"latency":12.4,"recall":0.982,"label":1,"qp$":4301570.9433962265},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":71.74,"latency":50.8,"recall":0.9883,"label":1,"qp$":812150.9433962264},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":287.0,"latency":14.9,"recall":0.9865,"label":1,"qp$":3249056.603773585},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":34.6654,"latency":64.69999999999999,"recall":0.9961,"label":1,"qp$":392438.4905660377},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":412.0,"latency":10.3,"recall":1.0,"label":1,"qp$":4664150.9433962265},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":42.169,"latency":46.800000000000004,"recall":1.0,"label":1,"qp$":477384.9056603773},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":330.0144,"latency":9.0,"recall":0.9507,"label":1,"qp$":7472024.150943397},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":271.6585,"latency":10.1,"recall":0.9678,"label":1,"qp$":6150758.490566038},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":216.5226,"latency":12.9,"recall":1.0,"label":1,"qp$":4902398.490566038},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":123.9553,"latency":23.0,"recall":0.971,"label":1,"qp$":1403267.5471698113},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":59.1479,"latency":44.5,"recall":0.9906,"label":1,"qp$":669598.8679245282},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":40.999,"latency":55.300000000000004,"recall":1.0,"label":1,"qp$":464139.62264150946},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":579.9416,"latency":9.4,"recall":0.9213,"label":1,"qp$":6565376.603773585},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":425.2529,"latency":11.299999999999999,"recall":0.9686,"label":1,"qp$":4814183.773584905},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":397.0539,"latency":13.799999999999999,"recall":1.0,"label":1,"qp$":4494949.811320755},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":516.27,"latency":7.0,"recall":0.9463,"label":1,"qp$":11689132.075471697},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":354.8416,"latency":10.0,"recall":0.9802,"label":1,"qp$":8034149.433962264},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":427.5229,"latency":8.7,"recall":1.0,"label":1,"qp$":9679763.773584906},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":2884.689,"latency":5.3,"recall":0.8801,"label":1,"qp$":8164214.150943396},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":1689.5799,"latency":6.6,"recall":0.9493,"label":1,"qp$":4781829.905660377},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1517.6792,"latency":10.0,"recall":1.0,"label":1,"qp$":4295318.4905660385},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":822.5318,"latency":5.6,"recall":0.9294,"label":1,"qp$":2327920.1886792453},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":378.9146,"latency":10.3,"recall":0.9758,"label":1,"qp$":1072399.8113207547},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":218.6854,"latency":16.2,"recall":1.0,"label":1,"qp$":618920.9433962263},{"db":"ZillizCloud","db_label":"8cu-perf-(Jan-2024)","db_name":"ZillizCloud-8cu-perf-(Jan-2024)","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":5115.5303,"latency":8.7,"recall":0.9469,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-(Jan-2024)","db_name":"ZillizCloud-8cu-perf-(Jan-2024)","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":3685.0767,"latency":9.1,"recall":0.9736,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-(Jan-2024)","db_name":"ZillizCloud-8cu-perf-(Jan-2024)","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":4742.1617,"latency":9.1,"recall":0.9936,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-(Jan-2024)","db_name":"ZillizCloud-8cu-perf-(Jan-2024)","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":6054.4428,"latency":8.1,"recall":0.9155,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-(Jan-2024)","db_name":"ZillizCloud-8cu-perf-(Jan-2024)","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":4104.2598,"latency":10.6,"recall":0.9506,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-(Jan-2024)","db_name":"ZillizCloud-8cu-perf-(Jan-2024)","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":4252.1267,"latency":9.2,"recall":0.9964,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-(Jan-2024)","db_name":"ZillizCloud-8cu-perf-(Jan-2024)","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":1685.3091,"latency":13.299999999999999,"recall":0.9718,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-(Jan-2024)","db_name":"ZillizCloud-8cu-perf-(Jan-2024)","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":769.8991,"latency":10.7,"recall":0.9884,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-(Jan-2024)","db_name":"ZillizCloud-8cu-perf-(Jan-2024)","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":945.4061,"latency":10.9,"recall":0.9941,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-(Jan-2024)","db_name":"ZillizCloud-8cu-perf-(Jan-2024)","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":2214.9028,"latency":8.4,"recall":0.9249,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-(Jan-2024)","db_name":"ZillizCloud-8cu-perf-(Jan-2024)","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":827.975,"latency":12.0,"recall":0.9692,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf-(Jan-2024)","db_name":"ZillizCloud-8cu-perf-(Jan-2024)","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":776.9454,"latency":11.4,"recall":0.9966,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-(Jan-2024)","db_name":"ZillizCloud-1cu-cap-(Jan-2024)","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":269.5464,"latency":9.799999999999999,"recall":0.9776,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-(Jan-2024)","db_name":"ZillizCloud-1cu-cap-(Jan-2024)","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":240.0363,"latency":10.6,"recall":0.9822,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-(Jan-2024)","db_name":"ZillizCloud-1cu-cap-(Jan-2024)","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":218.0627,"latency":11.4,"recall":0.9936,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-(Jan-2024)","db_name":"ZillizCloud-1cu-cap-(Jan-2024)","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":392.8825,"latency":6.8999999999999995,"recall":0.9581,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-(Jan-2024)","db_name":"ZillizCloud-1cu-cap-(Jan-2024)","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":343.8204,"latency":8.4,"recall":0.968,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-(Jan-2024)","db_name":"ZillizCloud-1cu-cap-(Jan-2024)","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":216.6773,"latency":10.1,"recall":0.9968,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-(Jan-2024)","db_name":"ZillizCloud-2cu-cap-(Jan-2024)","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":503.2284,"latency":9.0,"recall":0.9677,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-(Jan-2024)","db_name":"ZillizCloud-2cu-cap-(Jan-2024)","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":413.3232,"latency":9.6,"recall":0.981,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-(Jan-2024)","db_name":"ZillizCloud-2cu-cap-(Jan-2024)","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":425.5492,"latency":10.200000000000001,"recall":0.9936,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-(Jan-2024)","db_name":"ZillizCloud-2cu-cap-(Jan-2024)","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":789.1229,"latency":5.6,"recall":0.9396,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-(Jan-2024)","db_name":"ZillizCloud-2cu-cap-(Jan-2024)","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":571.4257,"latency":7.7,"recall":0.9668,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-(Jan-2024)","db_name":"ZillizCloud-2cu-cap-(Jan-2024)","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":411.7653,"latency":9.1,"recall":0.9968,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-(Jan-2024)","db_name":"ZillizCloud-2cu-cap-(Jan-2024)","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":98.0448,"latency":16.1,"recall":0.9803,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-(Jan-2024)","db_name":"ZillizCloud-2cu-cap-(Jan-2024)","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":58.3152,"latency":28.0,"recall":0.9891,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-(Jan-2024)","db_name":"ZillizCloud-2cu-cap-(Jan-2024)","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":46.8304,"latency":28.6,"recall":0.9941,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-(Jan-2024)","db_name":"ZillizCloud-2cu-cap-(Jan-2024)","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":170.5693,"latency":8.9,"recall":0.9605,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-(Jan-2024)","db_name":"ZillizCloud-2cu-cap-(Jan-2024)","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":94.7766,"latency":19.599999999999998,"recall":0.9843,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap-(Jan-2024)","db_name":"ZillizCloud-2cu-cap-(Jan-2024)","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":44.8695,"latency":30.900000000000002,"recall":0.9966,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-(Jan-2024)","db_name":"ZillizCloud-1cu-perf-(Jan-2024)","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":722.0315,"latency":7.7,"recall":0.976,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-(Jan-2024)","db_name":"ZillizCloud-1cu-perf-(Jan-2024)","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":467.5795,"latency":8.8,"recall":0.9898,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-(Jan-2024)","db_name":"ZillizCloud-1cu-perf-(Jan-2024)","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":975.2503,"latency":8.200000000000001,"recall":0.9936,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-(Jan-2024)","db_name":"ZillizCloud-1cu-perf-(Jan-2024)","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":873.3712,"latency":6.7,"recall":0.9477,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-(Jan-2024)","db_name":"ZillizCloud-1cu-perf-(Jan-2024)","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":544.6203,"latency":8.4,"recall":0.977,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-(Jan-2024)","db_name":"ZillizCloud-1cu-perf-(Jan-2024)","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":930.9164,"latency":9.6,"recall":0.9968,"label":1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-(Jan-2024)","db_name":"ZillizCloud-1cu-perf-(Jan-2024)","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-(Jan-2024)","db_name":"ZillizCloud-1cu-perf-(Jan-2024)","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-(Jan-2024)","db_name":"ZillizCloud-1cu-perf-(Jan-2024)","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-(Jan-2024)","db_name":"ZillizCloud-1cu-perf-(Jan-2024)","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-(Jan-2024)","db_name":"ZillizCloud-1cu-perf-(Jan-2024)","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf-(Jan-2024)","db_name":"ZillizCloud-1cu-perf-(Jan-2024)","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-(Jan-2024)","db_name":"ZillizCloud-1cu-cap-(Jan-2024)","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-(Jan-2024)","db_name":"ZillizCloud-1cu-cap-(Jan-2024)","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-(Jan-2024)","db_name":"ZillizCloud-1cu-cap-(Jan-2024)","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-(Jan-2024)","db_name":"ZillizCloud-1cu-cap-(Jan-2024)","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-(Jan-2024)","db_name":"ZillizCloud-1cu-cap-(Jan-2024)","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap-(Jan-2024)","db_name":"ZillizCloud-1cu-cap-(Jan-2024)","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"qp$":0.0}] \ No newline at end of file +[{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":15.2269,"latency":861.8,"recall":0.9888,"label":1,"note":"","version":"","test_time":"2023-08","qp$":114368.53745044857},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":15.1749,"latency":774.3,"recall":0.989,"label":1,"note":"","version":"","test_time":"2023-08","qp$":113977.96786981013},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":27.6181,"latency":305.5,"recall":0.9999,"label":1,"note":"","version":"","test_time":"2023-08","qp$":207438.26413519715},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":11.2945,"latency":3611.2000000000003,"recall":0.996,"label":1,"note":"","version":"","test_time":"2023-08","qp$":84832.4640100146},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":17.3271,"latency":3774.7999999999997,"recall":0.9961,"label":1,"note":"","version":"","test_time":"2023-08","qp$":130143.04193615691},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":26.26,"latency":556.1,"recall":0.9999,"label":1,"note":"","version":"","test_time":"2023-08","qp$":197237.6382224077},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":100.6667,"latency":21.1,"recall":0.9909,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":101.1399,"latency":19.7,"recall":0.9907,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":52.2606,"latency":18.3,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":61.0661,"latency":49.8,"recall":0.9911,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":58.9326,"latency":44.6,"recall":0.9911,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":42.5977,"latency":54.9,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":536.0726,"latency":8.200000000000001,"recall":0.9728,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":467.179,"latency":7.0,"recall":0.9697,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":431.7512,"latency":8.3,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":274.5407,"latency":4.8999999999999995,"recall":0.9807,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":236.5672,"latency":10.3,"recall":0.981,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":309.4833,"latency":4.3,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":178.6585,"latency":13.700000000000001,"recall":0.9843,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":178.3732,"latency":15.0,"recall":0.9844,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":229.3526,"latency":12.5,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":1258.7043,"latency":4.8999999999999995,"recall":0.9799,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":1075.8776,"latency":5.3,"recall":0.98,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1494.8493,"latency":4.7,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":37.432,"latency":75.0,"recall":0.9975,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":37.0696,"latency":73.5,"recall":0.9976,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":81.1915,"latency":53.0,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":321.6048,"latency":13.4,"recall":0.989,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":22.1467,"latency":86.8,"recall":0.9972,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":303.2551,"latency":10.9,"recall":0.9876,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":21.5388,"latency":81.69999999999999,"recall":0.997,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":394.5418,"latency":8.3,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":37.878,"latency":45.0,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":180.2757,"latency":6.0,"recall":0.9942,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":179.0033,"latency":6.4,"recall":0.9943,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":526.8846,"latency":3.6,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":626.5243,"latency":6.2,"recall":0.9954,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":78.4227,"latency":25.3,"recall":0.9982,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":599.4213,"latency":6.6,"recall":0.9955,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":78.5351,"latency":26.3,"recall":0.9982,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":2098.2113,"latency":3.4,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":275.6292,"latency":10.0,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":10.6271,"latency":730.7,"recall":0.8898,"label":1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":10.8507,"latency":733.1999999999999,"recall":0.8897,"label":1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":75.7055,"latency":121.2,"recall":0.9999,"label":1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":0.8836,"latency":2523.0,"recall":0.8528,"label":1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":0.8937,"latency":3720.2000000000003,"recall":0.8525,"label":1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":1.2145,"latency":3622.3999999999996,"recall":0.7487,"label":1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":18.7634,"latency":153.70000000000002,"recall":0.8737,"label":1,"note":"","version":"","test_time":"2023-08","qp$":694226.515930113},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":18.3619,"latency":79.8,"recall":0.8741,"label":1,"note":"","version":"","test_time":"2023-08","qp$":679371.4285714285},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":25.2744,"latency":61.199999999999996,"recall":0.9979,"label":1,"note":"","version":"","test_time":"2023-08","qp$":935126.8242548818},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":261.798,"latency":23.099999999999998,"recall":0.9262,"label":1,"note":"","version":"","test_time":"2023-08","qp$":6455293.150684931},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":166.1851,"latency":23.900000000000002,"recall":0.9264,"label":1,"note":"","version":"","test_time":"2023-08","qp$":4097714.7945205485},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":121.7169,"latency":29.0,"recall":0.9693,"label":1,"note":"","version":"","test_time":"2023-08","qp$":3001238.6301369863},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":46.6189,"latency":43.1,"recall":0.8737,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1724851.3874614593},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":42.4856,"latency":44.0,"recall":0.8741,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1571923.5354573485},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":138.9479,"latency":26.200000000000003,"recall":0.9979,"label":1,"note":"","version":"","test_time":"2023-08","qp$":5140929.496402878},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":20.7437,"latency":75.80000000000001,"recall":0.9291,"label":1,"note":"","version":"","test_time":"2023-08","qp$":382960.6153846154},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":20.2993,"latency":76.5,"recall":0.9293,"label":1,"note":"","version":"","test_time":"2023-08","qp$":374756.30769230763},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":26.4719,"latency":67.0,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":488712.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":8.6675,"latency":180.2,"recall":0.8369,"label":1,"note":"","version":"","test_time":"2023-08","qp$":160015.38461538462},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":7.8121,"latency":167.7,"recall":0.8369,"label":1,"note":"","version":"","test_time":"2023-08","qp$":144223.3846153846},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":16.869,"latency":87.8,"recall":0.9814,"label":1,"note":"","version":"","test_time":"2023-08","qp$":311427.6923076923},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":365.0835,"latency":23.599999999999998,"recall":0.945,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1125257.3630136987},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":325.5271,"latency":25.1,"recall":0.9452,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1003336.9520547946},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":596.7942,"latency":24.2,"recall":0.9693,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1839434.1780821919},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":303.2044,"latency":27.400000000000002,"recall":0.9246,"label":1,"note":"","version":"","test_time":"2023-08","qp$":934534.1095890411},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":136.0345,"latency":31.9,"recall":0.9244,"label":1,"note":"","version":"","test_time":"2023-08","qp$":419284.4178082192},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":66.7221,"latency":42.1,"recall":0.963,"label":1,"note":"","version":"","test_time":"2023-08","qp$":205650.30821917808},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":131.2549,"latency":30.200000000000003,"recall":0.9867,"label":1,"note":"","version":"","test_time":"2023-08","qp$":606569.4993581515},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":127.9337,"latency":30.099999999999998,"recall":0.9869,"label":1,"note":"","version":"","test_time":"2023-08","qp$":591221.2066752247},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":595.8462,"latency":23.400000000000002,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":2753589.627727856},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":228.4,"latency":22.2,"recall":0.9348,"label":1,"note":"","version":"","test_time":"2023-08","qp$":5631780.821917809},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":181.5,"latency":26.1,"recall":0.9345,"label":1,"note":"","version":"","test_time":"2023-08","qp$":4475342.465753425},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":205.7,"latency":24.2,"recall":0.9586,"label":1,"note":"","version":"","test_time":"2023-08","qp$":5072054.794520548},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":67.63,"latency":36.0,"recall":0.8064,"label":1,"note":"","version":"","test_time":"2023-08","qp$":2502240.4933196297},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":63.35,"latency":38.4,"recall":0.8065,"label":1,"note":"","version":"","test_time":"2023-08","qp$":2343884.892086331},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":176.7,"latency":27.599999999999998,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":6537718.396711202},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":15.33,"latency":84.9,"recall":0.8064,"label":1,"note":"","version":"","test_time":"2023-08","qp$":567194.2446043165},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":15.13,"latency":86.7,"recall":0.8065,"label":1,"note":"","version":"","test_time":"2023-08","qp$":559794.4501541625},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":17.41,"latency":74.30000000000001,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":644152.1068859198},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":16.34,"latency":88.7,"recall":0.879,"label":1,"note":"","version":"","test_time":"2023-08","qp$":301661.53846153844},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":10.45,"latency":126.8,"recall":0.8208,"label":1,"note":"","version":"","test_time":"2023-08","qp$":192923.0769230769},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":16.18,"latency":87.5,"recall":0.8793,"label":1,"note":"","version":"","test_time":"2023-08","qp$":298707.69230769225},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":9.8,"latency":130.89999999999998,"recall":0.8212,"label":1,"note":"","version":"","test_time":"2023-08","qp$":180923.0769230769},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":36.11,"latency":55.1,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":666646.1538461539},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":14.84,"latency":92.7,"recall":0.96,"label":1,"note":"","version":"","test_time":"2023-08","qp$":273969.23076923075},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":322.7,"latency":26.4,"recall":0.9478,"label":1,"note":"","version":"","test_time":"2023-08","qp$":994623.287671233},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":265.5,"latency":26.9,"recall":0.9332,"label":1,"note":"","version":"","test_time":"2023-08","qp$":818321.9178082192},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":303.8,"latency":27.3,"recall":0.9478,"label":1,"note":"","version":"","test_time":"2023-08","qp$":936369.8630136987},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":180.2,"latency":28.2,"recall":0.9335,"label":1,"note":"","version":"","test_time":"2023-08","qp$":555410.9589041097},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":730.7,"latency":24.6,"recall":0.9586,"label":1,"note":"","version":"","test_time":"2023-08","qp$":2252157.5342465756},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":104.3,"latency":31.7,"recall":0.9563,"label":1,"note":"","version":"","test_time":"2023-08","qp$":321472.602739726},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":147.7,"latency":35.3,"recall":0.9707,"label":1,"note":"","version":"","test_time":"2023-08","qp$":682567.3940949935},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":782.5,"latency":25.9,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":3616174.5827984596},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":537.4975,"latency":18.9,"recall":0.8903,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1356936.1851332397},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":372.0466,"latency":17.8,"recall":0.8904,"label":1,"note":"","version":"","test_time":"2023-08","qp$":939248.0785413746},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1156.2898,"latency":14.4,"recall":0.9989,"label":1,"note":"","version":"","test_time":"2023-08","qp$":2919104.684431978},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":110.248,"latency":69.0,"recall":0.898,"label":1,"note":"","version":"","test_time":"2023-08","qp$":278325.94670406735},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":87.2601,"latency":27.799999999999997,"recall":0.898,"label":1,"note":"","version":"","test_time":"2023-08","qp$":220291.97755960727},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":125.7846,"latency":23.099999999999998,"recall":0.975,"label":1,"note":"","version":"","test_time":"2023-08","qp$":317548.77980364655},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":240.7209,"latency":17.4,"recall":0.8887,"label":1,"note":"","version":"","test_time":"2023-08","qp$":3038552.734922861},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":189.4399,"latency":17.5,"recall":0.8889,"label":1,"note":"","version":"","test_time":"2023-08","qp$":2391246.98457223},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":313.5116,"latency":16.1,"recall":0.9999,"label":1,"note":"","version":"","test_time":"2023-08","qp$":3957369.424964937},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":78.7196,"latency":49.4,"recall":0.9203,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1707172.048192771},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":68.3111,"latency":35.5,"recall":0.9202,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1481445.5421686745},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":210.2147,"latency":26.8,"recall":0.9996,"label":1,"note":"","version":"","test_time":"2023-08","qp$":4558873.012048192},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":188.6436,"latency":917.5,"recall":0.9175,"label":1,"note":"","version":"","test_time":"2023-08","qp$":2381195.5119214584},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":155.6991,"latency":917.1,"recall":0.9171,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1965346.283309958},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":445.3289,"latency":14.1,"recall":0.9999,"label":1,"note":"","version":"","test_time":"2023-08","qp$":5621262.412342216},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":633.6033,"latency":24.6,"recall":0.919,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1599559.5231416551},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":95.5682,"latency":58.7,"recall":0.9463,"label":1,"note":"","version":"","test_time":"2023-08","qp$":241266.14305750353},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":434.4062,"latency":17.4,"recall":0.9181,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1096677.643758766},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":80.3192,"latency":26.599999999999998,"recall":0.9462,"label":1,"note":"","version":"","test_time":"2023-08","qp$":202769.3688639551},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":1509.3293,"latency":18.5,"recall":0.9995,"label":1,"note":"","version":"","test_time":"2023-08","qp$":3810368.4992987383},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":166.7252,"latency":19.599999999999998,"recall":0.9988,"label":1,"note":"","version":"","test_time":"2023-08","qp$":420905.1332398317},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":67.9121,"latency":179.5,"recall":0.9909,"label":1,"note":"","version":"","test_time":"2023-08","qp$":7499.495705521471},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.7636,"latency":1921.3,"recall":0.9908,"label":1,"note":"","version":"","test_time":"2023-08","qp$":84.32392638036809},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":32.0,"latency":124.5,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":3533.7423312883434},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":63.1365,"latency":145.7,"recall":0.991,"label":1,"note":"","version":"","test_time":"2023-08","qp$":22504.09900990099},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.7512,"latency":1983.8,"recall":0.9908,"label":1,"note":"","version":"","test_time":"2023-08","qp$":267.7544554455445},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":30.1358,"latency":129.8,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":10741.473267326734},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":46.8622,"latency":123.0,"recall":0.9957,"label":1,"note":"","version":"","test_time":"2023-08","qp$":16703.358415841583},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1.8388,"latency":1063.5,"recall":0.9957,"label":1,"note":"","version":"","test_time":"2023-08","qp$":655.4138613861386},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":45.0666,"latency":109.2,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":16063.342574257427},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":43.5017,"latency":228.7,"recall":0.9957,"label":1,"note":"","version":"","test_time":"2023-08","qp$":4803.8687116564415},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1.5668,"latency":1114.4,"recall":0.9957,"label":1,"note":"","version":"","test_time":"2023-08","qp$":173.02085889570552},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":41.5443,"latency":159.0,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":4587.714110429448},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":330.0144,"latency":9.0,"recall":0.9507,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":7472024.150943397},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":271.6585,"latency":10.1,"recall":0.9678,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":6150758.490566038},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":216.5226,"latency":12.9,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":4902398.490566038},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":123.9553,"latency":23.0,"recall":0.971,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":1403267.5471698113},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":59.1479,"latency":44.5,"recall":0.9906,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":669598.8679245282},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":40.999,"latency":55.300000000000004,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":464139.62264150946},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":579.9416,"latency":9.4,"recall":0.9213,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":6565376.603773585},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":425.2529,"latency":11.299999999999999,"recall":0.9686,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":4814183.773584905},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":397.0539,"latency":13.799999999999999,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":4494949.811320755},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":516.27,"latency":7.0,"recall":0.9463,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":11689132.075471697},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":354.8416,"latency":10.0,"recall":0.9802,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":8034149.433962264},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":427.5229,"latency":8.7,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":9679763.773584906},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":2884.689,"latency":5.3,"recall":0.8801,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":8164214.150943396},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":1689.5799,"latency":6.6,"recall":0.9493,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":4781829.905660377},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1517.6792,"latency":10.0,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":4295318.4905660385},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":822.5318,"latency":5.6,"recall":0.9294,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":2327920.1886792453},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":378.9146,"latency":10.3,"recall":0.9758,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":1072399.8113207547},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":218.6854,"latency":16.2,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":618920.9433962263},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":297.5,"latency":7.2,"recall":0.974,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":6735849.0566037735},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":228.3,"latency":10.6,"recall":0.994,"label":1,"note":"","version":"v","test_time":"2023-08","qp$":5169056.603773585},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":584.0,"latency":4.6,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":13222641.509433962},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":1871.0,"latency":7.0,"recall":0.9602,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":5295283.018867925},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":556.7,"latency":6.7,"recall":0.9723,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":1575566.0377358492},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1583.0,"latency":6.8,"recall":0.9836,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":4480188.679245283},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":294.3,"latency":10.9,"recall":0.9939,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":832924.5283018869},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":2345.0,"latency":8.9,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":6636792.452830189},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":295.6,"latency":12.3,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":836603.7735849057},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":143.0,"latency":33.5,"recall":0.9818,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":3237735.8490566034},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":106.0,"latency":20.7,"recall":0.9887,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":2400000.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":189.0,"latency":11.6,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":4279245.2830188675},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":379.9721,"latency":12.4,"recall":0.982,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":4301570.9433962265},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":71.74,"latency":50.8,"recall":0.9883,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":812150.9433962264},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":287.0,"latency":14.9,"recall":0.9865,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":3249056.603773585},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":34.6654,"latency":64.69999999999999,"recall":0.9961,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":392438.4905660377},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":412.0,"latency":10.3,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":4664150.9433962265},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":42.169,"latency":46.800000000000004,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":477384.9056603773},{"db":"Milvus","db_label":"test","db_name":"Milvus-test-v2","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":3419.8808,"latency":3.0,"recall":0.965,"label":1,"note":"note","version":"v2","test_time":"2024-10","qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":5115.5303,"latency":8.7,"recall":0.9469,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":14477915.943396227},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":3685.0767,"latency":9.1,"recall":0.9736,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":10429462.358490566},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":4742.1617,"latency":9.1,"recall":0.9936,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":13421212.358490566},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":6054.4428,"latency":8.1,"recall":0.9155,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":17135215.471698113},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":4104.2598,"latency":10.6,"recall":0.9506,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":11615829.62264151},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":4252.1267,"latency":9.2,"recall":0.9964,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":12034320.849056603},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":1685.3091,"latency":13.299999999999999,"recall":0.9718,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":4769742.735849056},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":769.8991,"latency":10.7,"recall":0.9884,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":2178959.716981132},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":945.4061,"latency":10.9,"recall":0.9941,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":2675677.641509434},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":2214.9028,"latency":8.4,"recall":0.9249,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":6268592.830188679},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":827.975,"latency":12.0,"recall":0.9692,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":2343325.4716981133},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":776.9454,"latency":11.4,"recall":0.9966,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":2198902.075471698},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":269.5464,"latency":9.799999999999999,"recall":0.9776,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":6102937.358490566},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":240.0363,"latency":10.6,"recall":0.9822,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":5434784.150943397},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":218.0627,"latency":11.4,"recall":0.9936,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":4937268.679245283},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":392.8825,"latency":6.8999999999999995,"recall":0.9581,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":8895452.830188679},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":343.8204,"latency":8.4,"recall":0.968,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":7784612.8301886795},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":216.6773,"latency":10.1,"recall":0.9968,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":4905901.132075472},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":503.2284,"latency":9.0,"recall":0.9677,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":5696925.2830188675},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":413.3232,"latency":9.6,"recall":0.981,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":4679130.566037735},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":425.5492,"latency":10.200000000000001,"recall":0.9936,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":4817538.113207547},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":789.1229,"latency":5.6,"recall":0.9396,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":8933466.792452829},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":571.4257,"latency":7.7,"recall":0.9668,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":6468970.188679245},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":411.7653,"latency":9.1,"recall":0.9968,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":4661493.96226415},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":98.0448,"latency":16.1,"recall":0.9803,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":1109941.1320754716},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":58.3152,"latency":28.0,"recall":0.9891,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":660172.0754716981},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":46.8304,"latency":28.6,"recall":0.9941,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":530155.4716981131},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":170.5693,"latency":8.9,"recall":0.9605,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":1930973.2075471696},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":94.7766,"latency":19.599999999999998,"recall":0.9843,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":1072942.641509434},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":44.8695,"latency":30.900000000000002,"recall":0.9966,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":507956.60377358494},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":722.0315,"latency":7.7,"recall":0.976,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":16347883.018867927},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":467.5795,"latency":8.8,"recall":0.9898,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":10586705.660377359},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":975.2503,"latency":8.200000000000001,"recall":0.9936,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":22081138.86792453},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":873.3712,"latency":6.7,"recall":0.9477,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":19774442.264150944},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":544.6203,"latency":8.4,"recall":0.977,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":12331025.660377359},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":930.9164,"latency":9.6,"recall":0.9968,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":21077352.452830188},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0}] \ No newline at end of file From a64d32693c771319d13a541a7bfb5e78853772a5 Mon Sep 17 00:00:00 2001 From: "min.tian" Date: Wed, 23 Oct 2024 18:02:23 +0800 Subject: [PATCH 088/327] fix bug: date to datetime Signed-off-by: min.tian --- vectordb_bench/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vectordb_bench/models.py b/vectordb_bench/models.py index e4c361855..7968e3e26 100644 --- a/vectordb_bench/models.py +++ b/vectordb_bench/models.py @@ -1,6 +1,6 @@ import logging import pathlib -from datetime import date +from datetime import date, datetime from enum import Enum, StrEnum, auto from typing import List, Self @@ -167,7 +167,7 @@ class TestResult(BaseModel): def flush(self): db2case = self.get_db_results() - timestamp = date.today().timestamp() + timestamp = datetime.combine(date.today(), datetime.min.time()).timestamp() result_root = config.RESULTS_LOCAL_DIR for db, result in db2case.items(): self.write_db_file( From dc73c2a52546756de92b5fed2b98e51babc21e9b Mon Sep 17 00:00:00 2001 From: "min.tian" Date: Thu, 24 Oct 2024 17:14:18 +0800 Subject: [PATCH 089/327] update leaderboard data Signed-off-by: min.tian --- vectordb_bench/results/leaderboard.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vectordb_bench/results/leaderboard.json b/vectordb_bench/results/leaderboard.json index 9f64ade18..768b4123c 100644 --- a/vectordb_bench/results/leaderboard.json +++ b/vectordb_bench/results/leaderboard.json @@ -1 +1 @@ -[{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":15.2269,"latency":861.8,"recall":0.9888,"label":1,"note":"","version":"","test_time":"2023-08","qp$":114368.53745044857},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":15.1749,"latency":774.3,"recall":0.989,"label":1,"note":"","version":"","test_time":"2023-08","qp$":113977.96786981013},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":27.6181,"latency":305.5,"recall":0.9999,"label":1,"note":"","version":"","test_time":"2023-08","qp$":207438.26413519715},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":11.2945,"latency":3611.2000000000003,"recall":0.996,"label":1,"note":"","version":"","test_time":"2023-08","qp$":84832.4640100146},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":17.3271,"latency":3774.7999999999997,"recall":0.9961,"label":1,"note":"","version":"","test_time":"2023-08","qp$":130143.04193615691},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":26.26,"latency":556.1,"recall":0.9999,"label":1,"note":"","version":"","test_time":"2023-08","qp$":197237.6382224077},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":100.6667,"latency":21.1,"recall":0.9909,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":101.1399,"latency":19.7,"recall":0.9907,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":52.2606,"latency":18.3,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":61.0661,"latency":49.8,"recall":0.9911,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":58.9326,"latency":44.6,"recall":0.9911,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":42.5977,"latency":54.9,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":536.0726,"latency":8.200000000000001,"recall":0.9728,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":467.179,"latency":7.0,"recall":0.9697,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":431.7512,"latency":8.3,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":274.5407,"latency":4.8999999999999995,"recall":0.9807,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":236.5672,"latency":10.3,"recall":0.981,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":309.4833,"latency":4.3,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":178.6585,"latency":13.700000000000001,"recall":0.9843,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":178.3732,"latency":15.0,"recall":0.9844,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":229.3526,"latency":12.5,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":1258.7043,"latency":4.8999999999999995,"recall":0.9799,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":1075.8776,"latency":5.3,"recall":0.98,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1494.8493,"latency":4.7,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":37.432,"latency":75.0,"recall":0.9975,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":37.0696,"latency":73.5,"recall":0.9976,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":81.1915,"latency":53.0,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":321.6048,"latency":13.4,"recall":0.989,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":22.1467,"latency":86.8,"recall":0.9972,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":303.2551,"latency":10.9,"recall":0.9876,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":21.5388,"latency":81.69999999999999,"recall":0.997,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":394.5418,"latency":8.3,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":37.878,"latency":45.0,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":180.2757,"latency":6.0,"recall":0.9942,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":179.0033,"latency":6.4,"recall":0.9943,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":526.8846,"latency":3.6,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":626.5243,"latency":6.2,"recall":0.9954,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":78.4227,"latency":25.3,"recall":0.9982,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":599.4213,"latency":6.6,"recall":0.9955,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":78.5351,"latency":26.3,"recall":0.9982,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":2098.2113,"latency":3.4,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":275.6292,"latency":10.0,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":10.6271,"latency":730.7,"recall":0.8898,"label":1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":10.8507,"latency":733.1999999999999,"recall":0.8897,"label":1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":75.7055,"latency":121.2,"recall":0.9999,"label":1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":0.8836,"latency":2523.0,"recall":0.8528,"label":1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":0.8937,"latency":3720.2000000000003,"recall":0.8525,"label":1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":1.2145,"latency":3622.3999999999996,"recall":0.7487,"label":1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":18.7634,"latency":153.70000000000002,"recall":0.8737,"label":1,"note":"","version":"","test_time":"2023-08","qp$":694226.515930113},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":18.3619,"latency":79.8,"recall":0.8741,"label":1,"note":"","version":"","test_time":"2023-08","qp$":679371.4285714285},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":25.2744,"latency":61.199999999999996,"recall":0.9979,"label":1,"note":"","version":"","test_time":"2023-08","qp$":935126.8242548818},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":261.798,"latency":23.099999999999998,"recall":0.9262,"label":1,"note":"","version":"","test_time":"2023-08","qp$":6455293.150684931},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":166.1851,"latency":23.900000000000002,"recall":0.9264,"label":1,"note":"","version":"","test_time":"2023-08","qp$":4097714.7945205485},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":121.7169,"latency":29.0,"recall":0.9693,"label":1,"note":"","version":"","test_time":"2023-08","qp$":3001238.6301369863},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":46.6189,"latency":43.1,"recall":0.8737,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1724851.3874614593},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":42.4856,"latency":44.0,"recall":0.8741,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1571923.5354573485},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":138.9479,"latency":26.200000000000003,"recall":0.9979,"label":1,"note":"","version":"","test_time":"2023-08","qp$":5140929.496402878},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":20.7437,"latency":75.80000000000001,"recall":0.9291,"label":1,"note":"","version":"","test_time":"2023-08","qp$":382960.6153846154},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":20.2993,"latency":76.5,"recall":0.9293,"label":1,"note":"","version":"","test_time":"2023-08","qp$":374756.30769230763},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":26.4719,"latency":67.0,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":488712.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":8.6675,"latency":180.2,"recall":0.8369,"label":1,"note":"","version":"","test_time":"2023-08","qp$":160015.38461538462},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":7.8121,"latency":167.7,"recall":0.8369,"label":1,"note":"","version":"","test_time":"2023-08","qp$":144223.3846153846},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":16.869,"latency":87.8,"recall":0.9814,"label":1,"note":"","version":"","test_time":"2023-08","qp$":311427.6923076923},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":365.0835,"latency":23.599999999999998,"recall":0.945,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1125257.3630136987},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":325.5271,"latency":25.1,"recall":0.9452,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1003336.9520547946},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":596.7942,"latency":24.2,"recall":0.9693,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1839434.1780821919},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":303.2044,"latency":27.400000000000002,"recall":0.9246,"label":1,"note":"","version":"","test_time":"2023-08","qp$":934534.1095890411},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":136.0345,"latency":31.9,"recall":0.9244,"label":1,"note":"","version":"","test_time":"2023-08","qp$":419284.4178082192},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":66.7221,"latency":42.1,"recall":0.963,"label":1,"note":"","version":"","test_time":"2023-08","qp$":205650.30821917808},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":131.2549,"latency":30.200000000000003,"recall":0.9867,"label":1,"note":"","version":"","test_time":"2023-08","qp$":606569.4993581515},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":127.9337,"latency":30.099999999999998,"recall":0.9869,"label":1,"note":"","version":"","test_time":"2023-08","qp$":591221.2066752247},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":595.8462,"latency":23.400000000000002,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":2753589.627727856},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":228.4,"latency":22.2,"recall":0.9348,"label":1,"note":"","version":"","test_time":"2023-08","qp$":5631780.821917809},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":181.5,"latency":26.1,"recall":0.9345,"label":1,"note":"","version":"","test_time":"2023-08","qp$":4475342.465753425},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":205.7,"latency":24.2,"recall":0.9586,"label":1,"note":"","version":"","test_time":"2023-08","qp$":5072054.794520548},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":67.63,"latency":36.0,"recall":0.8064,"label":1,"note":"","version":"","test_time":"2023-08","qp$":2502240.4933196297},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":63.35,"latency":38.4,"recall":0.8065,"label":1,"note":"","version":"","test_time":"2023-08","qp$":2343884.892086331},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":176.7,"latency":27.599999999999998,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":6537718.396711202},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":15.33,"latency":84.9,"recall":0.8064,"label":1,"note":"","version":"","test_time":"2023-08","qp$":567194.2446043165},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":15.13,"latency":86.7,"recall":0.8065,"label":1,"note":"","version":"","test_time":"2023-08","qp$":559794.4501541625},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":17.41,"latency":74.30000000000001,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":644152.1068859198},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":16.34,"latency":88.7,"recall":0.879,"label":1,"note":"","version":"","test_time":"2023-08","qp$":301661.53846153844},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":10.45,"latency":126.8,"recall":0.8208,"label":1,"note":"","version":"","test_time":"2023-08","qp$":192923.0769230769},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":16.18,"latency":87.5,"recall":0.8793,"label":1,"note":"","version":"","test_time":"2023-08","qp$":298707.69230769225},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":9.8,"latency":130.89999999999998,"recall":0.8212,"label":1,"note":"","version":"","test_time":"2023-08","qp$":180923.0769230769},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":36.11,"latency":55.1,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":666646.1538461539},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":14.84,"latency":92.7,"recall":0.96,"label":1,"note":"","version":"","test_time":"2023-08","qp$":273969.23076923075},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":322.7,"latency":26.4,"recall":0.9478,"label":1,"note":"","version":"","test_time":"2023-08","qp$":994623.287671233},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":265.5,"latency":26.9,"recall":0.9332,"label":1,"note":"","version":"","test_time":"2023-08","qp$":818321.9178082192},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":303.8,"latency":27.3,"recall":0.9478,"label":1,"note":"","version":"","test_time":"2023-08","qp$":936369.8630136987},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":180.2,"latency":28.2,"recall":0.9335,"label":1,"note":"","version":"","test_time":"2023-08","qp$":555410.9589041097},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":730.7,"latency":24.6,"recall":0.9586,"label":1,"note":"","version":"","test_time":"2023-08","qp$":2252157.5342465756},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":104.3,"latency":31.7,"recall":0.9563,"label":1,"note":"","version":"","test_time":"2023-08","qp$":321472.602739726},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":147.7,"latency":35.3,"recall":0.9707,"label":1,"note":"","version":"","test_time":"2023-08","qp$":682567.3940949935},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":782.5,"latency":25.9,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":3616174.5827984596},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":537.4975,"latency":18.9,"recall":0.8903,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1356936.1851332397},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":372.0466,"latency":17.8,"recall":0.8904,"label":1,"note":"","version":"","test_time":"2023-08","qp$":939248.0785413746},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1156.2898,"latency":14.4,"recall":0.9989,"label":1,"note":"","version":"","test_time":"2023-08","qp$":2919104.684431978},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":110.248,"latency":69.0,"recall":0.898,"label":1,"note":"","version":"","test_time":"2023-08","qp$":278325.94670406735},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":87.2601,"latency":27.799999999999997,"recall":0.898,"label":1,"note":"","version":"","test_time":"2023-08","qp$":220291.97755960727},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":125.7846,"latency":23.099999999999998,"recall":0.975,"label":1,"note":"","version":"","test_time":"2023-08","qp$":317548.77980364655},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":240.7209,"latency":17.4,"recall":0.8887,"label":1,"note":"","version":"","test_time":"2023-08","qp$":3038552.734922861},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":189.4399,"latency":17.5,"recall":0.8889,"label":1,"note":"","version":"","test_time":"2023-08","qp$":2391246.98457223},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":313.5116,"latency":16.1,"recall":0.9999,"label":1,"note":"","version":"","test_time":"2023-08","qp$":3957369.424964937},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":78.7196,"latency":49.4,"recall":0.9203,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1707172.048192771},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":68.3111,"latency":35.5,"recall":0.9202,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1481445.5421686745},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":210.2147,"latency":26.8,"recall":0.9996,"label":1,"note":"","version":"","test_time":"2023-08","qp$":4558873.012048192},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":188.6436,"latency":917.5,"recall":0.9175,"label":1,"note":"","version":"","test_time":"2023-08","qp$":2381195.5119214584},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":155.6991,"latency":917.1,"recall":0.9171,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1965346.283309958},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":445.3289,"latency":14.1,"recall":0.9999,"label":1,"note":"","version":"","test_time":"2023-08","qp$":5621262.412342216},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":633.6033,"latency":24.6,"recall":0.919,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1599559.5231416551},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":95.5682,"latency":58.7,"recall":0.9463,"label":1,"note":"","version":"","test_time":"2023-08","qp$":241266.14305750353},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":434.4062,"latency":17.4,"recall":0.9181,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1096677.643758766},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":80.3192,"latency":26.599999999999998,"recall":0.9462,"label":1,"note":"","version":"","test_time":"2023-08","qp$":202769.3688639551},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":1509.3293,"latency":18.5,"recall":0.9995,"label":1,"note":"","version":"","test_time":"2023-08","qp$":3810368.4992987383},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":166.7252,"latency":19.599999999999998,"recall":0.9988,"label":1,"note":"","version":"","test_time":"2023-08","qp$":420905.1332398317},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":67.9121,"latency":179.5,"recall":0.9909,"label":1,"note":"","version":"","test_time":"2023-08","qp$":7499.495705521471},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.7636,"latency":1921.3,"recall":0.9908,"label":1,"note":"","version":"","test_time":"2023-08","qp$":84.32392638036809},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":32.0,"latency":124.5,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":3533.7423312883434},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":63.1365,"latency":145.7,"recall":0.991,"label":1,"note":"","version":"","test_time":"2023-08","qp$":22504.09900990099},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.7512,"latency":1983.8,"recall":0.9908,"label":1,"note":"","version":"","test_time":"2023-08","qp$":267.7544554455445},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":30.1358,"latency":129.8,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":10741.473267326734},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":46.8622,"latency":123.0,"recall":0.9957,"label":1,"note":"","version":"","test_time":"2023-08","qp$":16703.358415841583},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1.8388,"latency":1063.5,"recall":0.9957,"label":1,"note":"","version":"","test_time":"2023-08","qp$":655.4138613861386},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":45.0666,"latency":109.2,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":16063.342574257427},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":43.5017,"latency":228.7,"recall":0.9957,"label":1,"note":"","version":"","test_time":"2023-08","qp$":4803.8687116564415},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1.5668,"latency":1114.4,"recall":0.9957,"label":1,"note":"","version":"","test_time":"2023-08","qp$":173.02085889570552},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":41.5443,"latency":159.0,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":4587.714110429448},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":330.0144,"latency":9.0,"recall":0.9507,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":7472024.150943397},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":271.6585,"latency":10.1,"recall":0.9678,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":6150758.490566038},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":216.5226,"latency":12.9,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":4902398.490566038},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":123.9553,"latency":23.0,"recall":0.971,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":1403267.5471698113},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":59.1479,"latency":44.5,"recall":0.9906,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":669598.8679245282},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":40.999,"latency":55.300000000000004,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":464139.62264150946},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":579.9416,"latency":9.4,"recall":0.9213,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":6565376.603773585},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":425.2529,"latency":11.299999999999999,"recall":0.9686,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":4814183.773584905},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":397.0539,"latency":13.799999999999999,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":4494949.811320755},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":516.27,"latency":7.0,"recall":0.9463,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":11689132.075471697},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":354.8416,"latency":10.0,"recall":0.9802,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":8034149.433962264},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":427.5229,"latency":8.7,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":9679763.773584906},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":2884.689,"latency":5.3,"recall":0.8801,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":8164214.150943396},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":1689.5799,"latency":6.6,"recall":0.9493,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":4781829.905660377},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1517.6792,"latency":10.0,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":4295318.4905660385},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":822.5318,"latency":5.6,"recall":0.9294,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":2327920.1886792453},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":378.9146,"latency":10.3,"recall":0.9758,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":1072399.8113207547},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":218.6854,"latency":16.2,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":618920.9433962263},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":297.5,"latency":7.2,"recall":0.974,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":6735849.0566037735},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":228.3,"latency":10.6,"recall":0.994,"label":1,"note":"","version":"v","test_time":"2023-08","qp$":5169056.603773585},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":584.0,"latency":4.6,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":13222641.509433962},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":1871.0,"latency":7.0,"recall":0.9602,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":5295283.018867925},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":556.7,"latency":6.7,"recall":0.9723,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":1575566.0377358492},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1583.0,"latency":6.8,"recall":0.9836,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":4480188.679245283},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":294.3,"latency":10.9,"recall":0.9939,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":832924.5283018869},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":2345.0,"latency":8.9,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":6636792.452830189},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":295.6,"latency":12.3,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":836603.7735849057},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":143.0,"latency":33.5,"recall":0.9818,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":3237735.8490566034},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":106.0,"latency":20.7,"recall":0.9887,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":2400000.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":189.0,"latency":11.6,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":4279245.2830188675},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":379.9721,"latency":12.4,"recall":0.982,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":4301570.9433962265},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":71.74,"latency":50.8,"recall":0.9883,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":812150.9433962264},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":287.0,"latency":14.9,"recall":0.9865,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":3249056.603773585},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":34.6654,"latency":64.69999999999999,"recall":0.9961,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":392438.4905660377},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":412.0,"latency":10.3,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":4664150.9433962265},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":42.169,"latency":46.800000000000004,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":477384.9056603773},{"db":"Milvus","db_label":"test","db_name":"Milvus-test-v2","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":3419.8808,"latency":3.0,"recall":0.965,"label":1,"note":"note","version":"v2","test_time":"2024-10","qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":5115.5303,"latency":8.7,"recall":0.9469,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":14477915.943396227},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":3685.0767,"latency":9.1,"recall":0.9736,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":10429462.358490566},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":4742.1617,"latency":9.1,"recall":0.9936,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":13421212.358490566},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":6054.4428,"latency":8.1,"recall":0.9155,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":17135215.471698113},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":4104.2598,"latency":10.6,"recall":0.9506,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":11615829.62264151},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":4252.1267,"latency":9.2,"recall":0.9964,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":12034320.849056603},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":1685.3091,"latency":13.299999999999999,"recall":0.9718,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":4769742.735849056},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":769.8991,"latency":10.7,"recall":0.9884,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":2178959.716981132},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":945.4061,"latency":10.9,"recall":0.9941,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":2675677.641509434},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":2214.9028,"latency":8.4,"recall":0.9249,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":6268592.830188679},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":827.975,"latency":12.0,"recall":0.9692,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":2343325.4716981133},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":776.9454,"latency":11.4,"recall":0.9966,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":2198902.075471698},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":269.5464,"latency":9.799999999999999,"recall":0.9776,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":6102937.358490566},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":240.0363,"latency":10.6,"recall":0.9822,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":5434784.150943397},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":218.0627,"latency":11.4,"recall":0.9936,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":4937268.679245283},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":392.8825,"latency":6.8999999999999995,"recall":0.9581,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":8895452.830188679},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":343.8204,"latency":8.4,"recall":0.968,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":7784612.8301886795},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":216.6773,"latency":10.1,"recall":0.9968,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":4905901.132075472},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":503.2284,"latency":9.0,"recall":0.9677,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":5696925.2830188675},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":413.3232,"latency":9.6,"recall":0.981,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":4679130.566037735},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":425.5492,"latency":10.200000000000001,"recall":0.9936,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":4817538.113207547},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":789.1229,"latency":5.6,"recall":0.9396,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":8933466.792452829},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":571.4257,"latency":7.7,"recall":0.9668,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":6468970.188679245},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":411.7653,"latency":9.1,"recall":0.9968,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":4661493.96226415},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":98.0448,"latency":16.1,"recall":0.9803,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":1109941.1320754716},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":58.3152,"latency":28.0,"recall":0.9891,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":660172.0754716981},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":46.8304,"latency":28.6,"recall":0.9941,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":530155.4716981131},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":170.5693,"latency":8.9,"recall":0.9605,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":1930973.2075471696},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":94.7766,"latency":19.599999999999998,"recall":0.9843,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":1072942.641509434},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":44.8695,"latency":30.900000000000002,"recall":0.9966,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":507956.60377358494},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":722.0315,"latency":7.7,"recall":0.976,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":16347883.018867927},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":467.5795,"latency":8.8,"recall":0.9898,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":10586705.660377359},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":975.2503,"latency":8.200000000000001,"recall":0.9936,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":22081138.86792453},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":873.3712,"latency":6.7,"recall":0.9477,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":19774442.264150944},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":544.6203,"latency":8.4,"recall":0.977,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":12331025.660377359},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":930.9164,"latency":9.6,"recall":0.9968,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":21077352.452830188},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0}] \ No newline at end of file +[{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":15.2269,"latency":861.8,"recall":0.9888,"label":1,"note":"","version":"","test_time":"2023-08","qp$":114368.53745044857},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":15.1749,"latency":774.3,"recall":0.989,"label":1,"note":"","version":"","test_time":"2023-08","qp$":113977.96786981013},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":27.6181,"latency":305.5,"recall":0.9999,"label":1,"note":"","version":"","test_time":"2023-08","qp$":207438.26413519715},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":11.2945,"latency":3611.2000000000003,"recall":0.996,"label":1,"note":"","version":"","test_time":"2023-08","qp$":84832.4640100146},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":17.3271,"latency":3774.7999999999997,"recall":0.9961,"label":1,"note":"","version":"","test_time":"2023-08","qp$":130143.04193615691},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":26.26,"latency":556.1,"recall":0.9999,"label":1,"note":"","version":"","test_time":"2023-08","qp$":197237.6382224077},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":100.6667,"latency":21.1,"recall":0.9909,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":101.1399,"latency":19.7,"recall":0.9907,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":52.2606,"latency":18.3,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":61.0661,"latency":49.8,"recall":0.9911,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":58.9326,"latency":44.6,"recall":0.9911,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":42.5977,"latency":54.9,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":536.0726,"latency":8.200000000000001,"recall":0.9728,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":467.179,"latency":7.0,"recall":0.9697,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":431.7512,"latency":8.3,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":274.5407,"latency":4.8999999999999995,"recall":0.9807,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":236.5672,"latency":10.3,"recall":0.981,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":309.4833,"latency":4.3,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":178.6585,"latency":13.700000000000001,"recall":0.9843,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":178.3732,"latency":15.0,"recall":0.9844,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":229.3526,"latency":12.5,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":1258.7043,"latency":4.8999999999999995,"recall":0.9799,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":1075.8776,"latency":5.3,"recall":0.98,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1494.8493,"latency":4.7,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":37.432,"latency":75.0,"recall":0.9975,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":37.0696,"latency":73.5,"recall":0.9976,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":81.1915,"latency":53.0,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":321.6048,"latency":13.4,"recall":0.989,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":22.1467,"latency":86.8,"recall":0.9972,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":303.2551,"latency":10.9,"recall":0.9876,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":21.5388,"latency":81.69999999999999,"recall":0.997,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":394.5418,"latency":8.3,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":37.878,"latency":45.0,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":180.2757,"latency":6.0,"recall":0.9942,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":179.0033,"latency":6.4,"recall":0.9943,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":526.8846,"latency":3.6,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":626.5243,"latency":6.2,"recall":0.9954,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":78.4227,"latency":25.3,"recall":0.9982,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":599.4213,"latency":6.6,"recall":0.9955,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":78.5351,"latency":26.3,"recall":0.9982,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":2098.2113,"latency":3.4,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":275.6292,"latency":10.0,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":10.6271,"latency":730.7,"recall":0.8898,"label":1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":10.8507,"latency":733.1999999999999,"recall":0.8897,"label":1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":75.7055,"latency":121.2,"recall":0.9999,"label":1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":0.8836,"latency":2523.0,"recall":0.8528,"label":1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":0.8937,"latency":3720.2000000000003,"recall":0.8525,"label":1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":1.2145,"latency":3622.3999999999996,"recall":0.7487,"label":1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":18.7634,"latency":153.70000000000002,"recall":0.8737,"label":1,"note":"","version":"","test_time":"2023-08","qp$":694226.515930113},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":18.3619,"latency":79.8,"recall":0.8741,"label":1,"note":"","version":"","test_time":"2023-08","qp$":679371.4285714285},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":25.2744,"latency":61.199999999999996,"recall":0.9979,"label":1,"note":"","version":"","test_time":"2023-08","qp$":935126.8242548818},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":261.798,"latency":23.099999999999998,"recall":0.9262,"label":1,"note":"","version":"","test_time":"2023-08","qp$":6455293.150684931},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":166.1851,"latency":23.900000000000002,"recall":0.9264,"label":1,"note":"","version":"","test_time":"2023-08","qp$":4097714.7945205485},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":121.7169,"latency":29.0,"recall":0.9693,"label":1,"note":"","version":"","test_time":"2023-08","qp$":3001238.6301369863},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":46.6189,"latency":43.1,"recall":0.8737,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1724851.3874614593},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":42.4856,"latency":44.0,"recall":0.8741,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1571923.5354573485},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":138.9479,"latency":26.200000000000003,"recall":0.9979,"label":1,"note":"","version":"","test_time":"2023-08","qp$":5140929.496402878},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":20.7437,"latency":75.80000000000001,"recall":0.9291,"label":1,"note":"","version":"","test_time":"2023-08","qp$":382960.6153846154},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":20.2993,"latency":76.5,"recall":0.9293,"label":1,"note":"","version":"","test_time":"2023-08","qp$":374756.30769230763},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":26.4719,"latency":67.0,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":488712.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":8.6675,"latency":180.2,"recall":0.8369,"label":1,"note":"","version":"","test_time":"2023-08","qp$":160015.38461538462},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":7.8121,"latency":167.7,"recall":0.8369,"label":1,"note":"","version":"","test_time":"2023-08","qp$":144223.3846153846},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":16.869,"latency":87.8,"recall":0.9814,"label":1,"note":"","version":"","test_time":"2023-08","qp$":311427.6923076923},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":365.0835,"latency":23.599999999999998,"recall":0.945,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1125257.3630136987},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":325.5271,"latency":25.1,"recall":0.9452,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1003336.9520547946},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":596.7942,"latency":24.2,"recall":0.9693,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1839434.1780821919},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":303.2044,"latency":27.400000000000002,"recall":0.9246,"label":1,"note":"","version":"","test_time":"2023-08","qp$":934534.1095890411},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":136.0345,"latency":31.9,"recall":0.9244,"label":1,"note":"","version":"","test_time":"2023-08","qp$":419284.4178082192},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":66.7221,"latency":42.1,"recall":0.963,"label":1,"note":"","version":"","test_time":"2023-08","qp$":205650.30821917808},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":131.2549,"latency":30.200000000000003,"recall":0.9867,"label":1,"note":"","version":"","test_time":"2023-08","qp$":606569.4993581515},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":127.9337,"latency":30.099999999999998,"recall":0.9869,"label":1,"note":"","version":"","test_time":"2023-08","qp$":591221.2066752247},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":595.8462,"latency":23.400000000000002,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":2753589.627727856},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":228.4,"latency":22.2,"recall":0.9348,"label":1,"note":"","version":"","test_time":"2023-08","qp$":5631780.821917809},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":181.5,"latency":26.1,"recall":0.9345,"label":1,"note":"","version":"","test_time":"2023-08","qp$":4475342.465753425},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":205.7,"latency":24.2,"recall":0.9586,"label":1,"note":"","version":"","test_time":"2023-08","qp$":5072054.794520548},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":67.63,"latency":36.0,"recall":0.8064,"label":1,"note":"","version":"","test_time":"2023-08","qp$":2502240.4933196297},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":63.35,"latency":38.4,"recall":0.8065,"label":1,"note":"","version":"","test_time":"2023-08","qp$":2343884.892086331},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":176.7,"latency":27.599999999999998,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":6537718.396711202},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":15.33,"latency":84.9,"recall":0.8064,"label":1,"note":"","version":"","test_time":"2023-08","qp$":567194.2446043165},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":15.13,"latency":86.7,"recall":0.8065,"label":1,"note":"","version":"","test_time":"2023-08","qp$":559794.4501541625},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":17.41,"latency":74.30000000000001,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":644152.1068859198},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":16.34,"latency":88.7,"recall":0.879,"label":1,"note":"","version":"","test_time":"2023-08","qp$":301661.53846153844},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":10.45,"latency":126.8,"recall":0.8208,"label":1,"note":"","version":"","test_time":"2023-08","qp$":192923.0769230769},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":16.18,"latency":87.5,"recall":0.8793,"label":1,"note":"","version":"","test_time":"2023-08","qp$":298707.69230769225},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":9.8,"latency":130.89999999999998,"recall":0.8212,"label":1,"note":"","version":"","test_time":"2023-08","qp$":180923.0769230769},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":36.11,"latency":55.1,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":666646.1538461539},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":14.84,"latency":92.7,"recall":0.96,"label":1,"note":"","version":"","test_time":"2023-08","qp$":273969.23076923075},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":322.7,"latency":26.4,"recall":0.9478,"label":1,"note":"","version":"","test_time":"2023-08","qp$":994623.287671233},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":265.5,"latency":26.9,"recall":0.9332,"label":1,"note":"","version":"","test_time":"2023-08","qp$":818321.9178082192},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":303.8,"latency":27.3,"recall":0.9478,"label":1,"note":"","version":"","test_time":"2023-08","qp$":936369.8630136987},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":180.2,"latency":28.2,"recall":0.9335,"label":1,"note":"","version":"","test_time":"2023-08","qp$":555410.9589041097},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":730.7,"latency":24.6,"recall":0.9586,"label":1,"note":"","version":"","test_time":"2023-08","qp$":2252157.5342465756},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":104.3,"latency":31.7,"recall":0.9563,"label":1,"note":"","version":"","test_time":"2023-08","qp$":321472.602739726},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":147.7,"latency":35.3,"recall":0.9707,"label":1,"note":"","version":"","test_time":"2023-08","qp$":682567.3940949935},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":782.5,"latency":25.9,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":3616174.5827984596},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":537.4975,"latency":18.9,"recall":0.8903,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1356936.1851332397},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":372.0466,"latency":17.8,"recall":0.8904,"label":1,"note":"","version":"","test_time":"2023-08","qp$":939248.0785413746},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1156.2898,"latency":14.4,"recall":0.9989,"label":1,"note":"","version":"","test_time":"2023-08","qp$":2919104.684431978},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":110.248,"latency":69.0,"recall":0.898,"label":1,"note":"","version":"","test_time":"2023-08","qp$":278325.94670406735},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":87.2601,"latency":27.799999999999997,"recall":0.898,"label":1,"note":"","version":"","test_time":"2023-08","qp$":220291.97755960727},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":125.7846,"latency":23.099999999999998,"recall":0.975,"label":1,"note":"","version":"","test_time":"2023-08","qp$":317548.77980364655},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":240.7209,"latency":17.4,"recall":0.8887,"label":1,"note":"","version":"","test_time":"2023-08","qp$":3038552.734922861},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":189.4399,"latency":17.5,"recall":0.8889,"label":1,"note":"","version":"","test_time":"2023-08","qp$":2391246.98457223},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":313.5116,"latency":16.1,"recall":0.9999,"label":1,"note":"","version":"","test_time":"2023-08","qp$":3957369.424964937},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":78.7196,"latency":49.4,"recall":0.9203,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1707172.048192771},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":68.3111,"latency":35.5,"recall":0.9202,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1481445.5421686745},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":210.2147,"latency":26.8,"recall":0.9996,"label":1,"note":"","version":"","test_time":"2023-08","qp$":4558873.012048192},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":188.6436,"latency":917.5,"recall":0.9175,"label":1,"note":"","version":"","test_time":"2023-08","qp$":2381195.5119214584},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":155.6991,"latency":917.1,"recall":0.9171,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1965346.283309958},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":445.3289,"latency":14.1,"recall":0.9999,"label":1,"note":"","version":"","test_time":"2023-08","qp$":5621262.412342216},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":633.6033,"latency":24.6,"recall":0.919,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1599559.5231416551},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":95.5682,"latency":58.7,"recall":0.9463,"label":1,"note":"","version":"","test_time":"2023-08","qp$":241266.14305750353},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":434.4062,"latency":17.4,"recall":0.9181,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1096677.643758766},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":80.3192,"latency":26.599999999999998,"recall":0.9462,"label":1,"note":"","version":"","test_time":"2023-08","qp$":202769.3688639551},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":1509.3293,"latency":18.5,"recall":0.9995,"label":1,"note":"","version":"","test_time":"2023-08","qp$":3810368.4992987383},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":166.7252,"latency":19.599999999999998,"recall":0.9988,"label":1,"note":"","version":"","test_time":"2023-08","qp$":420905.1332398317},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":67.9121,"latency":179.5,"recall":0.9909,"label":1,"note":"","version":"","test_time":"2023-08","qp$":7499.495705521471},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.7636,"latency":1921.3,"recall":0.9908,"label":1,"note":"","version":"","test_time":"2023-08","qp$":84.32392638036809},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":32.0,"latency":124.5,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":3533.7423312883434},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":63.1365,"latency":145.7,"recall":0.991,"label":1,"note":"","version":"","test_time":"2023-08","qp$":22504.09900990099},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.7512,"latency":1983.8,"recall":0.9908,"label":1,"note":"","version":"","test_time":"2023-08","qp$":267.7544554455445},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":30.1358,"latency":129.8,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":10741.473267326734},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":46.8622,"latency":123.0,"recall":0.9957,"label":1,"note":"","version":"","test_time":"2023-08","qp$":16703.358415841583},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1.8388,"latency":1063.5,"recall":0.9957,"label":1,"note":"","version":"","test_time":"2023-08","qp$":655.4138613861386},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":45.0666,"latency":109.2,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":16063.342574257427},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":43.5017,"latency":228.7,"recall":0.9957,"label":1,"note":"","version":"","test_time":"2023-08","qp$":4803.8687116564415},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1.5668,"latency":1114.4,"recall":0.9957,"label":1,"note":"","version":"","test_time":"2023-08","qp$":173.02085889570552},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":41.5443,"latency":159.0,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":4587.714110429448},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":330.0144,"latency":9.0,"recall":0.9507,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":7472024.150943397},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":271.6585,"latency":10.1,"recall":0.9678,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":6150758.490566038},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":216.5226,"latency":12.9,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":4902398.490566038},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":123.9553,"latency":23.0,"recall":0.971,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":1403267.5471698113},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":59.1479,"latency":44.5,"recall":0.9906,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":669598.8679245282},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":40.999,"latency":55.300000000000004,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":464139.62264150946},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":579.9416,"latency":9.4,"recall":0.9213,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":6565376.603773585},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":425.2529,"latency":11.299999999999999,"recall":0.9686,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":4814183.773584905},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":397.0539,"latency":13.799999999999999,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":4494949.811320755},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":516.27,"latency":7.0,"recall":0.9463,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":11689132.075471697},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":354.8416,"latency":10.0,"recall":0.9802,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":8034149.433962264},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":427.5229,"latency":8.7,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":9679763.773584906},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":2884.689,"latency":5.3,"recall":0.8801,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":8164214.150943396},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":1689.5799,"latency":6.6,"recall":0.9493,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":4781829.905660377},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1517.6792,"latency":10.0,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":4295318.4905660385},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":822.5318,"latency":5.6,"recall":0.9294,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":2327920.1886792453},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":378.9146,"latency":10.3,"recall":0.9758,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":1072399.8113207547},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":218.6854,"latency":16.2,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":618920.9433962263},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":297.5,"latency":7.2,"recall":0.974,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":6735849.0566037735},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":228.3,"latency":10.6,"recall":0.994,"label":1,"note":"","version":"v","test_time":"2023-08","qp$":5169056.603773585},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":584.0,"latency":4.6,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":13222641.509433962},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":1871.0,"latency":7.0,"recall":0.9602,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":5295283.018867925},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":556.7,"latency":6.7,"recall":0.9723,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":1575566.0377358492},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1583.0,"latency":6.8,"recall":0.9836,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":4480188.679245283},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":294.3,"latency":10.9,"recall":0.9939,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":832924.5283018869},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":2345.0,"latency":8.9,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":6636792.452830189},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":295.6,"latency":12.3,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":836603.7735849057},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":143.0,"latency":33.5,"recall":0.9818,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":3237735.8490566034},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":106.0,"latency":20.7,"recall":0.9887,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":2400000.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":189.0,"latency":11.6,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":4279245.2830188675},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":379.9721,"latency":12.4,"recall":0.982,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":4301570.9433962265},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":71.74,"latency":50.8,"recall":0.9883,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":812150.9433962264},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":287.0,"latency":14.9,"recall":0.9865,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":3249056.603773585},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":34.6654,"latency":64.69999999999999,"recall":0.9961,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":392438.4905660377},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":412.0,"latency":10.3,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":4664150.9433962265},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":42.169,"latency":46.800000000000004,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":477384.9056603773},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":5115.5303,"latency":8.7,"recall":0.9469,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":14477915.943396227},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":3685.0767,"latency":9.1,"recall":0.9736,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":10429462.358490566},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":4742.1617,"latency":9.1,"recall":0.9936,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":13421212.358490566},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":6054.4428,"latency":8.1,"recall":0.9155,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":17135215.471698113},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":4104.2598,"latency":10.6,"recall":0.9506,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":11615829.62264151},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":4252.1267,"latency":9.2,"recall":0.9964,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":12034320.849056603},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":1685.3091,"latency":13.299999999999999,"recall":0.9718,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":4769742.735849056},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":769.8991,"latency":10.7,"recall":0.9884,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":2178959.716981132},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":945.4061,"latency":10.9,"recall":0.9941,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":2675677.641509434},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":2214.9028,"latency":8.4,"recall":0.9249,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":6268592.830188679},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":827.975,"latency":12.0,"recall":0.9692,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":2343325.4716981133},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":776.9454,"latency":11.4,"recall":0.9966,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":2198902.075471698},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":269.5464,"latency":9.799999999999999,"recall":0.9776,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":6102937.358490566},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":240.0363,"latency":10.6,"recall":0.9822,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":5434784.150943397},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":218.0627,"latency":11.4,"recall":0.9936,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":4937268.679245283},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":392.8825,"latency":6.8999999999999995,"recall":0.9581,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":8895452.830188679},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":343.8204,"latency":8.4,"recall":0.968,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":7784612.8301886795},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":216.6773,"latency":10.1,"recall":0.9968,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":4905901.132075472},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":503.2284,"latency":9.0,"recall":0.9677,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":5696925.2830188675},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":413.3232,"latency":9.6,"recall":0.981,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":4679130.566037735},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":425.5492,"latency":10.200000000000001,"recall":0.9936,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":4817538.113207547},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":789.1229,"latency":5.6,"recall":0.9396,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":8933466.792452829},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":571.4257,"latency":7.7,"recall":0.9668,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":6468970.188679245},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":411.7653,"latency":9.1,"recall":0.9968,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":4661493.96226415},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":98.0448,"latency":16.1,"recall":0.9803,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":1109941.1320754716},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":58.3152,"latency":28.0,"recall":0.9891,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":660172.0754716981},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":46.8304,"latency":28.6,"recall":0.9941,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":530155.4716981131},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":170.5693,"latency":8.9,"recall":0.9605,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":1930973.2075471696},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":94.7766,"latency":19.599999999999998,"recall":0.9843,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":1072942.641509434},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":44.8695,"latency":30.900000000000002,"recall":0.9966,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":507956.60377358494},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":722.0315,"latency":7.7,"recall":0.976,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":16347883.018867927},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":467.5795,"latency":8.8,"recall":0.9898,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":10586705.660377359},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":975.2503,"latency":8.200000000000001,"recall":0.9936,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":22081138.86792453},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":873.3712,"latency":6.7,"recall":0.9477,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":19774442.264150944},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":544.6203,"latency":8.4,"recall":0.977,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":12331025.660377359},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":930.9164,"latency":9.6,"recall":0.9968,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":21077352.452830188},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0}] \ No newline at end of file From b9a0ce59ae4cb848094d53831bcd5da9a93fe564 Mon Sep 17 00:00:00 2001 From: "min.tian" Date: Fri, 25 Oct 2024 14:28:55 +0800 Subject: [PATCH 090/327] fix leaderboard data: zillizcloud version Signed-off-by: min.tian --- .../ZillizCloud/result_20230808_standard_zillizcloud.json | 2 +- vectordb_bench/results/leaderboard.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/vectordb_bench/results/ZillizCloud/result_20230808_standard_zillizcloud.json b/vectordb_bench/results/ZillizCloud/result_20230808_standard_zillizcloud.json index a0047bc45..e48158a02 100644 --- a/vectordb_bench/results/ZillizCloud/result_20230808_standard_zillizcloud.json +++ b/vectordb_bench/results/ZillizCloud/result_20230808_standard_zillizcloud.json @@ -70,7 +70,7 @@ "db": "ZillizCloud", "db_config": { "db_label": "1cu-perf", - "version": "v", + "version": "v2023.6", "uri": "**********", "user": "root", "password": "**********" diff --git a/vectordb_bench/results/leaderboard.json b/vectordb_bench/results/leaderboard.json index 768b4123c..2aa911cc2 100644 --- a/vectordb_bench/results/leaderboard.json +++ b/vectordb_bench/results/leaderboard.json @@ -1 +1 @@ -[{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":15.2269,"latency":861.8,"recall":0.9888,"label":1,"note":"","version":"","test_time":"2023-08","qp$":114368.53745044857},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":15.1749,"latency":774.3,"recall":0.989,"label":1,"note":"","version":"","test_time":"2023-08","qp$":113977.96786981013},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":27.6181,"latency":305.5,"recall":0.9999,"label":1,"note":"","version":"","test_time":"2023-08","qp$":207438.26413519715},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":11.2945,"latency":3611.2000000000003,"recall":0.996,"label":1,"note":"","version":"","test_time":"2023-08","qp$":84832.4640100146},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":17.3271,"latency":3774.7999999999997,"recall":0.9961,"label":1,"note":"","version":"","test_time":"2023-08","qp$":130143.04193615691},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":26.26,"latency":556.1,"recall":0.9999,"label":1,"note":"","version":"","test_time":"2023-08","qp$":197237.6382224077},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":100.6667,"latency":21.1,"recall":0.9909,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":101.1399,"latency":19.7,"recall":0.9907,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":52.2606,"latency":18.3,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":61.0661,"latency":49.8,"recall":0.9911,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":58.9326,"latency":44.6,"recall":0.9911,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":42.5977,"latency":54.9,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":536.0726,"latency":8.200000000000001,"recall":0.9728,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":467.179,"latency":7.0,"recall":0.9697,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":431.7512,"latency":8.3,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":274.5407,"latency":4.8999999999999995,"recall":0.9807,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":236.5672,"latency":10.3,"recall":0.981,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":309.4833,"latency":4.3,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":178.6585,"latency":13.700000000000001,"recall":0.9843,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":178.3732,"latency":15.0,"recall":0.9844,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":229.3526,"latency":12.5,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":1258.7043,"latency":4.8999999999999995,"recall":0.9799,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":1075.8776,"latency":5.3,"recall":0.98,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1494.8493,"latency":4.7,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":37.432,"latency":75.0,"recall":0.9975,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":37.0696,"latency":73.5,"recall":0.9976,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":81.1915,"latency":53.0,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":321.6048,"latency":13.4,"recall":0.989,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":22.1467,"latency":86.8,"recall":0.9972,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":303.2551,"latency":10.9,"recall":0.9876,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":21.5388,"latency":81.69999999999999,"recall":0.997,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":394.5418,"latency":8.3,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":37.878,"latency":45.0,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":180.2757,"latency":6.0,"recall":0.9942,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":179.0033,"latency":6.4,"recall":0.9943,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":526.8846,"latency":3.6,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":626.5243,"latency":6.2,"recall":0.9954,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":78.4227,"latency":25.3,"recall":0.9982,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":599.4213,"latency":6.6,"recall":0.9955,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":78.5351,"latency":26.3,"recall":0.9982,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":2098.2113,"latency":3.4,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":275.6292,"latency":10.0,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":10.6271,"latency":730.7,"recall":0.8898,"label":1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":10.8507,"latency":733.1999999999999,"recall":0.8897,"label":1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":75.7055,"latency":121.2,"recall":0.9999,"label":1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":0.8836,"latency":2523.0,"recall":0.8528,"label":1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":0.8937,"latency":3720.2000000000003,"recall":0.8525,"label":1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":1.2145,"latency":3622.3999999999996,"recall":0.7487,"label":1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":18.7634,"latency":153.70000000000002,"recall":0.8737,"label":1,"note":"","version":"","test_time":"2023-08","qp$":694226.515930113},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":18.3619,"latency":79.8,"recall":0.8741,"label":1,"note":"","version":"","test_time":"2023-08","qp$":679371.4285714285},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":25.2744,"latency":61.199999999999996,"recall":0.9979,"label":1,"note":"","version":"","test_time":"2023-08","qp$":935126.8242548818},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":261.798,"latency":23.099999999999998,"recall":0.9262,"label":1,"note":"","version":"","test_time":"2023-08","qp$":6455293.150684931},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":166.1851,"latency":23.900000000000002,"recall":0.9264,"label":1,"note":"","version":"","test_time":"2023-08","qp$":4097714.7945205485},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":121.7169,"latency":29.0,"recall":0.9693,"label":1,"note":"","version":"","test_time":"2023-08","qp$":3001238.6301369863},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":46.6189,"latency":43.1,"recall":0.8737,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1724851.3874614593},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":42.4856,"latency":44.0,"recall":0.8741,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1571923.5354573485},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":138.9479,"latency":26.200000000000003,"recall":0.9979,"label":1,"note":"","version":"","test_time":"2023-08","qp$":5140929.496402878},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":20.7437,"latency":75.80000000000001,"recall":0.9291,"label":1,"note":"","version":"","test_time":"2023-08","qp$":382960.6153846154},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":20.2993,"latency":76.5,"recall":0.9293,"label":1,"note":"","version":"","test_time":"2023-08","qp$":374756.30769230763},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":26.4719,"latency":67.0,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":488712.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":8.6675,"latency":180.2,"recall":0.8369,"label":1,"note":"","version":"","test_time":"2023-08","qp$":160015.38461538462},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":7.8121,"latency":167.7,"recall":0.8369,"label":1,"note":"","version":"","test_time":"2023-08","qp$":144223.3846153846},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":16.869,"latency":87.8,"recall":0.9814,"label":1,"note":"","version":"","test_time":"2023-08","qp$":311427.6923076923},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":365.0835,"latency":23.599999999999998,"recall":0.945,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1125257.3630136987},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":325.5271,"latency":25.1,"recall":0.9452,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1003336.9520547946},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":596.7942,"latency":24.2,"recall":0.9693,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1839434.1780821919},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":303.2044,"latency":27.400000000000002,"recall":0.9246,"label":1,"note":"","version":"","test_time":"2023-08","qp$":934534.1095890411},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":136.0345,"latency":31.9,"recall":0.9244,"label":1,"note":"","version":"","test_time":"2023-08","qp$":419284.4178082192},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":66.7221,"latency":42.1,"recall":0.963,"label":1,"note":"","version":"","test_time":"2023-08","qp$":205650.30821917808},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":131.2549,"latency":30.200000000000003,"recall":0.9867,"label":1,"note":"","version":"","test_time":"2023-08","qp$":606569.4993581515},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":127.9337,"latency":30.099999999999998,"recall":0.9869,"label":1,"note":"","version":"","test_time":"2023-08","qp$":591221.2066752247},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":595.8462,"latency":23.400000000000002,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":2753589.627727856},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":228.4,"latency":22.2,"recall":0.9348,"label":1,"note":"","version":"","test_time":"2023-08","qp$":5631780.821917809},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":181.5,"latency":26.1,"recall":0.9345,"label":1,"note":"","version":"","test_time":"2023-08","qp$":4475342.465753425},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":205.7,"latency":24.2,"recall":0.9586,"label":1,"note":"","version":"","test_time":"2023-08","qp$":5072054.794520548},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":67.63,"latency":36.0,"recall":0.8064,"label":1,"note":"","version":"","test_time":"2023-08","qp$":2502240.4933196297},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":63.35,"latency":38.4,"recall":0.8065,"label":1,"note":"","version":"","test_time":"2023-08","qp$":2343884.892086331},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":176.7,"latency":27.599999999999998,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":6537718.396711202},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":15.33,"latency":84.9,"recall":0.8064,"label":1,"note":"","version":"","test_time":"2023-08","qp$":567194.2446043165},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":15.13,"latency":86.7,"recall":0.8065,"label":1,"note":"","version":"","test_time":"2023-08","qp$":559794.4501541625},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":17.41,"latency":74.30000000000001,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":644152.1068859198},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":16.34,"latency":88.7,"recall":0.879,"label":1,"note":"","version":"","test_time":"2023-08","qp$":301661.53846153844},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":10.45,"latency":126.8,"recall":0.8208,"label":1,"note":"","version":"","test_time":"2023-08","qp$":192923.0769230769},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":16.18,"latency":87.5,"recall":0.8793,"label":1,"note":"","version":"","test_time":"2023-08","qp$":298707.69230769225},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":9.8,"latency":130.89999999999998,"recall":0.8212,"label":1,"note":"","version":"","test_time":"2023-08","qp$":180923.0769230769},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":36.11,"latency":55.1,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":666646.1538461539},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":14.84,"latency":92.7,"recall":0.96,"label":1,"note":"","version":"","test_time":"2023-08","qp$":273969.23076923075},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":322.7,"latency":26.4,"recall":0.9478,"label":1,"note":"","version":"","test_time":"2023-08","qp$":994623.287671233},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":265.5,"latency":26.9,"recall":0.9332,"label":1,"note":"","version":"","test_time":"2023-08","qp$":818321.9178082192},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":303.8,"latency":27.3,"recall":0.9478,"label":1,"note":"","version":"","test_time":"2023-08","qp$":936369.8630136987},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":180.2,"latency":28.2,"recall":0.9335,"label":1,"note":"","version":"","test_time":"2023-08","qp$":555410.9589041097},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":730.7,"latency":24.6,"recall":0.9586,"label":1,"note":"","version":"","test_time":"2023-08","qp$":2252157.5342465756},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":104.3,"latency":31.7,"recall":0.9563,"label":1,"note":"","version":"","test_time":"2023-08","qp$":321472.602739726},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":147.7,"latency":35.3,"recall":0.9707,"label":1,"note":"","version":"","test_time":"2023-08","qp$":682567.3940949935},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":782.5,"latency":25.9,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":3616174.5827984596},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":537.4975,"latency":18.9,"recall":0.8903,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1356936.1851332397},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":372.0466,"latency":17.8,"recall":0.8904,"label":1,"note":"","version":"","test_time":"2023-08","qp$":939248.0785413746},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1156.2898,"latency":14.4,"recall":0.9989,"label":1,"note":"","version":"","test_time":"2023-08","qp$":2919104.684431978},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":110.248,"latency":69.0,"recall":0.898,"label":1,"note":"","version":"","test_time":"2023-08","qp$":278325.94670406735},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":87.2601,"latency":27.799999999999997,"recall":0.898,"label":1,"note":"","version":"","test_time":"2023-08","qp$":220291.97755960727},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":125.7846,"latency":23.099999999999998,"recall":0.975,"label":1,"note":"","version":"","test_time":"2023-08","qp$":317548.77980364655},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":240.7209,"latency":17.4,"recall":0.8887,"label":1,"note":"","version":"","test_time":"2023-08","qp$":3038552.734922861},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":189.4399,"latency":17.5,"recall":0.8889,"label":1,"note":"","version":"","test_time":"2023-08","qp$":2391246.98457223},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":313.5116,"latency":16.1,"recall":0.9999,"label":1,"note":"","version":"","test_time":"2023-08","qp$":3957369.424964937},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":78.7196,"latency":49.4,"recall":0.9203,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1707172.048192771},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":68.3111,"latency":35.5,"recall":0.9202,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1481445.5421686745},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":210.2147,"latency":26.8,"recall":0.9996,"label":1,"note":"","version":"","test_time":"2023-08","qp$":4558873.012048192},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":188.6436,"latency":917.5,"recall":0.9175,"label":1,"note":"","version":"","test_time":"2023-08","qp$":2381195.5119214584},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":155.6991,"latency":917.1,"recall":0.9171,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1965346.283309958},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":445.3289,"latency":14.1,"recall":0.9999,"label":1,"note":"","version":"","test_time":"2023-08","qp$":5621262.412342216},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":633.6033,"latency":24.6,"recall":0.919,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1599559.5231416551},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":95.5682,"latency":58.7,"recall":0.9463,"label":1,"note":"","version":"","test_time":"2023-08","qp$":241266.14305750353},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":434.4062,"latency":17.4,"recall":0.9181,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1096677.643758766},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":80.3192,"latency":26.599999999999998,"recall":0.9462,"label":1,"note":"","version":"","test_time":"2023-08","qp$":202769.3688639551},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":1509.3293,"latency":18.5,"recall":0.9995,"label":1,"note":"","version":"","test_time":"2023-08","qp$":3810368.4992987383},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":166.7252,"latency":19.599999999999998,"recall":0.9988,"label":1,"note":"","version":"","test_time":"2023-08","qp$":420905.1332398317},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":67.9121,"latency":179.5,"recall":0.9909,"label":1,"note":"","version":"","test_time":"2023-08","qp$":7499.495705521471},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.7636,"latency":1921.3,"recall":0.9908,"label":1,"note":"","version":"","test_time":"2023-08","qp$":84.32392638036809},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":32.0,"latency":124.5,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":3533.7423312883434},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":63.1365,"latency":145.7,"recall":0.991,"label":1,"note":"","version":"","test_time":"2023-08","qp$":22504.09900990099},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.7512,"latency":1983.8,"recall":0.9908,"label":1,"note":"","version":"","test_time":"2023-08","qp$":267.7544554455445},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":30.1358,"latency":129.8,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":10741.473267326734},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":46.8622,"latency":123.0,"recall":0.9957,"label":1,"note":"","version":"","test_time":"2023-08","qp$":16703.358415841583},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1.8388,"latency":1063.5,"recall":0.9957,"label":1,"note":"","version":"","test_time":"2023-08","qp$":655.4138613861386},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":45.0666,"latency":109.2,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":16063.342574257427},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":43.5017,"latency":228.7,"recall":0.9957,"label":1,"note":"","version":"","test_time":"2023-08","qp$":4803.8687116564415},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1.5668,"latency":1114.4,"recall":0.9957,"label":1,"note":"","version":"","test_time":"2023-08","qp$":173.02085889570552},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":41.5443,"latency":159.0,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":4587.714110429448},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":330.0144,"latency":9.0,"recall":0.9507,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":7472024.150943397},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":271.6585,"latency":10.1,"recall":0.9678,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":6150758.490566038},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":216.5226,"latency":12.9,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":4902398.490566038},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":123.9553,"latency":23.0,"recall":0.971,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":1403267.5471698113},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":59.1479,"latency":44.5,"recall":0.9906,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":669598.8679245282},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":40.999,"latency":55.300000000000004,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":464139.62264150946},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":579.9416,"latency":9.4,"recall":0.9213,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":6565376.603773585},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":425.2529,"latency":11.299999999999999,"recall":0.9686,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":4814183.773584905},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":397.0539,"latency":13.799999999999999,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":4494949.811320755},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":516.27,"latency":7.0,"recall":0.9463,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":11689132.075471697},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":354.8416,"latency":10.0,"recall":0.9802,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":8034149.433962264},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":427.5229,"latency":8.7,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":9679763.773584906},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":2884.689,"latency":5.3,"recall":0.8801,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":8164214.150943396},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":1689.5799,"latency":6.6,"recall":0.9493,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":4781829.905660377},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1517.6792,"latency":10.0,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":4295318.4905660385},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":822.5318,"latency":5.6,"recall":0.9294,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":2327920.1886792453},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":378.9146,"latency":10.3,"recall":0.9758,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":1072399.8113207547},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":218.6854,"latency":16.2,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":618920.9433962263},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":297.5,"latency":7.2,"recall":0.974,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":6735849.0566037735},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":228.3,"latency":10.6,"recall":0.994,"label":1,"note":"","version":"v","test_time":"2023-08","qp$":5169056.603773585},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":584.0,"latency":4.6,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":13222641.509433962},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":1871.0,"latency":7.0,"recall":0.9602,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":5295283.018867925},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":556.7,"latency":6.7,"recall":0.9723,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":1575566.0377358492},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1583.0,"latency":6.8,"recall":0.9836,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":4480188.679245283},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":294.3,"latency":10.9,"recall":0.9939,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":832924.5283018869},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":2345.0,"latency":8.9,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":6636792.452830189},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":295.6,"latency":12.3,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":836603.7735849057},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":143.0,"latency":33.5,"recall":0.9818,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":3237735.8490566034},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":106.0,"latency":20.7,"recall":0.9887,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":2400000.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":189.0,"latency":11.6,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":4279245.2830188675},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":379.9721,"latency":12.4,"recall":0.982,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":4301570.9433962265},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":71.74,"latency":50.8,"recall":0.9883,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":812150.9433962264},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":287.0,"latency":14.9,"recall":0.9865,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":3249056.603773585},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":34.6654,"latency":64.69999999999999,"recall":0.9961,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":392438.4905660377},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":412.0,"latency":10.3,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":4664150.9433962265},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":42.169,"latency":46.800000000000004,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":477384.9056603773},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":5115.5303,"latency":8.7,"recall":0.9469,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":14477915.943396227},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":3685.0767,"latency":9.1,"recall":0.9736,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":10429462.358490566},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":4742.1617,"latency":9.1,"recall":0.9936,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":13421212.358490566},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":6054.4428,"latency":8.1,"recall":0.9155,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":17135215.471698113},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":4104.2598,"latency":10.6,"recall":0.9506,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":11615829.62264151},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":4252.1267,"latency":9.2,"recall":0.9964,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":12034320.849056603},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":1685.3091,"latency":13.299999999999999,"recall":0.9718,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":4769742.735849056},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":769.8991,"latency":10.7,"recall":0.9884,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":2178959.716981132},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":945.4061,"latency":10.9,"recall":0.9941,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":2675677.641509434},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":2214.9028,"latency":8.4,"recall":0.9249,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":6268592.830188679},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":827.975,"latency":12.0,"recall":0.9692,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":2343325.4716981133},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":776.9454,"latency":11.4,"recall":0.9966,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":2198902.075471698},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":269.5464,"latency":9.799999999999999,"recall":0.9776,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":6102937.358490566},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":240.0363,"latency":10.6,"recall":0.9822,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":5434784.150943397},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":218.0627,"latency":11.4,"recall":0.9936,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":4937268.679245283},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":392.8825,"latency":6.8999999999999995,"recall":0.9581,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":8895452.830188679},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":343.8204,"latency":8.4,"recall":0.968,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":7784612.8301886795},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":216.6773,"latency":10.1,"recall":0.9968,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":4905901.132075472},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":503.2284,"latency":9.0,"recall":0.9677,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":5696925.2830188675},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":413.3232,"latency":9.6,"recall":0.981,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":4679130.566037735},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":425.5492,"latency":10.200000000000001,"recall":0.9936,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":4817538.113207547},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":789.1229,"latency":5.6,"recall":0.9396,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":8933466.792452829},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":571.4257,"latency":7.7,"recall":0.9668,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":6468970.188679245},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":411.7653,"latency":9.1,"recall":0.9968,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":4661493.96226415},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":98.0448,"latency":16.1,"recall":0.9803,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":1109941.1320754716},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":58.3152,"latency":28.0,"recall":0.9891,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":660172.0754716981},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":46.8304,"latency":28.6,"recall":0.9941,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":530155.4716981131},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":170.5693,"latency":8.9,"recall":0.9605,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":1930973.2075471696},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":94.7766,"latency":19.599999999999998,"recall":0.9843,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":1072942.641509434},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":44.8695,"latency":30.900000000000002,"recall":0.9966,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":507956.60377358494},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":722.0315,"latency":7.7,"recall":0.976,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":16347883.018867927},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":467.5795,"latency":8.8,"recall":0.9898,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":10586705.660377359},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":975.2503,"latency":8.200000000000001,"recall":0.9936,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":22081138.86792453},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":873.3712,"latency":6.7,"recall":0.9477,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":19774442.264150944},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":544.6203,"latency":8.4,"recall":0.977,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":12331025.660377359},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":930.9164,"latency":9.6,"recall":0.9968,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":21077352.452830188},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0}] \ No newline at end of file +[{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":15.2269,"latency":861.8,"recall":0.9888,"label":1,"note":"","version":"","test_time":"2023-08","qp$":114368.53745044857},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":15.1749,"latency":774.3,"recall":0.989,"label":1,"note":"","version":"","test_time":"2023-08","qp$":113977.96786981013},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":27.6181,"latency":305.5,"recall":0.9999,"label":1,"note":"","version":"","test_time":"2023-08","qp$":207438.26413519715},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":11.2945,"latency":3611.2000000000003,"recall":0.996,"label":1,"note":"","version":"","test_time":"2023-08","qp$":84832.4640100146},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":17.3271,"latency":3774.7999999999997,"recall":0.9961,"label":1,"note":"","version":"","test_time":"2023-08","qp$":130143.04193615691},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":26.26,"latency":556.1,"recall":0.9999,"label":1,"note":"","version":"","test_time":"2023-08","qp$":197237.6382224077},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":100.6667,"latency":21.1,"recall":0.9909,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":101.1399,"latency":19.7,"recall":0.9907,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":52.2606,"latency":18.3,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":61.0661,"latency":49.8,"recall":0.9911,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":58.9326,"latency":44.6,"recall":0.9911,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":42.5977,"latency":54.9,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":536.0726,"latency":8.200000000000001,"recall":0.9728,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":467.179,"latency":7.0,"recall":0.9697,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":431.7512,"latency":8.3,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":274.5407,"latency":4.8999999999999995,"recall":0.9807,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":236.5672,"latency":10.3,"recall":0.981,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":309.4833,"latency":4.3,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":178.6585,"latency":13.700000000000001,"recall":0.9843,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":178.3732,"latency":15.0,"recall":0.9844,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":229.3526,"latency":12.5,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":1258.7043,"latency":4.8999999999999995,"recall":0.9799,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":1075.8776,"latency":5.3,"recall":0.98,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1494.8493,"latency":4.7,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":37.432,"latency":75.0,"recall":0.9975,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":37.0696,"latency":73.5,"recall":0.9976,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":81.1915,"latency":53.0,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":321.6048,"latency":13.4,"recall":0.989,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":22.1467,"latency":86.8,"recall":0.9972,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":303.2551,"latency":10.9,"recall":0.9876,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":21.5388,"latency":81.69999999999999,"recall":0.997,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":394.5418,"latency":8.3,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":37.878,"latency":45.0,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":180.2757,"latency":6.0,"recall":0.9942,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":179.0033,"latency":6.4,"recall":0.9943,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":526.8846,"latency":3.6,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":626.5243,"latency":6.2,"recall":0.9954,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":78.4227,"latency":25.3,"recall":0.9982,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":599.4213,"latency":6.6,"recall":0.9955,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":78.5351,"latency":26.3,"recall":0.9982,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":2098.2113,"latency":3.4,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":275.6292,"latency":10.0,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":10.6271,"latency":730.7,"recall":0.8898,"label":1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":10.8507,"latency":733.1999999999999,"recall":0.8897,"label":1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":75.7055,"latency":121.2,"recall":0.9999,"label":1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":0.8836,"latency":2523.0,"recall":0.8528,"label":1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":0.8937,"latency":3720.2000000000003,"recall":0.8525,"label":1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":1.2145,"latency":3622.3999999999996,"recall":0.7487,"label":1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":18.7634,"latency":153.70000000000002,"recall":0.8737,"label":1,"note":"","version":"","test_time":"2023-08","qp$":694226.515930113},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":18.3619,"latency":79.8,"recall":0.8741,"label":1,"note":"","version":"","test_time":"2023-08","qp$":679371.4285714285},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":25.2744,"latency":61.199999999999996,"recall":0.9979,"label":1,"note":"","version":"","test_time":"2023-08","qp$":935126.8242548818},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":261.798,"latency":23.099999999999998,"recall":0.9262,"label":1,"note":"","version":"","test_time":"2023-08","qp$":6455293.150684931},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":166.1851,"latency":23.900000000000002,"recall":0.9264,"label":1,"note":"","version":"","test_time":"2023-08","qp$":4097714.7945205485},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":121.7169,"latency":29.0,"recall":0.9693,"label":1,"note":"","version":"","test_time":"2023-08","qp$":3001238.6301369863},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":46.6189,"latency":43.1,"recall":0.8737,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1724851.3874614593},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":42.4856,"latency":44.0,"recall":0.8741,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1571923.5354573485},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":138.9479,"latency":26.200000000000003,"recall":0.9979,"label":1,"note":"","version":"","test_time":"2023-08","qp$":5140929.496402878},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":20.7437,"latency":75.80000000000001,"recall":0.9291,"label":1,"note":"","version":"","test_time":"2023-08","qp$":382960.6153846154},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":20.2993,"latency":76.5,"recall":0.9293,"label":1,"note":"","version":"","test_time":"2023-08","qp$":374756.30769230763},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":26.4719,"latency":67.0,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":488712.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":8.6675,"latency":180.2,"recall":0.8369,"label":1,"note":"","version":"","test_time":"2023-08","qp$":160015.38461538462},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":7.8121,"latency":167.7,"recall":0.8369,"label":1,"note":"","version":"","test_time":"2023-08","qp$":144223.3846153846},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":16.869,"latency":87.8,"recall":0.9814,"label":1,"note":"","version":"","test_time":"2023-08","qp$":311427.6923076923},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":365.0835,"latency":23.599999999999998,"recall":0.945,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1125257.3630136987},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":325.5271,"latency":25.1,"recall":0.9452,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1003336.9520547946},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":596.7942,"latency":24.2,"recall":0.9693,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1839434.1780821919},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":303.2044,"latency":27.400000000000002,"recall":0.9246,"label":1,"note":"","version":"","test_time":"2023-08","qp$":934534.1095890411},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":136.0345,"latency":31.9,"recall":0.9244,"label":1,"note":"","version":"","test_time":"2023-08","qp$":419284.4178082192},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":66.7221,"latency":42.1,"recall":0.963,"label":1,"note":"","version":"","test_time":"2023-08","qp$":205650.30821917808},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":131.2549,"latency":30.200000000000003,"recall":0.9867,"label":1,"note":"","version":"","test_time":"2023-08","qp$":606569.4993581515},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":127.9337,"latency":30.099999999999998,"recall":0.9869,"label":1,"note":"","version":"","test_time":"2023-08","qp$":591221.2066752247},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":595.8462,"latency":23.400000000000002,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":2753589.627727856},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":228.4,"latency":22.2,"recall":0.9348,"label":1,"note":"","version":"","test_time":"2023-08","qp$":5631780.821917809},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":181.5,"latency":26.1,"recall":0.9345,"label":1,"note":"","version":"","test_time":"2023-08","qp$":4475342.465753425},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":205.7,"latency":24.2,"recall":0.9586,"label":1,"note":"","version":"","test_time":"2023-08","qp$":5072054.794520548},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":67.63,"latency":36.0,"recall":0.8064,"label":1,"note":"","version":"","test_time":"2023-08","qp$":2502240.4933196297},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":63.35,"latency":38.4,"recall":0.8065,"label":1,"note":"","version":"","test_time":"2023-08","qp$":2343884.892086331},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":176.7,"latency":27.599999999999998,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":6537718.396711202},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":15.33,"latency":84.9,"recall":0.8064,"label":1,"note":"","version":"","test_time":"2023-08","qp$":567194.2446043165},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":15.13,"latency":86.7,"recall":0.8065,"label":1,"note":"","version":"","test_time":"2023-08","qp$":559794.4501541625},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":17.41,"latency":74.30000000000001,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":644152.1068859198},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":16.34,"latency":88.7,"recall":0.879,"label":1,"note":"","version":"","test_time":"2023-08","qp$":301661.53846153844},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":10.45,"latency":126.8,"recall":0.8208,"label":1,"note":"","version":"","test_time":"2023-08","qp$":192923.0769230769},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":16.18,"latency":87.5,"recall":0.8793,"label":1,"note":"","version":"","test_time":"2023-08","qp$":298707.69230769225},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":9.8,"latency":130.89999999999998,"recall":0.8212,"label":1,"note":"","version":"","test_time":"2023-08","qp$":180923.0769230769},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":36.11,"latency":55.1,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":666646.1538461539},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":14.84,"latency":92.7,"recall":0.96,"label":1,"note":"","version":"","test_time":"2023-08","qp$":273969.23076923075},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":322.7,"latency":26.4,"recall":0.9478,"label":1,"note":"","version":"","test_time":"2023-08","qp$":994623.287671233},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":265.5,"latency":26.9,"recall":0.9332,"label":1,"note":"","version":"","test_time":"2023-08","qp$":818321.9178082192},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":303.8,"latency":27.3,"recall":0.9478,"label":1,"note":"","version":"","test_time":"2023-08","qp$":936369.8630136987},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":180.2,"latency":28.2,"recall":0.9335,"label":1,"note":"","version":"","test_time":"2023-08","qp$":555410.9589041097},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":730.7,"latency":24.6,"recall":0.9586,"label":1,"note":"","version":"","test_time":"2023-08","qp$":2252157.5342465756},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":104.3,"latency":31.7,"recall":0.9563,"label":1,"note":"","version":"","test_time":"2023-08","qp$":321472.602739726},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":147.7,"latency":35.3,"recall":0.9707,"label":1,"note":"","version":"","test_time":"2023-08","qp$":682567.3940949935},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":782.5,"latency":25.9,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":3616174.5827984596},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":537.4975,"latency":18.9,"recall":0.8903,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1356936.1851332397},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":372.0466,"latency":17.8,"recall":0.8904,"label":1,"note":"","version":"","test_time":"2023-08","qp$":939248.0785413746},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1156.2898,"latency":14.4,"recall":0.9989,"label":1,"note":"","version":"","test_time":"2023-08","qp$":2919104.684431978},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":110.248,"latency":69.0,"recall":0.898,"label":1,"note":"","version":"","test_time":"2023-08","qp$":278325.94670406735},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":87.2601,"latency":27.799999999999997,"recall":0.898,"label":1,"note":"","version":"","test_time":"2023-08","qp$":220291.97755960727},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":125.7846,"latency":23.099999999999998,"recall":0.975,"label":1,"note":"","version":"","test_time":"2023-08","qp$":317548.77980364655},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":240.7209,"latency":17.4,"recall":0.8887,"label":1,"note":"","version":"","test_time":"2023-08","qp$":3038552.734922861},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":189.4399,"latency":17.5,"recall":0.8889,"label":1,"note":"","version":"","test_time":"2023-08","qp$":2391246.98457223},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":313.5116,"latency":16.1,"recall":0.9999,"label":1,"note":"","version":"","test_time":"2023-08","qp$":3957369.424964937},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":78.7196,"latency":49.4,"recall":0.9203,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1707172.048192771},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":68.3111,"latency":35.5,"recall":0.9202,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1481445.5421686745},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":210.2147,"latency":26.8,"recall":0.9996,"label":1,"note":"","version":"","test_time":"2023-08","qp$":4558873.012048192},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":188.6436,"latency":917.5,"recall":0.9175,"label":1,"note":"","version":"","test_time":"2023-08","qp$":2381195.5119214584},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":155.6991,"latency":917.1,"recall":0.9171,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1965346.283309958},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":445.3289,"latency":14.1,"recall":0.9999,"label":1,"note":"","version":"","test_time":"2023-08","qp$":5621262.412342216},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":633.6033,"latency":24.6,"recall":0.919,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1599559.5231416551},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":95.5682,"latency":58.7,"recall":0.9463,"label":1,"note":"","version":"","test_time":"2023-08","qp$":241266.14305750353},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":434.4062,"latency":17.4,"recall":0.9181,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1096677.643758766},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":80.3192,"latency":26.599999999999998,"recall":0.9462,"label":1,"note":"","version":"","test_time":"2023-08","qp$":202769.3688639551},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":1509.3293,"latency":18.5,"recall":0.9995,"label":1,"note":"","version":"","test_time":"2023-08","qp$":3810368.4992987383},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":166.7252,"latency":19.599999999999998,"recall":0.9988,"label":1,"note":"","version":"","test_time":"2023-08","qp$":420905.1332398317},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":67.9121,"latency":179.5,"recall":0.9909,"label":1,"note":"","version":"","test_time":"2023-08","qp$":7499.495705521471},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.7636,"latency":1921.3,"recall":0.9908,"label":1,"note":"","version":"","test_time":"2023-08","qp$":84.32392638036809},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":32.0,"latency":124.5,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":3533.7423312883434},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":63.1365,"latency":145.7,"recall":0.991,"label":1,"note":"","version":"","test_time":"2023-08","qp$":22504.09900990099},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.7512,"latency":1983.8,"recall":0.9908,"label":1,"note":"","version":"","test_time":"2023-08","qp$":267.7544554455445},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":30.1358,"latency":129.8,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":10741.473267326734},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":46.8622,"latency":123.0,"recall":0.9957,"label":1,"note":"","version":"","test_time":"2023-08","qp$":16703.358415841583},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1.8388,"latency":1063.5,"recall":0.9957,"label":1,"note":"","version":"","test_time":"2023-08","qp$":655.4138613861386},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":45.0666,"latency":109.2,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":16063.342574257427},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":43.5017,"latency":228.7,"recall":0.9957,"label":1,"note":"","version":"","test_time":"2023-08","qp$":4803.8687116564415},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1.5668,"latency":1114.4,"recall":0.9957,"label":1,"note":"","version":"","test_time":"2023-08","qp$":173.02085889570552},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":41.5443,"latency":159.0,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":4587.714110429448},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":330.0144,"latency":9.0,"recall":0.9507,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":7472024.150943397},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":271.6585,"latency":10.1,"recall":0.9678,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":6150758.490566038},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":216.5226,"latency":12.9,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":4902398.490566038},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":123.9553,"latency":23.0,"recall":0.971,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":1403267.5471698113},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":59.1479,"latency":44.5,"recall":0.9906,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":669598.8679245282},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":40.999,"latency":55.300000000000004,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":464139.62264150946},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":579.9416,"latency":9.4,"recall":0.9213,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":6565376.603773585},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":425.2529,"latency":11.299999999999999,"recall":0.9686,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":4814183.773584905},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":397.0539,"latency":13.799999999999999,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":4494949.811320755},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":516.27,"latency":7.0,"recall":0.9463,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":11689132.075471697},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":354.8416,"latency":10.0,"recall":0.9802,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":8034149.433962264},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":427.5229,"latency":8.7,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":9679763.773584906},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":2884.689,"latency":5.3,"recall":0.8801,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":8164214.150943396},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":1689.5799,"latency":6.6,"recall":0.9493,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":4781829.905660377},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1517.6792,"latency":10.0,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":4295318.4905660385},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":822.5318,"latency":5.6,"recall":0.9294,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":2327920.1886792453},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":378.9146,"latency":10.3,"recall":0.9758,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":1072399.8113207547},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":218.6854,"latency":16.2,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":618920.9433962263},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":297.5,"latency":7.2,"recall":0.974,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":6735849.0566037735},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":228.3,"latency":10.6,"recall":0.994,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":5169056.603773585},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":584.0,"latency":4.6,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":13222641.509433962},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":1871.0,"latency":7.0,"recall":0.9602,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":5295283.018867925},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":556.7,"latency":6.7,"recall":0.9723,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":1575566.0377358492},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1583.0,"latency":6.8,"recall":0.9836,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":4480188.679245283},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":294.3,"latency":10.9,"recall":0.9939,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":832924.5283018869},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":2345.0,"latency":8.9,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":6636792.452830189},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":295.6,"latency":12.3,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":836603.7735849057},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":143.0,"latency":33.5,"recall":0.9818,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":3237735.8490566034},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":106.0,"latency":20.7,"recall":0.9887,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":2400000.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":189.0,"latency":11.6,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":4279245.2830188675},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":379.9721,"latency":12.4,"recall":0.982,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":4301570.9433962265},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":71.74,"latency":50.8,"recall":0.9883,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":812150.9433962264},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":287.0,"latency":14.9,"recall":0.9865,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":3249056.603773585},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":34.6654,"latency":64.69999999999999,"recall":0.9961,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":392438.4905660377},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":412.0,"latency":10.3,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":4664150.9433962265},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":42.169,"latency":46.800000000000004,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":477384.9056603773},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":5115.5303,"latency":8.7,"recall":0.9469,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":14477915.943396227},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":3685.0767,"latency":9.1,"recall":0.9736,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":10429462.358490566},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":4742.1617,"latency":9.1,"recall":0.9936,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":13421212.358490566},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":6054.4428,"latency":8.1,"recall":0.9155,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":17135215.471698113},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":4104.2598,"latency":10.6,"recall":0.9506,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":11615829.62264151},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":4252.1267,"latency":9.2,"recall":0.9964,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":12034320.849056603},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":1685.3091,"latency":13.299999999999999,"recall":0.9718,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":4769742.735849056},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":769.8991,"latency":10.7,"recall":0.9884,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":2178959.716981132},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":945.4061,"latency":10.9,"recall":0.9941,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":2675677.641509434},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":2214.9028,"latency":8.4,"recall":0.9249,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":6268592.830188679},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":827.975,"latency":12.0,"recall":0.9692,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":2343325.4716981133},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":776.9454,"latency":11.4,"recall":0.9966,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":2198902.075471698},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":269.5464,"latency":9.799999999999999,"recall":0.9776,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":6102937.358490566},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":240.0363,"latency":10.6,"recall":0.9822,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":5434784.150943397},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":218.0627,"latency":11.4,"recall":0.9936,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":4937268.679245283},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":392.8825,"latency":6.8999999999999995,"recall":0.9581,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":8895452.830188679},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":343.8204,"latency":8.4,"recall":0.968,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":7784612.8301886795},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":216.6773,"latency":10.1,"recall":0.9968,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":4905901.132075472},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":503.2284,"latency":9.0,"recall":0.9677,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":5696925.2830188675},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":413.3232,"latency":9.6,"recall":0.981,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":4679130.566037735},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":425.5492,"latency":10.200000000000001,"recall":0.9936,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":4817538.113207547},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":789.1229,"latency":5.6,"recall":0.9396,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":8933466.792452829},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":571.4257,"latency":7.7,"recall":0.9668,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":6468970.188679245},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":411.7653,"latency":9.1,"recall":0.9968,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":4661493.96226415},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":98.0448,"latency":16.1,"recall":0.9803,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":1109941.1320754716},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":58.3152,"latency":28.0,"recall":0.9891,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":660172.0754716981},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":46.8304,"latency":28.6,"recall":0.9941,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":530155.4716981131},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":170.5693,"latency":8.9,"recall":0.9605,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":1930973.2075471696},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":94.7766,"latency":19.599999999999998,"recall":0.9843,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":1072942.641509434},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":44.8695,"latency":30.900000000000002,"recall":0.9966,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":507956.60377358494},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":722.0315,"latency":7.7,"recall":0.976,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":16347883.018867927},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":467.5795,"latency":8.8,"recall":0.9898,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":10586705.660377359},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":975.2503,"latency":8.200000000000001,"recall":0.9936,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":22081138.86792453},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":873.3712,"latency":6.7,"recall":0.9477,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":19774442.264150944},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":544.6203,"latency":8.4,"recall":0.977,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":12331025.660377359},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":930.9164,"latency":9.6,"recall":0.9968,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":21077352.452830188},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0}] \ No newline at end of file From 37b0c7c6dbb81365534780ef1ea07d18072a1378 Mon Sep 17 00:00:00 2001 From: Sheharyar Ahmad Date: Fri, 25 Oct 2024 15:10:27 +0500 Subject: [PATCH 091/327] Fixed custom_case key error in parameters dict in CLI command. --- vectordb_bench/cli/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vectordb_bench/cli/cli.py b/vectordb_bench/cli/cli.py index 950bf730b..7cc430aca 100644 --- a/vectordb_bench/cli/cli.py +++ b/vectordb_bench/cli/cli.py @@ -479,7 +479,7 @@ def run( concurrency_duration=parameters["concurrency_duration"], num_concurrency=[int(s) for s in parameters["num_concurrency"]], ), - custom_case=parameters["custom_case"], + custom_case=parameters.get("custom_case", {}), ), stages=parse_task_stages( ( From c5aa67e3c05875d086192bbf612d67124276b85e Mon Sep 17 00:00:00 2001 From: Sheharyar Ahmad Date: Fri, 25 Oct 2024 15:22:28 +0500 Subject: [PATCH 092/327] Refactored command options for consistency. --- vectordb_bench/cli/cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vectordb_bench/cli/cli.py b/vectordb_bench/cli/cli.py index 7cc430aca..edce758e1 100644 --- a/vectordb_bench/cli/cli.py +++ b/vectordb_bench/cli/cli.py @@ -379,7 +379,7 @@ class CommonTypedDict(TypedDict): custom_dataset_use_shuffled: Annotated[ bool, click.option( - "--custom-dataset-use-shuffled/--no-custom-dataset-use-shuffled", + "--custom-dataset-use-shuffled/--skip-custom-dataset-use-shuffled", help="Custom dataset use shuffled", default=False, show_default=True, @@ -388,7 +388,7 @@ class CommonTypedDict(TypedDict): custom_dataset_with_gt: Annotated[ bool, click.option( - "--custom-dataset-with-gt/--no-custom-dataset-with-gt", + "--custom-dataset-with-gt/--skip-custom-dataset-with-gt", help="Custom dataset with ground truth", default=True, show_default=True, From 61e2808b682e0b22f4b1dbba1acb47f79f6eba24 Mon Sep 17 00:00:00 2001 From: Sheharyar Ahmad Date: Fri, 25 Oct 2024 15:23:57 +0500 Subject: [PATCH 093/327] Updated readme, added custom case related command options information. Added instruction to include custom_case while adding the CLI support for the client. --- README.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/README.md b/README.md index f76ac5c10..373be031d 100644 --- a/README.md +++ b/README.md @@ -119,6 +119,27 @@ Options: --ef-search INTEGER hnsw ef-search --quantization-type [none|halfvec] quantization type for vectors + --custom-case-name TEXT Custom case name i.e. PerformanceCase1536D50K + --custom-case-description TEXT Custom name description + --custom-case-load-timeout INTEGER + Custom case load timeout [default: 36000] + --custom-case-optimize-timeout INTEGER + Custom case optimize timeout [default: 36000] + --custom-dataset-name TEXT + Dataset name i.e OpenAI + --custom-dataset-dir TEXT Dataset directory i.e. openai_medium_500k + --custom-dataset-size INTEGER Dataset size i.e. 500000 + --custom-dataset-dim INTEGER Dataset dimension + --custom-dataset-metric-type TEXT + Dataset distance metric [default: COSINE] + --custom-dataset-file-count INTEGER + Dataset file count + --custom-dataset-use-shuffled / --skip-custom-dataset-use-shuffled + Use shuffled custom dataset or skip [default: custom-dataset- + use-shuffled] + --custom-dataset-with-gt / --skip-custom-dataset-with-gt + Custom dataset with ground truth or skip [default: custom-dataset- + with-gt] --help Show this message and exit. ``` #### Using a configuration file. @@ -463,6 +484,8 @@ def ZillizAutoIndex(**parameters: Unpack[ZillizTypedDict]): 3. Update db_config and db_case_config to match client requirements 4. Continue to add new functions for each index config. 5. Import the client cli module and command to vectordb_bench/cli/vectordbbench.py (for databases with multiple commands (index configs), this only needs to be done for one command) + 6. Import the `get_custom_case_config` function from `vectordb_bench/cli/cli.py` and add a new key `custom_case` to the `parameters` variable within the command. + > cli modules with multiple index configs: > - pgvector: vectordb_bench/backend/clients/pgvector/cli.py From b5316801bd615430a89334534a99a08d233c3fdc Mon Sep 17 00:00:00 2001 From: Sheharyar Ahmad Date: Fri, 25 Oct 2024 15:34:02 +0500 Subject: [PATCH 094/327] update the instruction for adding custom_case support in new CLI implementation --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 373be031d..b779af11f 100644 --- a/README.md +++ b/README.md @@ -484,7 +484,7 @@ def ZillizAutoIndex(**parameters: Unpack[ZillizTypedDict]): 3. Update db_config and db_case_config to match client requirements 4. Continue to add new functions for each index config. 5. Import the client cli module and command to vectordb_bench/cli/vectordbbench.py (for databases with multiple commands (index configs), this only needs to be done for one command) - 6. Import the `get_custom_case_config` function from `vectordb_bench/cli/cli.py` and add a new key `custom_case` to the `parameters` variable within the command. + 6. Import the `get_custom_case_config` function from `vectordb_bench/cli/cli.py` and use it to add a new key `custom_case` to the `parameters` variable within the command. > cli modules with multiple index configs: From 32c1a538bca6cd884688ba50e1b48b41bc095bfc Mon Sep 17 00:00:00 2001 From: "min.tian" Date: Mon, 28 Oct 2024 09:38:46 +0800 Subject: [PATCH 095/327] add key for plotly_chart Signed-off-by: min.tian --- .../components/check_results/charts.py | 15 +++++++++------ .../frontend/pages/quries_per_dollar.py | 18 +++++++++++++----- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/vectordb_bench/frontend/components/check_results/charts.py b/vectordb_bench/frontend/components/check_results/charts.py index c2b2813b8..9e869b479 100644 --- a/vectordb_bench/frontend/components/check_results/charts.py +++ b/vectordb_bench/frontend/components/check_results/charts.py @@ -1,5 +1,7 @@ from vectordb_bench.backend.cases import Case -from vectordb_bench.frontend.components.check_results.expanderStyle import initMainExpanderStyle +from vectordb_bench.frontend.components.check_results.expanderStyle import ( + initMainExpanderStyle, +) from vectordb_bench.metric import metricOrder, isLowerIsBetterMetric, metricUnitMap from vectordb_bench.frontend.config.styles import * from vectordb_bench.models import ResultLabel @@ -11,7 +13,7 @@ def drawCharts(st, allData, failedTasks, caseNames: list[str]): for caseName in caseNames: chartContainer = st.expander(caseName, True) data = [data for data in allData if data["case_name"] == caseName] - drawChart(data, chartContainer) + drawChart(data, chartContainer, key_prefix=caseName) errorDBs = failedTasks[caseName] showFailedDBs(chartContainer, errorDBs) @@ -35,7 +37,7 @@ def showFailedText(st, text, dbs): ) -def drawChart(data, st): +def drawChart(data, st, key_prefix: str): metricsSet = set() for d in data: metricsSet = metricsSet.union(d["metricsSet"]) @@ -43,7 +45,8 @@ def drawChart(data, st): for i, metric in enumerate(showMetrics): container = st.container() - drawMetricChart(data, metric, container) + key = f"{key_prefix}-{metric}" + drawMetricChart(data, metric, container, key=key) def getLabelToShapeMap(data): @@ -75,7 +78,7 @@ def getLabelToShapeMap(data): return labelToShapeMap -def drawMetricChart(data, metric, st): +def drawMetricChart(data, metric, st, key: str): dataWithMetric = [d for d in data if d.get(metric, 0) > 1e-7] # dataWithMetric = data if len(dataWithMetric) == 0: @@ -161,4 +164,4 @@ def drawMetricChart(data, metric, st): ), ) - chart.plotly_chart(fig, use_container_width=True) + chart.plotly_chart(fig, use_container_width=True, key=key) diff --git a/vectordb_bench/frontend/pages/quries_per_dollar.py b/vectordb_bench/frontend/pages/quries_per_dollar.py index 0bb05294b..4a45181de 100644 --- a/vectordb_bench/frontend/pages/quries_per_dollar.py +++ b/vectordb_bench/frontend/pages/quries_per_dollar.py @@ -1,10 +1,17 @@ import streamlit as st from vectordb_bench.frontend.components.check_results.footer import footer -from vectordb_bench.frontend.components.check_results.expanderStyle import initMainExpanderStyle +from vectordb_bench.frontend.components.check_results.expanderStyle import ( + initMainExpanderStyle, +) from vectordb_bench.frontend.components.check_results.priceTable import priceTable -from vectordb_bench.frontend.components.check_results.stPageConfig import initResultsPageConfig +from vectordb_bench.frontend.components.check_results.stPageConfig import ( + initResultsPageConfig, +) from vectordb_bench.frontend.components.check_results.headerIcon import drawHeaderIcon -from vectordb_bench.frontend.components.check_results.nav import NavToResults, NavToRunTest +from vectordb_bench.frontend.components.check_results.nav import ( + NavToResults, + NavToRunTest, +) from vectordb_bench.frontend.components.check_results.charts import drawMetricChart from vectordb_bench.frontend.components.check_results.filters import getshownData from vectordb_bench.frontend.components.get_results.saveAsImage import getResults @@ -16,7 +23,7 @@ def main(): # set page config initResultsPageConfig(st) - + # header drawHeaderIcon(st) @@ -57,7 +64,8 @@ def main(): dataWithMetric.append(d) if len(dataWithMetric) > 0: chartContainer = st.expander(caseName, True) - drawMetricChart(data, metric, chartContainer) + key = f"{caseName}-{metric}" + drawMetricChart(data, metric, chartContainer, key=key) # footer footer(st.container()) From 523726fc0ff74a6024efe4bdb09db03bf39e80b3 Mon Sep 17 00:00:00 2001 From: "min.tian" Date: Mon, 28 Oct 2024 09:52:31 +0800 Subject: [PATCH 096/327] add key for plotly_chart Signed-off-by: min.tian --- vectordb_bench/frontend/components/concurrent/charts.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/vectordb_bench/frontend/components/concurrent/charts.py b/vectordb_bench/frontend/components/concurrent/charts.py index 7fc509ea5..68e7f3210 100644 --- a/vectordb_bench/frontend/components/concurrent/charts.py +++ b/vectordb_bench/frontend/components/concurrent/charts.py @@ -22,7 +22,7 @@ def drawChartsByCase(allData, showCaseNames: list[str], st): for caseData in caseDataList for i in range(len(caseData["conc_num_list"])) ] - drawChart(data, chartContainer) + drawChart(data, chartContainer, key=f"{caseName}-qps-p99") def getRange(metric, data, padding_multipliers): @@ -36,7 +36,7 @@ def getRange(metric, data, padding_multipliers): return rangeV -def drawChart(data, st): +def drawChart(data, st, key: str): if len(data) == 0: return @@ -73,7 +73,4 @@ def drawChart(data, st): fig.update_yaxes(range=yrange, title_text="QPS") fig.update_traces(textposition="bottom right", texttemplate="conc-%{text:,.4~r}") - st.plotly_chart( - fig, - use_container_width=True, - ) + st.plotly_chart(fig, use_container_width=True, key=key) From 6939b91e120475b9e2372851c7f8d0ca475eb7b7 Mon Sep 17 00:00:00 2001 From: "min.tian" Date: Mon, 28 Oct 2024 14:24:52 +0800 Subject: [PATCH 097/327] fix pinecone client Signed-off-by: min.tian --- .../backend/clients/pinecone/config.py | 2 - .../backend/clients/pinecone/pinecone.py | 70 +++++++++---------- 2 files changed, 34 insertions(+), 38 deletions(-) diff --git a/vectordb_bench/backend/clients/pinecone/config.py b/vectordb_bench/backend/clients/pinecone/config.py index dc1596379..2bbcbb350 100644 --- a/vectordb_bench/backend/clients/pinecone/config.py +++ b/vectordb_bench/backend/clients/pinecone/config.py @@ -4,12 +4,10 @@ class PineconeConfig(DBConfig): api_key: SecretStr - environment: SecretStr index_name: str def to_dict(self) -> dict: return { "api_key": self.api_key.get_secret_value(), - "environment": self.environment.get_secret_value(), "index_name": self.index_name, } diff --git a/vectordb_bench/backend/clients/pinecone/pinecone.py b/vectordb_bench/backend/clients/pinecone/pinecone.py index c2653ee27..c1351f7a9 100644 --- a/vectordb_bench/backend/clients/pinecone/pinecone.py +++ b/vectordb_bench/backend/clients/pinecone/pinecone.py @@ -3,7 +3,7 @@ import logging from contextlib import contextmanager from typing import Type - +import pinecone from ..api import VectorDB, DBConfig, DBCaseConfig, EmptyDBCaseConfig, IndexType from .config import PineconeConfig @@ -11,7 +11,8 @@ log = logging.getLogger(__name__) PINECONE_MAX_NUM_PER_BATCH = 1000 -PINECONE_MAX_SIZE_PER_BATCH = 2 * 1024 * 1024 # 2MB +PINECONE_MAX_SIZE_PER_BATCH = 2 * 1024 * 1024 # 2MB + class Pinecone(VectorDB): def __init__( @@ -23,30 +24,25 @@ def __init__( **kwargs, ): """Initialize wrapper around the milvus vector database.""" - self.index_name = db_config["index_name"] - self.api_key = db_config["api_key"] - self.environment = db_config["environment"] - self.batch_size = int(min(PINECONE_MAX_SIZE_PER_BATCH / (dim * 5), PINECONE_MAX_NUM_PER_BATCH)) - # Pincone will make connections with server while import - # so place the import here. - import pinecone - pinecone.init( - api_key=self.api_key, environment=self.environment) + self.index_name = db_config.get("index_name", "") + self.api_key = db_config.get("api_key", "") + self.batch_size = int( + min(PINECONE_MAX_SIZE_PER_BATCH / (dim * 5), PINECONE_MAX_NUM_PER_BATCH) + ) + + pc = pinecone.Pinecone(api_key=self.api_key) + index = pc.Index(self.index_name) + if drop_old: - list_indexes = pinecone.list_indexes() - if self.index_name in list_indexes: - index = pinecone.Index(self.index_name) - index_dim = index.describe_index_stats()["dimension"] - if (index_dim != dim): - raise ValueError( - f"Pinecone index {self.index_name} dimension mismatch, expected {index_dim} got {dim}") - log.info( - f"Pinecone client delete old index: {self.index_name}") - index.delete(delete_all=True) - index.close() - else: + index_stats = index.describe_index_stats() + index_dim = index_stats["dimension"] + if index_dim != dim: raise ValueError( - f"Pinecone index {self.index_name} does not exist") + f"Pinecone index {self.index_name} dimension mismatch, expected {index_dim} got {dim}" + ) + for namespace in index_stats["namespaces"]: + log.info(f"Pinecone index delete namespace: {namespace}") + index.delete(delete_all=True, namespace=namespace) self._metadata_key = "meta" @@ -59,13 +55,10 @@ def case_config_cls(cls, index_type: IndexType | None = None) -> Type[DBCaseConf return EmptyDBCaseConfig @contextmanager - def init(self) -> None: - import pinecone - pinecone.init( - api_key=self.api_key, environment=self.environment) - self.index = pinecone.Index(self.index_name) + def init(self): + pc = pinecone.Pinecone(api_key=self.api_key) + self.index = pc.Index(self.index_name) yield - self.index.close() def ready_to_load(self): pass @@ -83,11 +76,16 @@ def insert_embeddings( insert_count = 0 try: for batch_start_offset in range(0, len(embeddings), self.batch_size): - batch_end_offset = min(batch_start_offset + self.batch_size, len(embeddings)) + batch_end_offset = min( + batch_start_offset + self.batch_size, len(embeddings) + ) insert_datas = [] for i in range(batch_start_offset, batch_end_offset): - insert_data = (str(metadata[i]), embeddings[i], { - self._metadata_key: metadata[i]}) + insert_data = ( + str(metadata[i]), + embeddings[i], + {self._metadata_key: metadata[i]}, + ) insert_datas.append(insert_data) self.index.upsert(insert_datas) insert_count += batch_end_offset - batch_start_offset @@ -101,7 +99,7 @@ def search_embedding( k: int = 100, filters: dict | None = None, timeout: int | None = None, - ) -> list[tuple[int, float]]: + ) -> list[int]: if filters is None: pinecone_filters = {} else: @@ -111,9 +109,9 @@ def search_embedding( top_k=k, vector=query, filter=pinecone_filters, - )['matches'] + )["matches"] except Exception as e: print(f"Error querying index: {e}") raise e - id_res = [int(one_res['id']) for one_res in res] + id_res = [int(one_res["id"]) for one_res in res] return id_res From d2096595816ac2562802a325415b236e3a61d906 Mon Sep 17 00:00:00 2001 From: Wahaj Ali <894764+wahajali@users.noreply.github.com> Date: Tue, 29 Oct 2024 13:42:04 +0500 Subject: [PATCH 098/327] Support for pgdiskann client (#388) * Add pgdiskann client * Add CLI support in pgdiskann * add pgdiskann load config in frontend. --------- Co-authored-by: Sheharyar Ahmad --- README.md | 1 + pyproject.toml | 1 + vectordb_bench/backend/clients/__init__.py | 13 + .../backend/clients/pgdiskann/cli.py | 99 +++++ .../backend/clients/pgdiskann/config.py | 145 ++++++++ .../backend/clients/pgdiskann/pgdiskann.py | 350 ++++++++++++++++++ vectordb_bench/cli/vectordbbench.py | 2 + .../frontend/config/dbCaseConfigs.py | 63 ++++ vectordb_bench/models.py | 3 + 9 files changed, 677 insertions(+) create mode 100644 vectordb_bench/backend/clients/pgdiskann/cli.py create mode 100644 vectordb_bench/backend/clients/pgdiskann/config.py create mode 100644 vectordb_bench/backend/clients/pgdiskann/pgdiskann.py diff --git a/README.md b/README.md index b779af11f..15767a980 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,7 @@ All the database client supported | pgvector | `pip install vectordb-bench[pgvector]` | | pgvecto.rs | `pip install vectordb-bench[pgvecto_rs]` | | pgvectorscale | `pip install vectordb-bench[pgvectorscale]` | +| pgdiskann | `pip install vectordb-bench[pgdiskann]` | | redis | `pip install vectordb-bench[redis]` | | memorydb | `pip install vectordb-bench[memorydb]` | | chromadb | `pip install vectordb-bench[chromadb]` | diff --git a/pyproject.toml b/pyproject.toml index 015ed8c3f..000800389 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,6 +72,7 @@ weaviate = [ "weaviate-client" ] elastic = [ "elasticsearch" ] pgvector = [ "psycopg", "psycopg-binary", "pgvector" ] pgvectorscale = [ "psycopg", "psycopg-binary", "pgvector" ] +pgdiskann = [ "psycopg", "psycopg-binary", "pgvector" ] pgvecto_rs = [ "pgvecto_rs[psycopg3]>=0.2.2" ] redis = [ "redis" ] memorydb = [ "memorydb" ] diff --git a/vectordb_bench/backend/clients/__init__.py b/vectordb_bench/backend/clients/__init__.py index 3e87e1fbe..c26aa3d6d 100644 --- a/vectordb_bench/backend/clients/__init__.py +++ b/vectordb_bench/backend/clients/__init__.py @@ -31,6 +31,7 @@ class DB(Enum): PgVector = "PgVector" PgVectoRS = "PgVectoRS" PgVectorScale = "PgVectorScale" + PgDiskANN = "PgDiskANN" Redis = "Redis" MemoryDB = "MemoryDB" Chroma = "Chroma" @@ -77,6 +78,10 @@ def init_cls(self) -> Type[VectorDB]: from .pgvectorscale.pgvectorscale import PgVectorScale return PgVectorScale + if self == DB.PgDiskANN: + from .pgdiskann.pgdiskann import PgDiskANN + return PgDiskANN + if self == DB.Redis: from .redis.redis import Redis return Redis @@ -132,6 +137,10 @@ def config_cls(self) -> Type[DBConfig]: from .pgvectorscale.config import PgVectorScaleConfig return PgVectorScaleConfig + if self == DB.PgDiskANN: + from .pgdiskann.config import PgDiskANNConfig + return PgDiskANNConfig + if self == DB.Redis: from .redis.config import RedisConfig return RedisConfig @@ -185,6 +194,10 @@ def case_config_cls(self, index_type: IndexType | None = None) -> Type[DBCaseCon from .pgvectorscale.config import _pgvectorscale_case_config return _pgvectorscale_case_config.get(index_type) + if self == DB.PgDiskANN: + from .pgdiskann.config import _pgdiskann_case_config + return _pgdiskann_case_config.get(index_type) + # DB.Pinecone, DB.Chroma, DB.Redis return EmptyDBCaseConfig diff --git a/vectordb_bench/backend/clients/pgdiskann/cli.py b/vectordb_bench/backend/clients/pgdiskann/cli.py new file mode 100644 index 000000000..18a9ecbd5 --- /dev/null +++ b/vectordb_bench/backend/clients/pgdiskann/cli.py @@ -0,0 +1,99 @@ +import click +import os +from pydantic import SecretStr + +from ....cli.cli import ( + CommonTypedDict, + cli, + click_parameter_decorators_from_typed_dict, + run, +) +from typing import Annotated, Optional, Unpack +from vectordb_bench.backend.clients import DB + + +class PgDiskAnnTypedDict(CommonTypedDict): + user_name: Annotated[ + str, click.option("--user-name", type=str, help="Db username", required=True) + ] + password: Annotated[ + str, + click.option("--password", + type=str, + help="Postgres database password", + default=lambda: os.environ.get("POSTGRES_PASSWORD", ""), + show_default="$POSTGRES_PASSWORD", + ), + ] + + host: Annotated[ + str, click.option("--host", type=str, help="Db host", required=True) + ] + db_name: Annotated[ + str, click.option("--db-name", type=str, help="Db name", required=True) + ] + max_neighbors: Annotated[ + int, + click.option( + "--max-neighbors", type=int, help="PgDiskAnn max neighbors", + ), + ] + l_value_ib: Annotated[ + int, + click.option( + "--l-value-ib", type=int, help="PgDiskAnn l_value_ib", + ), + ] + l_value_is: Annotated[ + float, + click.option( + "--l-value-is", type=float, help="PgDiskAnn l_value_is", + ), + ] + maintenance_work_mem: Annotated[ + Optional[str], + click.option( + "--maintenance-work-mem", + type=str, + help="Sets the maximum memory to be used for maintenance operations (index creation). " + "Can be entered as string with unit like '64GB' or as an integer number of KB." + "This will set the parameters: max_parallel_maintenance_workers," + " max_parallel_workers & table(parallel_workers)", + required=False, + ), + ] + max_parallel_workers: Annotated[ + Optional[int], + click.option( + "--max-parallel-workers", + type=int, + help="Sets the maximum number of parallel processes per maintenance operation (index creation)", + required=False, + ), + ] + +@cli.command() +@click_parameter_decorators_from_typed_dict(PgDiskAnnTypedDict) +def PgDiskAnn( + **parameters: Unpack[PgDiskAnnTypedDict], +): + from .config import PgDiskANNConfig, PgDiskANNImplConfig + + run( + db=DB.PgDiskANN, + db_config=PgDiskANNConfig( + db_label=parameters["db_label"], + user_name=SecretStr(parameters["user_name"]), + password=SecretStr(parameters["password"]), + host=parameters["host"], + db_name=parameters["db_name"], + ), + db_case_config=PgDiskANNImplConfig( + max_neighbors=parameters["max_neighbors"], + l_value_ib=parameters["l_value_ib"], + l_value_is=parameters["l_value_is"], + max_parallel_workers=parameters["max_parallel_workers"], + maintenance_work_mem=parameters["maintenance_work_mem"], + ), + **parameters, + ) \ No newline at end of file diff --git a/vectordb_bench/backend/clients/pgdiskann/config.py b/vectordb_bench/backend/clients/pgdiskann/config.py new file mode 100644 index 000000000..970720afa --- /dev/null +++ b/vectordb_bench/backend/clients/pgdiskann/config.py @@ -0,0 +1,145 @@ +from abc import abstractmethod +from typing import Any, Mapping, Optional, Sequence, TypedDict +from pydantic import BaseModel, SecretStr +from typing_extensions import LiteralString +from ..api import DBCaseConfig, DBConfig, IndexType, MetricType + +POSTGRE_URL_PLACEHOLDER = "postgresql://%s:%s@%s/%s" + + +class PgDiskANNConfigDict(TypedDict): + """These keys will be directly used as kwargs in psycopg connection string, + so the names must match exactly psycopg API""" + + user: str + password: str + host: str + port: int + dbname: str + + +class PgDiskANNConfig(DBConfig): + user_name: SecretStr = SecretStr("postgres") + password: SecretStr + host: str = "localhost" + port: int = 5432 + db_name: str + + def to_dict(self) -> PgDiskANNConfigDict: + user_str = self.user_name.get_secret_value() + pwd_str = self.password.get_secret_value() + return { + "host": self.host, + "port": self.port, + "dbname": self.db_name, + "user": user_str, + "password": pwd_str, + } + + +class PgDiskANNIndexConfig(BaseModel, DBCaseConfig): + metric_type: MetricType | None = None + create_index_before_load: bool = False + create_index_after_load: bool = True + maintenance_work_mem: Optional[str] + max_parallel_workers: Optional[int] + + def parse_metric(self) -> str: + if self.metric_type == MetricType.L2: + return "vector_l2_ops" + elif self.metric_type == MetricType.IP: + return "vector_ip_ops" + return "vector_cosine_ops" + + def parse_metric_fun_op(self) -> LiteralString: + if self.metric_type == MetricType.L2: + return "<->" + elif self.metric_type == MetricType.IP: + return "<#>" + return "<=>" + + def parse_metric_fun_str(self) -> str: + if self.metric_type == MetricType.L2: + return "l2_distance" + elif self.metric_type == MetricType.IP: + return "max_inner_product" + return "cosine_distance" + + @abstractmethod + def index_param(self) -> dict: + ... + + @abstractmethod + def search_param(self) -> dict: + ... + + @abstractmethod + def session_param(self) -> dict: + ... + + @staticmethod + def _optionally_build_with_options(with_options: Mapping[str, Any]) -> Sequence[dict[str, Any]]: + """Walk through mappings, creating a List of {key1 = value} pairs. That will be used to build a where clause""" + options = [] + for option_name, value in with_options.items(): + if value is not None: + options.append( + { + "option_name": option_name, + "val": str(value), + } + ) + return options + + @staticmethod + def _optionally_build_set_options( + set_mapping: Mapping[str, Any] + ) -> Sequence[dict[str, Any]]: + """Walk through options, creating 'SET 'key1 = "value1";' list""" + session_options = [] + for setting_name, value in set_mapping.items(): + if value: + session_options.append( + {"parameter": { + "setting_name": setting_name, + "val": str(value), + }, + } + ) + return session_options + + +class PgDiskANNImplConfig(PgDiskANNIndexConfig): + index: IndexType = IndexType.DISKANN + max_neighbors: int | None + l_value_ib: int | None + l_value_is: float | None + maintenance_work_mem: Optional[str] = None + max_parallel_workers: Optional[int] = None + + def index_param(self) -> dict: + return { + "metric": self.parse_metric(), + "index_type": self.index.value, + "options": { + "max_neighbors": self.max_neighbors, + "l_value_ib": self.l_value_ib, + }, + "maintenance_work_mem": self.maintenance_work_mem, + "max_parallel_workers": self.max_parallel_workers, + } + + def search_param(self) -> dict: + return { + "metric": self.parse_metric(), + "metric_fun_op": self.parse_metric_fun_op(), + } + + def session_param(self) -> dict: + return { + "diskann.l_value_is": self.l_value_is, + } + +_pgdiskann_case_config = { + IndexType.DISKANN: PgDiskANNImplConfig, +} diff --git a/vectordb_bench/backend/clients/pgdiskann/pgdiskann.py b/vectordb_bench/backend/clients/pgdiskann/pgdiskann.py new file mode 100644 index 000000000..c363490f7 --- /dev/null +++ b/vectordb_bench/backend/clients/pgdiskann/pgdiskann.py @@ -0,0 +1,350 @@ +"""Wrapper around the pg_diskann vector database over VectorDB""" + +import logging +import pprint +from contextlib import contextmanager +from typing import Any, Generator, Optional, Tuple + +import numpy as np +import psycopg +from pgvector.psycopg import register_vector +from psycopg import Connection, Cursor, sql + +from ..api import VectorDB +from .config import PgDiskANNConfigDict, PgDiskANNIndexConfig + +log = logging.getLogger(__name__) + + +class PgDiskANN(VectorDB): + """Use psycopg instructions""" + + conn: psycopg.Connection[Any] | None = None + coursor: psycopg.Cursor[Any] | None = None + + _filtered_search: sql.Composed + _unfiltered_search: sql.Composed + + def __init__( + self, + dim: int, + db_config: PgDiskANNConfigDict, + db_case_config: PgDiskANNIndexConfig, + collection_name: str = "pg_diskann_collection", + drop_old: bool = False, + **kwargs, + ): + self.name = "PgDiskANN" + self.db_config = db_config + self.case_config = db_case_config + self.table_name = collection_name + self.dim = dim + + self._index_name = "pgdiskann_index" + self._primary_field = "id" + self._vector_field = "embedding" + + self.conn, self.cursor = self._create_connection(**self.db_config) + + log.info(f"{self.name} config values: {self.db_config}\n{self.case_config}") + if not any( + ( + self.case_config.create_index_before_load, + self.case_config.create_index_after_load, + ) + ): + err = f"{self.name} config must create an index using create_index_before_load or create_index_after_load" + log.error(err) + raise RuntimeError( + f"{err}\n{pprint.pformat(self.db_config)}\n{pprint.pformat(self.case_config)}" + ) + + if drop_old: + self._drop_index() + self._drop_table() + self._create_table(dim) + if self.case_config.create_index_before_load: + self._create_index() + + self.cursor.close() + self.conn.close() + self.cursor = None + self.conn = None + + @staticmethod + def _create_connection(**kwargs) -> Tuple[Connection, Cursor]: + conn = psycopg.connect(**kwargs) + conn.cursor().execute("CREATE EXTENSION IF NOT EXISTS pg_diskann CASCADE") + conn.commit() + register_vector(conn) + conn.autocommit = False + cursor = conn.cursor() + + assert conn is not None, "Connection is not initialized" + assert cursor is not None, "Cursor is not initialized" + + return conn, cursor + + @contextmanager + def init(self) -> Generator[None, None, None]: + self.conn, self.cursor = self._create_connection(**self.db_config) + + # index configuration may have commands defined that we should set during each client session + session_options: dict[str, Any] = self.case_config.session_param() + + if len(session_options) > 0: + for setting_name, setting_val in session_options.items(): + command = sql.SQL("SET {setting_name} " + "= {setting_val};").format( + setting_name=sql.Identifier(setting_name), + setting_val=sql.Identifier(str(setting_val)), + ) + log.debug(command.as_string(self.cursor)) + self.cursor.execute(command) + self.conn.commit() + + self._filtered_search = sql.Composed( + [ + sql.SQL( + "SELECT id FROM public.{table_name} WHERE id >= %s ORDER BY embedding " + ).format(table_name=sql.Identifier(self.table_name)), + sql.SQL(self.case_config.search_param()["metric_fun_op"]), + sql.SQL(" %s::vector LIMIT %s::int"), + ] + ) + + self._unfiltered_search = sql.Composed( + [ + sql.SQL("SELECT id FROM public.{} ORDER BY embedding ").format( + sql.Identifier(self.table_name) + ), + sql.SQL(self.case_config.search_param()["metric_fun_op"]), + sql.SQL(" %s::vector LIMIT %s::int"), + ] + ) + + try: + yield + finally: + self.cursor.close() + self.conn.close() + self.cursor = None + self.conn = None + + def _drop_table(self): + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + log.info(f"{self.name} client drop table : {self.table_name}") + + self.cursor.execute( + sql.SQL("DROP TABLE IF EXISTS public.{table_name}").format( + table_name=sql.Identifier(self.table_name) + ) + ) + self.conn.commit() + + def ready_to_load(self): + pass + + def optimize(self): + self._post_insert() + + def _post_insert(self): + log.info(f"{self.name} post insert before optimize") + if self.case_config.create_index_after_load: + self._drop_index() + self._create_index() + + def _drop_index(self): + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + log.info(f"{self.name} client drop index : {self._index_name}") + + drop_index_sql = sql.SQL("DROP INDEX IF EXISTS {index_name}").format( + index_name=sql.Identifier(self._index_name) + ) + log.debug(drop_index_sql.as_string(self.cursor)) + self.cursor.execute(drop_index_sql) + self.conn.commit() + + def _set_parallel_index_build_param(self): + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + index_param = self.case_config.index_param() + + if index_param["maintenance_work_mem"] is not None: + self.cursor.execute( + sql.SQL("SET maintenance_work_mem TO {};").format( + index_param["maintenance_work_mem"] + ) + ) + self.cursor.execute( + sql.SQL("ALTER USER {} SET maintenance_work_mem TO {};").format( + sql.Identifier(self.db_config["user"]), + index_param["maintenance_work_mem"], + ) + ) + self.conn.commit() + + if index_param["max_parallel_workers"] is not None: + self.cursor.execute( + sql.SQL("SET max_parallel_maintenance_workers TO '{}';").format( + index_param["max_parallel_workers"] + ) + ) + self.cursor.execute( + sql.SQL( + "ALTER USER {} SET max_parallel_maintenance_workers TO '{}';" + ).format( + sql.Identifier(self.db_config["user"]), + index_param["max_parallel_workers"], + ) + ) + self.cursor.execute( + sql.SQL("SET max_parallel_workers TO '{}';").format( + index_param["max_parallel_workers"] + ) + ) + self.cursor.execute( + sql.SQL( + "ALTER USER {} SET max_parallel_workers TO '{}';" + ).format( + sql.Identifier(self.db_config["user"]), + index_param["max_parallel_workers"], + ) + ) + self.cursor.execute( + sql.SQL( + "ALTER TABLE {} SET (parallel_workers = {});" + ).format( + sql.Identifier(self.table_name), + index_param["max_parallel_workers"], + ) + ) + self.conn.commit() + + results = self.cursor.execute( + sql.SQL("SHOW max_parallel_maintenance_workers;") + ).fetchall() + results.extend( + self.cursor.execute(sql.SQL("SHOW max_parallel_workers;")).fetchall() + ) + results.extend( + self.cursor.execute(sql.SQL("SHOW maintenance_work_mem;")).fetchall() + ) + log.info(f"{self.name} parallel index creation parameters: {results}") + def _create_index(self): + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + log.info(f"{self.name} client create index : {self._index_name}") + + index_param: dict[str, Any] = self.case_config.index_param() + self._set_parallel_index_build_param() + + options = [] + for option_name, option_val in index_param["options"].items(): + if option_val is not None: + options.append( + sql.SQL("{option_name} = {val}").format( + option_name=sql.Identifier(option_name), + val=sql.Identifier(str(option_val)), + ) + ) + + if any(options): + with_clause = sql.SQL("WITH ({});").format(sql.SQL(", ").join(options)) + else: + with_clause = sql.Composed(()) + + index_create_sql = sql.SQL( + """ + CREATE INDEX IF NOT EXISTS {index_name} ON public.{table_name} + USING {index_type} (embedding {embedding_metric}) + """ + ).format( + index_name=sql.Identifier(self._index_name), + table_name=sql.Identifier(self.table_name), + index_type=sql.Identifier(index_param["index_type"].lower()), + embedding_metric=sql.Identifier(index_param["metric"]), + ) + index_create_sql_with_with_clause = ( + index_create_sql + with_clause + ).join(" ") + log.debug(index_create_sql_with_with_clause.as_string(self.cursor)) + self.cursor.execute(index_create_sql_with_with_clause) + self.conn.commit() + + def _create_table(self, dim: int): + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + try: + log.info(f"{self.name} client create table : {self.table_name}") + + self.cursor.execute( + sql.SQL( + "CREATE TABLE IF NOT EXISTS public.{table_name} (id BIGINT PRIMARY KEY, embedding vector({dim}));" + ).format(table_name=sql.Identifier(self.table_name), dim=dim) + ) + self.conn.commit() + except Exception as e: + log.warning( + f"Failed to create pgdiskann table: {self.table_name} error: {e}" + ) + raise e from None + + def insert_embeddings( + self, + embeddings: list[list[float]], + metadata: list[int], + **kwargs: Any, + ) -> Tuple[int, Optional[Exception]]: + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + try: + metadata_arr = np.array(metadata) + embeddings_arr = np.array(embeddings) + + with self.cursor.copy( + sql.SQL("COPY public.{table_name} FROM STDIN (FORMAT BINARY)").format( + table_name=sql.Identifier(self.table_name) + ) + ) as copy: + copy.set_types(["bigint", "vector"]) + for i, row in enumerate(metadata_arr): + copy.write_row((row, embeddings_arr[i])) + self.conn.commit() + + if kwargs.get("last_batch"): + self._post_insert() + + return len(metadata), None + except Exception as e: + log.warning( + f"Failed to insert data into table ({self.table_name}), error: {e}" + ) + return 0, e + + def search_embedding( + self, + query: list[float], + k: int = 100, + filters: dict | None = None, + timeout: int | None = None, + ) -> list[int]: + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + q = np.asarray(query) + if filters: + gt = filters.get("id") + result = self.cursor.execute( + self._filtered_search, (gt, q, k), prepare=True, binary=True + ) + else: + result = self.cursor.execute( + self._unfiltered_search, (q, k), prepare=True, binary=True + ) + + return [int(i[0]) for i in result.fetchall()] diff --git a/vectordb_bench/cli/vectordbbench.py b/vectordb_bench/cli/vectordbbench.py index e62c25a3d..4d23ed952 100644 --- a/vectordb_bench/cli/vectordbbench.py +++ b/vectordb_bench/cli/vectordbbench.py @@ -1,6 +1,7 @@ from ..backend.clients.pgvector.cli import PgVectorHNSW from ..backend.clients.pgvecto_rs.cli import PgVectoRSHNSW, PgVectoRSIVFFlat from ..backend.clients.pgvectorscale.cli import PgVectorScaleDiskAnn +from ..backend.clients.pgdiskann.cli import PgDiskAnn from ..backend.clients.redis.cli import Redis from ..backend.clients.memorydb.cli import MemoryDB from ..backend.clients.test.cli import Test @@ -22,6 +23,7 @@ cli.add_command(MilvusAutoIndex) cli.add_command(AWSOpenSearch) cli.add_command(PgVectorScaleDiskAnn) +cli.add_command(PgDiskAnn) if __name__ == "__main__": diff --git a/vectordb_bench/frontend/config/dbCaseConfigs.py b/vectordb_bench/frontend/config/dbCaseConfigs.py index 68bf83f19..7736a4e03 100644 --- a/vectordb_bench/frontend/config/dbCaseConfigs.py +++ b/vectordb_bench/frontend/config/dbCaseConfigs.py @@ -180,6 +180,16 @@ class CaseConfigInput(BaseModel): }, ) +CaseConfigParamInput_IndexType_PgDiskANN = CaseConfigInput( + label=CaseConfigParamType.IndexType, + inputHelp="Select Index Type", + inputType=InputType.Option, + inputConfig={ + "options": [ + IndexType.DISKANN.value, + ], + }, +) CaseConfigParamInput_IndexType_PgVectorScale = CaseConfigInput( label=CaseConfigParamType.IndexType, @@ -205,6 +215,42 @@ class CaseConfigInput(BaseModel): }, ) +CaseConfigParamInput_max_neighbors = CaseConfigInput( + label=CaseConfigParamType.max_neighbors, + inputType=InputType.Number, + inputConfig={ + "min": 10, + "max": 300, + "value": 32, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) + == IndexType.DISKANN.value, +) + +CaseConfigParamInput_l_value_ib = CaseConfigInput( + label=CaseConfigParamType.l_value_ib, + inputType=InputType.Number, + inputConfig={ + "min": 10, + "max": 300, + "value": 50, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) + == IndexType.DISKANN.value, +) + +CaseConfigParamInput_l_value_is = CaseConfigInput( + label=CaseConfigParamType.l_value_is, + inputType=InputType.Number, + inputConfig={ + "min": 10, + "max": 300, + "value": 40, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) + == IndexType.DISKANN.value, +) + CaseConfigParamInput_num_neighbors = CaseConfigInput( label=CaseConfigParamType.num_neighbors, inputType=InputType.Number, @@ -942,6 +988,19 @@ class CaseConfigInput(BaseModel): CaseConfigParamInput_query_search_list_size, ] +PgDiskANNLoadConfig = [ + CaseConfigParamInput_IndexType_PgDiskANN, + CaseConfigParamInput_max_neighbors, + CaseConfigParamInput_l_value_ib, +] + +PgDiskANNPerformanceConfig = [ + CaseConfigParamInput_IndexType_PgDiskANN, + CaseConfigParamInput_max_neighbors, + CaseConfigParamInput_l_value_ib, + CaseConfigParamInput_l_value_is, +] + CASE_CONFIG_MAP = { DB.Milvus: { CaseLabel.Load: MilvusLoadConfig, @@ -974,4 +1033,8 @@ class CaseConfigInput(BaseModel): CaseLabel.Load: PgVectorScaleLoadingConfig, CaseLabel.Performance: PgVectorScalePerformanceConfig, }, + DB.PgDiskANN: { + CaseLabel.Load: PgDiskANNLoadConfig, + CaseLabel.Performance: PgDiskANNPerformanceConfig, + }, } diff --git a/vectordb_bench/models.py b/vectordb_bench/models.py index 7968e3e26..d431479ed 100644 --- a/vectordb_bench/models.py +++ b/vectordb_bench/models.py @@ -64,6 +64,9 @@ class CaseConfigParamType(Enum): max_parallel_workers = "max_parallel_workers" storage_layout = "storage_layout" num_neighbors = "num_neighbors" + max_neighbors = "max_neighbors" + l_value_ib = "l_value_ib" + l_value_is = "l_value_is" search_list_size = "search_list_size" max_alpha = "max_alpha" num_dimensions = "num_dimensions" From 6a05c30073936a126075c6381870d89b04150cdc Mon Sep 17 00:00:00 2001 From: "min.tian" Date: Tue, 29 Oct 2024 14:55:35 +0800 Subject: [PATCH 099/327] increase timeout Signed-off-by: min.tian --- vectordb_bench/__init__.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/vectordb_bench/__init__.py b/vectordb_bench/__init__.py index f7664502f..f81c9bb3f 100644 --- a/vectordb_bench/__init__.py +++ b/vectordb_bench/__init__.py @@ -37,23 +37,24 @@ class config: K_DEFAULT = 100 # default return top k nearest neighbors during search CUSTOM_CONFIG_DIR = pathlib.Path(__file__).parent.joinpath("custom/custom_case.json") - CAPACITY_TIMEOUT_IN_SECONDS = 24 * 3600 # 24h - LOAD_TIMEOUT_DEFAULT = 2.5 * 3600 # 2.5h - LOAD_TIMEOUT_768D_1M = 2.5 * 3600 # 2.5h - LOAD_TIMEOUT_768D_10M = 25 * 3600 # 25h - LOAD_TIMEOUT_768D_100M = 250 * 3600 # 10.41d + CAPACITY_TIMEOUT_IN_SECONDS = 24 * 3600 # 24h + LOAD_TIMEOUT_DEFAULT = 24 * 3600 # 24h + LOAD_TIMEOUT_768D_1M = 24 * 3600 # 24h + LOAD_TIMEOUT_768D_10M = 240 * 3600 # 10d + LOAD_TIMEOUT_768D_100M = 2400 * 3600 # 100d - LOAD_TIMEOUT_1536D_500K = 2.5 * 3600 # 2.5h - LOAD_TIMEOUT_1536D_5M = 25 * 3600 # 25h + LOAD_TIMEOUT_1536D_500K = 24 * 3600 # 24h + LOAD_TIMEOUT_1536D_5M = 240 * 3600 # 10d - OPTIMIZE_TIMEOUT_DEFAULT = 30 * 60 # 30min - OPTIMIZE_TIMEOUT_768D_1M = 30 * 60 # 30min - OPTIMIZE_TIMEOUT_768D_10M = 5 * 3600 # 5h - OPTIMIZE_TIMEOUT_768D_100M = 50 * 3600 # 50h + OPTIMIZE_TIMEOUT_DEFAULT = 24 * 3600 # 24h + OPTIMIZE_TIMEOUT_768D_1M = 24 * 3600 # 24h + OPTIMIZE_TIMEOUT_768D_10M = 240 * 3600 # 10d + OPTIMIZE_TIMEOUT_768D_100M = 2400 * 3600 # 100d - OPTIMIZE_TIMEOUT_1536D_500K = 15 * 60 # 15min - OPTIMIZE_TIMEOUT_1536D_5M = 2.5 * 3600 # 2.5h + OPTIMIZE_TIMEOUT_1536D_500K = 24 * 3600 # 24h + OPTIMIZE_TIMEOUT_1536D_5M = 240 * 3600 # 10d + def display(self) -> str: tmp = [ i for i in inspect.getmembers(self) From 1d8b218e542bb749be637ca091ce840bba638c57 Mon Sep 17 00:00:00 2001 From: Sheharyar Ahmad Date: Tue, 29 Oct 2024 14:50:51 +0500 Subject: [PATCH 100/327] Binary Quantization Support for pgvector HNSW Algorithm (#389) * Added binary quantization support in pgvector hnsw * Parameterized search sql queries. Added distance operator used for reranking, and quantized vector fetch limit in CLI. * remove debug logs * update pgvectorhnsw command option name. * Binary quantization option added in frontend for pgvectorhnsw * remove redundant code * Refactored code * Removed hamming and jaccard distance options for full vectors. Moved reranking_metric to hnsw config class. * refactored code, removed duplicate code. * Reverted code changes for float input type. --- README.md | 2 +- vectordb_bench/backend/clients/api.py | 2 + .../backend/clients/pgvector/cli.py | 48 ++++- .../backend/clients/pgvector/config.py | 33 +++- .../backend/clients/pgvector/pgvector.py | 177 +++++++++++------- .../components/run_test/caseSelector.py | 6 + .../frontend/config/dbCaseConfigs.py | 48 ++++- vectordb_bench/models.py | 3 + 8 files changed, 246 insertions(+), 73 deletions(-) diff --git a/README.md b/README.md index 15767a980..bc550a7f2 100644 --- a/README.md +++ b/README.md @@ -118,7 +118,7 @@ Options: --m INTEGER hnsw m --ef-construction INTEGER hnsw ef-construction --ef-search INTEGER hnsw ef-search - --quantization-type [none|halfvec] + --quantization-type [none|bit|halfvec] quantization type for vectors --custom-case-name TEXT Custom case name i.e. PerformanceCase1536D50K --custom-case-description TEXT Custom name description diff --git a/vectordb_bench/backend/clients/api.py b/vectordb_bench/backend/clients/api.py index faa36712d..0c26fdd3b 100644 --- a/vectordb_bench/backend/clients/api.py +++ b/vectordb_bench/backend/clients/api.py @@ -10,6 +10,8 @@ class MetricType(str, Enum): L2 = "L2" COSINE = "COSINE" IP = "IP" + HAMMING = "HAMMING" + JACCARD = "JACCARD" class IndexType(str, Enum): diff --git a/vectordb_bench/backend/clients/pgvector/cli.py b/vectordb_bench/backend/clients/pgvector/cli.py index d5779caf6..cde125cc1 100644 --- a/vectordb_bench/backend/clients/pgvector/cli.py +++ b/vectordb_bench/backend/clients/pgvector/cli.py @@ -4,6 +4,8 @@ import os from pydantic import SecretStr +from vectordb_bench.backend.clients.api import MetricType + from ....cli.cli import ( CommonTypedDict, HNSWFlavor1, @@ -16,6 +18,13 @@ from vectordb_bench.backend.clients import DB + +def set_default_quantized_fetch_limit(ctx, param, value): + if ctx.params.get("reranking") and value is None: + # ef_search is the default value for quantized_fetch_limit as it's bound by ef_search. + return ctx.params["ef_search"] + return value + class PgVectorTypedDict(CommonTypedDict): user_name: Annotated[ str, click.option("--user-name", type=str, help="Db username", required=True) @@ -61,11 +70,45 @@ class PgVectorTypedDict(CommonTypedDict): Optional[str], click.option( "--quantization-type", - type=click.Choice(["none", "halfvec"]), + type=click.Choice(["none", "bit", "halfvec"]), help="quantization type for vectors", required=False, ), ] + reranking: Annotated[ + Optional[bool], + click.option( + "--reranking/--skip-reranking", + type=bool, + help="Enable reranking for HNSW search for binary quantization", + default=False, + ), + ] + reranking_metric: Annotated[ + Optional[str], + click.option( + "--reranking-metric", + type=click.Choice( + [metric.value for metric in MetricType if metric.value not in ["HAMMING", "JACCARD"]] + ), + help="Distance metric for reranking", + default="COSINE", + show_default=True, + ), + ] + quantized_fetch_limit: Annotated[ + Optional[int], + click.option( + "--quantized-fetch-limit", + type=int, + help="Limit of fetching quantized vector ranked by distance for reranking \ + -- bound by ef_search", + required=False, + callback=set_default_quantized_fetch_limit, + ) + ] + + class PgVectorIVFFlatTypedDict(PgVectorTypedDict, IVFFlatTypedDict): ... @@ -126,6 +169,9 @@ def PgVectorHNSW( maintenance_work_mem=parameters["maintenance_work_mem"], max_parallel_workers=parameters["max_parallel_workers"], quantization_type=parameters["quantization_type"], + reranking=parameters["reranking"], + reranking_metric=parameters["reranking_metric"], + quantized_fetch_limit=parameters["quantized_fetch_limit"], ), **parameters, ) diff --git a/vectordb_bench/backend/clients/pgvector/config.py b/vectordb_bench/backend/clients/pgvector/config.py index 857211234..31d832f13 100644 --- a/vectordb_bench/backend/clients/pgvector/config.py +++ b/vectordb_bench/backend/clients/pgvector/config.py @@ -65,6 +65,10 @@ def parse_metric(self) -> str: elif self.metric_type == MetricType.IP: return "halfvec_ip_ops" return "halfvec_cosine_ops" + elif self.quantization_type == "bit": + if self.metric_type == MetricType.JACCARD: + return "bit_jaccard_ops" + return "bit_hamming_ops" else: if self.metric_type == MetricType.L2: return "vector_l2_ops" @@ -73,11 +77,16 @@ def parse_metric(self) -> str: return "vector_cosine_ops" def parse_metric_fun_op(self) -> LiteralString: - if self.metric_type == MetricType.L2: - return "<->" - elif self.metric_type == MetricType.IP: - return "<#>" - return "<=>" + if self.quantization_type == "bit": + if self.metric_type == MetricType.JACCARD: + return "<%>" + return "<~>" + else: + if self.metric_type == MetricType.L2: + return "<->" + elif self.metric_type == MetricType.IP: + return "<#>" + return "<=>" def parse_metric_fun_str(self) -> str: if self.metric_type == MetricType.L2: @@ -85,6 +94,14 @@ def parse_metric_fun_str(self) -> str: elif self.metric_type == MetricType.IP: return "max_inner_product" return "cosine_distance" + + def parse_reranking_metric_fun_op(self) -> LiteralString: + if self.reranking_metric == MetricType.L2: + return "<->" + elif self.reranking_metric == MetricType.IP: + return "<#>" + return "<=>" + @abstractmethod def index_param(self) -> PgVectorIndexParam: @@ -195,6 +212,9 @@ class PgVectorHNSWConfig(PgVectorIndexConfig): maintenance_work_mem: Optional[str] = None max_parallel_workers: Optional[int] = None quantization_type: Optional[str] = None + reranking: Optional[bool] = None + quantized_fetch_limit: Optional[int] = None + reranking_metric: Optional[str] = None def index_param(self) -> PgVectorIndexParam: index_parameters = {"m": self.m, "ef_construction": self.ef_construction} @@ -214,6 +234,9 @@ def index_param(self) -> PgVectorIndexParam: def search_param(self) -> PgVectorSearchParam: return { "metric_fun_op": self.parse_metric_fun_op(), + "reranking": self.reranking, + "reranking_metric_fun_op": self.parse_reranking_metric_fun_op(), + "quantized_fetch_limit": self.quantized_fetch_limit, } def session_param(self) -> PgVectorSessionCommands: diff --git a/vectordb_bench/backend/clients/pgvector/pgvector.py b/vectordb_bench/backend/clients/pgvector/pgvector.py index 8123acf18..069b89381 100644 --- a/vectordb_bench/backend/clients/pgvector/pgvector.py +++ b/vectordb_bench/backend/clients/pgvector/pgvector.py @@ -11,7 +11,7 @@ from psycopg import Connection, Cursor, sql from ..api import VectorDB -from .config import PgVectorConfigDict, PgVectorIndexConfig +from .config import PgVectorConfigDict, PgVectorIndexConfig, PgVectorHNSWConfig log = logging.getLogger(__name__) @@ -87,6 +87,92 @@ def _create_connection(**kwargs) -> Tuple[Connection, Cursor]: assert cursor is not None, "Cursor is not initialized" return conn, cursor + + def _generate_search_query(self, filtered: bool=False) -> sql.Composed: + index_param = self.case_config.index_param() + reranking = self.case_config.search_param()["reranking"] + column_name = ( + sql.SQL("binary_quantize({0})").format(sql.Identifier("embedding")) + if index_param["quantization_type"] == "bit" + else sql.SQL("embedding") + ) + search_vector = ( + sql.SQL("binary_quantize({0})").format(sql.Placeholder()) + if index_param["quantization_type"] == "bit" + else sql.Placeholder() + ) + + # The following sections assume that the quantization_type value matches the quantization function name + if index_param["quantization_type"] != None: + if index_param["quantization_type"] == "bit" and reranking: + # Embeddings needs to be passed to binary_quantize function if quantization_type is bit + search_query = sql.Composed( + [ + sql.SQL( + """ + SELECT i.id + FROM ( + SELECT id, embedding {reranking_metric_fun_op} %s::vector AS distance + FROM public.{table_name} {where_clause} + ORDER BY {column_name}::{quantization_type}({dim}) + """ + ).format( + table_name=sql.Identifier(self.table_name), + column_name=column_name, + reranking_metric_fun_op=sql.SQL(self.case_config.search_param()["reranking_metric_fun_op"]), + quantization_type=sql.SQL(index_param["quantization_type"]), + dim=sql.Literal(self.dim), + where_clause=sql.SQL("WHERE id >= %s") if filtered else sql.SQL(""), + ), + sql.SQL(self.case_config.search_param()["metric_fun_op"]), + sql.SQL( + """ + {search_vector} + LIMIT {quantized_fetch_limit} + ) i + ORDER BY i.distance + LIMIT %s::int + """ + ).format( + search_vector=search_vector, + quantized_fetch_limit=sql.Literal( + self.case_config.search_param()["quantized_fetch_limit"] + ), + ), + ] + ) + else: + search_query = sql.Composed( + [ + sql.SQL( + "SELECT id FROM public.{table_name} {where_clause} ORDER BY {column_name}::{quantization_type}({dim}) " + ).format( + table_name=sql.Identifier(self.table_name), + column_name=column_name, + quantization_type=sql.SQL(index_param["quantization_type"]), + dim=sql.Literal(self.dim), + where_clause=sql.SQL("WHERE id >= %s") if filtered else sql.SQL(""), + ), + sql.SQL(self.case_config.search_param()["metric_fun_op"]), + sql.SQL(" {search_vector} LIMIT %s::int").format(search_vector=search_vector), + ] + ) + else: + search_query = sql.Composed( + [ + sql.SQL( + "SELECT id FROM public.{table_name} {where_clause} ORDER BY embedding " + ).format( + table_name=sql.Identifier(self.table_name), + where_clause=sql.SQL("WHERE id >= %s") if filtered else sql.SQL(""), + ), + sql.SQL(self.case_config.search_param()["metric_fun_op"]), + sql.SQL(" %s::vector LIMIT %s::int"), + ] + ) + + return search_query + @contextmanager def init(self) -> Generator[None, None, None]: @@ -112,63 +198,8 @@ def init(self) -> Generator[None, None, None]: self.cursor.execute(command) self.conn.commit() - index_param = self.case_config.index_param() - # The following sections assume that the quantization_type value matches the quantization function name - if index_param["quantization_type"] != None: - self._filtered_search = sql.Composed( - [ - sql.SQL( - "SELECT id FROM public.{table_name} WHERE id >= %s ORDER BY embedding::{quantization_type}({dim}) " - ).format( - table_name=sql.Identifier(self.table_name), - quantization_type=sql.SQL(index_param["quantization_type"]), - dim=sql.Literal(self.dim), - ), - sql.SQL(self.case_config.search_param()["metric_fun_op"]), - sql.SQL(" %s::{quantization_type}({dim}) LIMIT %s::int").format( - quantization_type=sql.SQL(index_param["quantization_type"]), - dim=sql.Literal(self.dim), - ), - ] - ) - else: - self._filtered_search = sql.Composed( - [ - sql.SQL( - "SELECT id FROM public.{table_name} WHERE id >= %s ORDER BY embedding " - ).format(table_name=sql.Identifier(self.table_name)), - sql.SQL(self.case_config.search_param()["metric_fun_op"]), - sql.SQL(" %s::vector LIMIT %s::int"), - ] - ) - - if index_param["quantization_type"] != None: - self._unfiltered_search = sql.Composed( - [ - sql.SQL( - "SELECT id FROM public.{table_name} ORDER BY embedding::{quantization_type}({dim}) " - ).format( - table_name=sql.Identifier(self.table_name), - quantization_type=sql.SQL(index_param["quantization_type"]), - dim=sql.Literal(self.dim), - ), - sql.SQL(self.case_config.search_param()["metric_fun_op"]), - sql.SQL(" %s::{quantization_type}({dim}) LIMIT %s::int").format( - quantization_type=sql.SQL(index_param["quantization_type"]), - dim=sql.Literal(self.dim), - ), - ] - ) - else: - self._unfiltered_search = sql.Composed( - [ - sql.SQL("SELECT id FROM public.{} ORDER BY embedding ").format( - sql.Identifier(self.table_name) - ), - sql.SQL(self.case_config.search_param()["metric_fun_op"]), - sql.SQL(" %s::vector LIMIT %s::int"), - ] - ) + self._filtered_search = self._generate_search_query(filtered=True) + self._unfiltered_search = self._generate_search_query() try: yield @@ -306,12 +337,17 @@ def _create_index(self): if index_param["quantization_type"] != None: index_create_sql = sql.SQL( """ - CREATE INDEX IF NOT EXISTS {index_name} ON public.{table_name} - USING {index_type} ((embedding::{quantization_type}({dim})) {embedding_metric}) + CREATE INDEX IF NOT EXISTS {index_name} ON public.{table_name} + USING {index_type} (({column_name}::{quantization_type}({dim})) {embedding_metric}) """ ).format( index_name=sql.Identifier(self._index_name), table_name=sql.Identifier(self.table_name), + column_name=( + sql.SQL("binary_quantize({0})").format(sql.Identifier("embedding")) + if index_param["quantization_type"] == "bit" + else sql.Identifier("embedding") + ), index_type=sql.Identifier(index_param["index_type"]), # This assumes that the quantization_type value matches the quantization function name quantization_type=sql.SQL(index_param["quantization_type"]), @@ -406,15 +442,28 @@ def search_embedding( assert self.conn is not None, "Connection is not initialized" assert self.cursor is not None, "Cursor is not initialized" + index_param = self.case_config.index_param() + search_param = self.case_config.search_param() q = np.asarray(query) if filters: gt = filters.get("id") - result = self.cursor.execute( + if index_param["quantization_type"] == "bit" and search_param["reranking"]: + result = self.cursor.execute( + self._filtered_search, (q, gt, q, k), prepare=True, binary=True + ) + else: + result = self.cursor.execute( self._filtered_search, (gt, q, k), prepare=True, binary=True - ) + ) + else: - result = self.cursor.execute( + if index_param["quantization_type"] == "bit" and search_param["reranking"]: + result = self.cursor.execute( + self._unfiltered_search, (q, q, k), prepare=True, binary=True + ) + else: + result = self.cursor.execute( self._unfiltered_search, (q, k), prepare=True, binary=True - ) + ) return [int(i[0]) for i in result.fetchall()] diff --git a/vectordb_bench/frontend/components/run_test/caseSelector.py b/vectordb_bench/frontend/components/run_test/caseSelector.py index 5597bbc61..b25618271 100644 --- a/vectordb_bench/frontend/components/run_test/caseSelector.py +++ b/vectordb_bench/frontend/components/run_test/caseSelector.py @@ -110,6 +110,12 @@ def caseConfigSetting(st, dbToCaseClusterConfigs, uiCaseItem: UICaseItem, active value=config.inputConfig["value"], help=config.inputHelp, ) + elif config.inputType == InputType.Bool: + caseConfig[config.label] = column.checkbox( + config.displayLabel if config.displayLabel else config.label.value, + value=config.inputConfig["value"], + help=config.inputHelp, + ) k += 1 if k == 0: columns[1].write("Auto") diff --git a/vectordb_bench/frontend/config/dbCaseConfigs.py b/vectordb_bench/frontend/config/dbCaseConfigs.py index 7736a4e03..3db547fef 100644 --- a/vectordb_bench/frontend/config/dbCaseConfigs.py +++ b/vectordb_bench/frontend/config/dbCaseConfigs.py @@ -3,7 +3,7 @@ from pydantic import BaseModel from vectordb_bench.backend.cases import CaseLabel, CaseType from vectordb_bench.backend.clients import DB -from vectordb_bench.backend.clients.api import IndexType +from vectordb_bench.backend.clients.api import IndexType, MetricType from vectordb_bench.frontend.components.custom.getCustomConfig import get_custom_configs from vectordb_bench.models import CaseConfig, CaseConfigParamType @@ -149,6 +149,7 @@ class InputType(IntEnum): Number = 20002 Option = 20003 Float = 20004 + Bool = 20005 class CaseConfigInput(BaseModel): @@ -819,7 +820,7 @@ class CaseConfigInput(BaseModel): label=CaseConfigParamType.quantizationType, inputType=InputType.Option, inputConfig={ - "options": ["none", "halfvec"], + "options": ["none", "bit", "halfvec"], }, isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) in [ @@ -865,6 +866,46 @@ class CaseConfigInput(BaseModel): }, ) +CaseConfigParamInput_reranking_PgVector = CaseConfigInput( + label=CaseConfigParamType.reranking, + inputType=InputType.Bool, + displayLabel="Enable Reranking", + inputHelp="Enable if you want to use reranking while performing \ + similarity search in binary quantization", + inputConfig={ + "value": False, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.quantizationType, None) + == "bit" +) + +CaseConfigParamInput_quantized_fetch_limit_PgVector = CaseConfigInput( + label=CaseConfigParamType.quantizedFetchLimit, + displayLabel="Quantized vector fetch limit", + inputHelp="Limit top-k vectors using the quantized vector comparison --bound by ef_search", + inputType=InputType.Number, + inputConfig={ + "min": 20, + "max": 1000, + "value": 200, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.quantizationType, None) + == "bit" and config.get(CaseConfigParamType.reranking, False) +) + + +CaseConfigParamInput_reranking_metric_PgVector = CaseConfigInput( + label=CaseConfigParamType.rerankingMetric, + inputType=InputType.Option, + inputConfig={ + "options": [ + metric.value for metric in MetricType if metric.value not in ["HAMMING", "JACCARD"] + ], + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.quantizationType, None) + == "bit" and config.get(CaseConfigParamType.reranking, False) +) + MilvusLoadConfig = [ CaseConfigParamInput_IndexType, CaseConfigParamInput_M, @@ -942,6 +983,9 @@ class CaseConfigInput(BaseModel): CaseConfigParamInput_QuantizationType_PgVector, CaseConfigParamInput_maintenance_work_mem_PgVector, CaseConfigParamInput_max_parallel_workers_PgVector, + CaseConfigParamInput_reranking_PgVector, + CaseConfigParamInput_reranking_metric_PgVector, + CaseConfigParamInput_quantized_fetch_limit_PgVector, ] PgVectoRSLoadingConfig = [ diff --git a/vectordb_bench/models.py b/vectordb_bench/models.py index d431479ed..9e064c564 100644 --- a/vectordb_bench/models.py +++ b/vectordb_bench/models.py @@ -47,6 +47,9 @@ class CaseConfigParamType(Enum): probes = "probes" quantizationType = "quantization_type" quantizationRatio = "quantization_ratio" + reranking = "reranking" + rerankingMetric = "reranking_metric" + quantizedFetchLimit = "quantized_fetch_limit" m = "m" nbits = "nbits" intermediate_graph_degree = "intermediate_graph_degree" From 7619acb969693d13c6c4dbd2a7db2d6a140424c9 Mon Sep 17 00:00:00 2001 From: "min.tian" Date: Wed, 30 Oct 2024 15:46:16 +0800 Subject: [PATCH 101/327] fix weaviate client bug Signed-off-by: min.tian --- vectordb_bench/backend/clients/weaviate_cloud/weaviate_cloud.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vectordb_bench/backend/clients/weaviate_cloud/weaviate_cloud.py b/vectordb_bench/backend/clients/weaviate_cloud/weaviate_cloud.py index 4c8bd12da..d8fde5f09 100644 --- a/vectordb_bench/backend/clients/weaviate_cloud/weaviate_cloud.py +++ b/vectordb_bench/backend/clients/weaviate_cloud/weaviate_cloud.py @@ -23,7 +23,7 @@ def __init__( **kwargs, ): """Initialize wrapper around the weaviate vector database.""" - db_config.update("auth_client_secret", weaviate.AuthApiKey(api_key=db_config.get("auth_client_secret"))) + db_config.update({"auth_client_secret": weaviate.AuthApiKey(api_key=db_config.get("auth_client_secret"))}) self.db_config = db_config self.case_config = db_case_config self.collection_name = collection_name From 36316d413d0595858f1edf0f9a3283cc0d27e565 Mon Sep 17 00:00:00 2001 From: acanadil Date: Thu, 31 Oct 2024 16:52:59 +0100 Subject: [PATCH 102/327] Fix code --- vectordb_bench/cli/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vectordb_bench/cli/cli.py b/vectordb_bench/cli/cli.py index edce758e1..44c13400d 100644 --- a/vectordb_bench/cli/cli.py +++ b/vectordb_bench/cli/cli.py @@ -479,7 +479,7 @@ def run( concurrency_duration=parameters["concurrency_duration"], num_concurrency=[int(s) for s in parameters["num_concurrency"]], ), - custom_case=parameters.get("custom_case", {}), + custom_case=get_custom_case_config(parameters), ), stages=parse_task_stages( ( From 6cb5898fe7a6d43f04921803abda8d7a1712e64e Mon Sep 17 00:00:00 2001 From: "min.tian" Date: Tue, 5 Nov 2024 10:02:14 +0800 Subject: [PATCH 103/327] remove older zillizcloud test results from leaderboard Signed-off-by: min.tian --- vectordb_bench/results/getLeaderboardData.py | 3 +++ vectordb_bench/results/leaderboard.json | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/vectordb_bench/results/getLeaderboardData.py b/vectordb_bench/results/getLeaderboardData.py index 222c568ab..aef024bdc 100644 --- a/vectordb_bench/results/getLeaderboardData.py +++ b/vectordb_bench/results/getLeaderboardData.py @@ -2,6 +2,7 @@ import ujson import pathlib from vectordb_bench.backend.cases import CaseType +from vectordb_bench.backend.clients import DB from vectordb_bench.frontend.config.dbPrices import DB_DBLABEL_TO_PRICE from vectordb_bench.interface import benchMarkRunner from vectordb_bench.models import ResultLabel, TestResult @@ -45,6 +46,8 @@ def main(): for d in test_result.results if d.task_config.case_config.case_id != CaseType.CapacityDim128 and d.task_config.case_config.case_id != CaseType.CapacityDim960 + if d.task_config.db != DB.ZillizCloud + or test_result.timestamp >= datetime(2024, 1, 1).timestamp() ] # compute qp$ diff --git a/vectordb_bench/results/leaderboard.json b/vectordb_bench/results/leaderboard.json index 2aa911cc2..3f6176017 100644 --- a/vectordb_bench/results/leaderboard.json +++ b/vectordb_bench/results/leaderboard.json @@ -1 +1 @@ -[{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":15.2269,"latency":861.8,"recall":0.9888,"label":1,"note":"","version":"","test_time":"2023-08","qp$":114368.53745044857},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":15.1749,"latency":774.3,"recall":0.989,"label":1,"note":"","version":"","test_time":"2023-08","qp$":113977.96786981013},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":27.6181,"latency":305.5,"recall":0.9999,"label":1,"note":"","version":"","test_time":"2023-08","qp$":207438.26413519715},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":11.2945,"latency":3611.2000000000003,"recall":0.996,"label":1,"note":"","version":"","test_time":"2023-08","qp$":84832.4640100146},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":17.3271,"latency":3774.7999999999997,"recall":0.9961,"label":1,"note":"","version":"","test_time":"2023-08","qp$":130143.04193615691},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":26.26,"latency":556.1,"recall":0.9999,"label":1,"note":"","version":"","test_time":"2023-08","qp$":197237.6382224077},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":100.6667,"latency":21.1,"recall":0.9909,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":101.1399,"latency":19.7,"recall":0.9907,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":52.2606,"latency":18.3,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":61.0661,"latency":49.8,"recall":0.9911,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":58.9326,"latency":44.6,"recall":0.9911,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":42.5977,"latency":54.9,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":536.0726,"latency":8.200000000000001,"recall":0.9728,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":467.179,"latency":7.0,"recall":0.9697,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":431.7512,"latency":8.3,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":274.5407,"latency":4.8999999999999995,"recall":0.9807,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":236.5672,"latency":10.3,"recall":0.981,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":309.4833,"latency":4.3,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":178.6585,"latency":13.700000000000001,"recall":0.9843,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":178.3732,"latency":15.0,"recall":0.9844,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":229.3526,"latency":12.5,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":1258.7043,"latency":4.8999999999999995,"recall":0.9799,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":1075.8776,"latency":5.3,"recall":0.98,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1494.8493,"latency":4.7,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":37.432,"latency":75.0,"recall":0.9975,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":37.0696,"latency":73.5,"recall":0.9976,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":81.1915,"latency":53.0,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":321.6048,"latency":13.4,"recall":0.989,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":22.1467,"latency":86.8,"recall":0.9972,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":303.2551,"latency":10.9,"recall":0.9876,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":21.5388,"latency":81.69999999999999,"recall":0.997,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":394.5418,"latency":8.3,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":37.878,"latency":45.0,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":180.2757,"latency":6.0,"recall":0.9942,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":179.0033,"latency":6.4,"recall":0.9943,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":526.8846,"latency":3.6,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":626.5243,"latency":6.2,"recall":0.9954,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":78.4227,"latency":25.3,"recall":0.9982,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":599.4213,"latency":6.6,"recall":0.9955,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":78.5351,"latency":26.3,"recall":0.9982,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":2098.2113,"latency":3.4,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":275.6292,"latency":10.0,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":10.6271,"latency":730.7,"recall":0.8898,"label":1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":10.8507,"latency":733.1999999999999,"recall":0.8897,"label":1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":75.7055,"latency":121.2,"recall":0.9999,"label":1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":0.8836,"latency":2523.0,"recall":0.8528,"label":1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":0.8937,"latency":3720.2000000000003,"recall":0.8525,"label":1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":1.2145,"latency":3622.3999999999996,"recall":0.7487,"label":1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":18.7634,"latency":153.70000000000002,"recall":0.8737,"label":1,"note":"","version":"","test_time":"2023-08","qp$":694226.515930113},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":18.3619,"latency":79.8,"recall":0.8741,"label":1,"note":"","version":"","test_time":"2023-08","qp$":679371.4285714285},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":25.2744,"latency":61.199999999999996,"recall":0.9979,"label":1,"note":"","version":"","test_time":"2023-08","qp$":935126.8242548818},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":261.798,"latency":23.099999999999998,"recall":0.9262,"label":1,"note":"","version":"","test_time":"2023-08","qp$":6455293.150684931},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":166.1851,"latency":23.900000000000002,"recall":0.9264,"label":1,"note":"","version":"","test_time":"2023-08","qp$":4097714.7945205485},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":121.7169,"latency":29.0,"recall":0.9693,"label":1,"note":"","version":"","test_time":"2023-08","qp$":3001238.6301369863},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":46.6189,"latency":43.1,"recall":0.8737,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1724851.3874614593},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":42.4856,"latency":44.0,"recall":0.8741,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1571923.5354573485},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":138.9479,"latency":26.200000000000003,"recall":0.9979,"label":1,"note":"","version":"","test_time":"2023-08","qp$":5140929.496402878},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":20.7437,"latency":75.80000000000001,"recall":0.9291,"label":1,"note":"","version":"","test_time":"2023-08","qp$":382960.6153846154},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":20.2993,"latency":76.5,"recall":0.9293,"label":1,"note":"","version":"","test_time":"2023-08","qp$":374756.30769230763},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":26.4719,"latency":67.0,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":488712.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":8.6675,"latency":180.2,"recall":0.8369,"label":1,"note":"","version":"","test_time":"2023-08","qp$":160015.38461538462},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":7.8121,"latency":167.7,"recall":0.8369,"label":1,"note":"","version":"","test_time":"2023-08","qp$":144223.3846153846},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":16.869,"latency":87.8,"recall":0.9814,"label":1,"note":"","version":"","test_time":"2023-08","qp$":311427.6923076923},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":365.0835,"latency":23.599999999999998,"recall":0.945,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1125257.3630136987},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":325.5271,"latency":25.1,"recall":0.9452,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1003336.9520547946},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":596.7942,"latency":24.2,"recall":0.9693,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1839434.1780821919},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":303.2044,"latency":27.400000000000002,"recall":0.9246,"label":1,"note":"","version":"","test_time":"2023-08","qp$":934534.1095890411},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":136.0345,"latency":31.9,"recall":0.9244,"label":1,"note":"","version":"","test_time":"2023-08","qp$":419284.4178082192},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":66.7221,"latency":42.1,"recall":0.963,"label":1,"note":"","version":"","test_time":"2023-08","qp$":205650.30821917808},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":131.2549,"latency":30.200000000000003,"recall":0.9867,"label":1,"note":"","version":"","test_time":"2023-08","qp$":606569.4993581515},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":127.9337,"latency":30.099999999999998,"recall":0.9869,"label":1,"note":"","version":"","test_time":"2023-08","qp$":591221.2066752247},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":595.8462,"latency":23.400000000000002,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":2753589.627727856},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":228.4,"latency":22.2,"recall":0.9348,"label":1,"note":"","version":"","test_time":"2023-08","qp$":5631780.821917809},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":181.5,"latency":26.1,"recall":0.9345,"label":1,"note":"","version":"","test_time":"2023-08","qp$":4475342.465753425},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":205.7,"latency":24.2,"recall":0.9586,"label":1,"note":"","version":"","test_time":"2023-08","qp$":5072054.794520548},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":67.63,"latency":36.0,"recall":0.8064,"label":1,"note":"","version":"","test_time":"2023-08","qp$":2502240.4933196297},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":63.35,"latency":38.4,"recall":0.8065,"label":1,"note":"","version":"","test_time":"2023-08","qp$":2343884.892086331},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":176.7,"latency":27.599999999999998,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":6537718.396711202},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":15.33,"latency":84.9,"recall":0.8064,"label":1,"note":"","version":"","test_time":"2023-08","qp$":567194.2446043165},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":15.13,"latency":86.7,"recall":0.8065,"label":1,"note":"","version":"","test_time":"2023-08","qp$":559794.4501541625},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":17.41,"latency":74.30000000000001,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":644152.1068859198},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":16.34,"latency":88.7,"recall":0.879,"label":1,"note":"","version":"","test_time":"2023-08","qp$":301661.53846153844},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":10.45,"latency":126.8,"recall":0.8208,"label":1,"note":"","version":"","test_time":"2023-08","qp$":192923.0769230769},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":16.18,"latency":87.5,"recall":0.8793,"label":1,"note":"","version":"","test_time":"2023-08","qp$":298707.69230769225},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":9.8,"latency":130.89999999999998,"recall":0.8212,"label":1,"note":"","version":"","test_time":"2023-08","qp$":180923.0769230769},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":36.11,"latency":55.1,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":666646.1538461539},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":14.84,"latency":92.7,"recall":0.96,"label":1,"note":"","version":"","test_time":"2023-08","qp$":273969.23076923075},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":322.7,"latency":26.4,"recall":0.9478,"label":1,"note":"","version":"","test_time":"2023-08","qp$":994623.287671233},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":265.5,"latency":26.9,"recall":0.9332,"label":1,"note":"","version":"","test_time":"2023-08","qp$":818321.9178082192},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":303.8,"latency":27.3,"recall":0.9478,"label":1,"note":"","version":"","test_time":"2023-08","qp$":936369.8630136987},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":180.2,"latency":28.2,"recall":0.9335,"label":1,"note":"","version":"","test_time":"2023-08","qp$":555410.9589041097},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":730.7,"latency":24.6,"recall":0.9586,"label":1,"note":"","version":"","test_time":"2023-08","qp$":2252157.5342465756},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":104.3,"latency":31.7,"recall":0.9563,"label":1,"note":"","version":"","test_time":"2023-08","qp$":321472.602739726},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":147.7,"latency":35.3,"recall":0.9707,"label":1,"note":"","version":"","test_time":"2023-08","qp$":682567.3940949935},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":782.5,"latency":25.9,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":3616174.5827984596},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":537.4975,"latency":18.9,"recall":0.8903,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1356936.1851332397},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":372.0466,"latency":17.8,"recall":0.8904,"label":1,"note":"","version":"","test_time":"2023-08","qp$":939248.0785413746},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1156.2898,"latency":14.4,"recall":0.9989,"label":1,"note":"","version":"","test_time":"2023-08","qp$":2919104.684431978},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":110.248,"latency":69.0,"recall":0.898,"label":1,"note":"","version":"","test_time":"2023-08","qp$":278325.94670406735},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":87.2601,"latency":27.799999999999997,"recall":0.898,"label":1,"note":"","version":"","test_time":"2023-08","qp$":220291.97755960727},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":125.7846,"latency":23.099999999999998,"recall":0.975,"label":1,"note":"","version":"","test_time":"2023-08","qp$":317548.77980364655},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":240.7209,"latency":17.4,"recall":0.8887,"label":1,"note":"","version":"","test_time":"2023-08","qp$":3038552.734922861},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":189.4399,"latency":17.5,"recall":0.8889,"label":1,"note":"","version":"","test_time":"2023-08","qp$":2391246.98457223},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":313.5116,"latency":16.1,"recall":0.9999,"label":1,"note":"","version":"","test_time":"2023-08","qp$":3957369.424964937},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":78.7196,"latency":49.4,"recall":0.9203,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1707172.048192771},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":68.3111,"latency":35.5,"recall":0.9202,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1481445.5421686745},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":210.2147,"latency":26.8,"recall":0.9996,"label":1,"note":"","version":"","test_time":"2023-08","qp$":4558873.012048192},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":188.6436,"latency":917.5,"recall":0.9175,"label":1,"note":"","version":"","test_time":"2023-08","qp$":2381195.5119214584},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":155.6991,"latency":917.1,"recall":0.9171,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1965346.283309958},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":445.3289,"latency":14.1,"recall":0.9999,"label":1,"note":"","version":"","test_time":"2023-08","qp$":5621262.412342216},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":633.6033,"latency":24.6,"recall":0.919,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1599559.5231416551},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":95.5682,"latency":58.7,"recall":0.9463,"label":1,"note":"","version":"","test_time":"2023-08","qp$":241266.14305750353},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":434.4062,"latency":17.4,"recall":0.9181,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1096677.643758766},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":80.3192,"latency":26.599999999999998,"recall":0.9462,"label":1,"note":"","version":"","test_time":"2023-08","qp$":202769.3688639551},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":1509.3293,"latency":18.5,"recall":0.9995,"label":1,"note":"","version":"","test_time":"2023-08","qp$":3810368.4992987383},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":166.7252,"latency":19.599999999999998,"recall":0.9988,"label":1,"note":"","version":"","test_time":"2023-08","qp$":420905.1332398317},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":67.9121,"latency":179.5,"recall":0.9909,"label":1,"note":"","version":"","test_time":"2023-08","qp$":7499.495705521471},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.7636,"latency":1921.3,"recall":0.9908,"label":1,"note":"","version":"","test_time":"2023-08","qp$":84.32392638036809},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":32.0,"latency":124.5,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":3533.7423312883434},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":63.1365,"latency":145.7,"recall":0.991,"label":1,"note":"","version":"","test_time":"2023-08","qp$":22504.09900990099},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.7512,"latency":1983.8,"recall":0.9908,"label":1,"note":"","version":"","test_time":"2023-08","qp$":267.7544554455445},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":30.1358,"latency":129.8,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":10741.473267326734},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":46.8622,"latency":123.0,"recall":0.9957,"label":1,"note":"","version":"","test_time":"2023-08","qp$":16703.358415841583},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1.8388,"latency":1063.5,"recall":0.9957,"label":1,"note":"","version":"","test_time":"2023-08","qp$":655.4138613861386},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":45.0666,"latency":109.2,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":16063.342574257427},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":43.5017,"latency":228.7,"recall":0.9957,"label":1,"note":"","version":"","test_time":"2023-08","qp$":4803.8687116564415},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1.5668,"latency":1114.4,"recall":0.9957,"label":1,"note":"","version":"","test_time":"2023-08","qp$":173.02085889570552},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":41.5443,"latency":159.0,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":4587.714110429448},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":330.0144,"latency":9.0,"recall":0.9507,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":7472024.150943397},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":271.6585,"latency":10.1,"recall":0.9678,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":6150758.490566038},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":216.5226,"latency":12.9,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":4902398.490566038},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":123.9553,"latency":23.0,"recall":0.971,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":1403267.5471698113},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":59.1479,"latency":44.5,"recall":0.9906,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":669598.8679245282},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":40.999,"latency":55.300000000000004,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":464139.62264150946},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":579.9416,"latency":9.4,"recall":0.9213,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":6565376.603773585},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":425.2529,"latency":11.299999999999999,"recall":0.9686,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":4814183.773584905},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":397.0539,"latency":13.799999999999999,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":4494949.811320755},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":516.27,"latency":7.0,"recall":0.9463,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":11689132.075471697},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":354.8416,"latency":10.0,"recall":0.9802,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":8034149.433962264},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":427.5229,"latency":8.7,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":9679763.773584906},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":2884.689,"latency":5.3,"recall":0.8801,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":8164214.150943396},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":1689.5799,"latency":6.6,"recall":0.9493,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":4781829.905660377},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1517.6792,"latency":10.0,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":4295318.4905660385},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":822.5318,"latency":5.6,"recall":0.9294,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":2327920.1886792453},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":378.9146,"latency":10.3,"recall":0.9758,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":1072399.8113207547},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":218.6854,"latency":16.2,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":618920.9433962263},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":297.5,"latency":7.2,"recall":0.974,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":6735849.0566037735},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":228.3,"latency":10.6,"recall":0.994,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":5169056.603773585},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":584.0,"latency":4.6,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":13222641.509433962},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2023.6","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":1871.0,"latency":7.0,"recall":0.9602,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":5295283.018867925},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":556.7,"latency":6.7,"recall":0.9723,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":1575566.0377358492},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1583.0,"latency":6.8,"recall":0.9836,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":4480188.679245283},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":294.3,"latency":10.9,"recall":0.9939,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":832924.5283018869},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":2345.0,"latency":8.9,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":6636792.452830189},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2023.6","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":295.6,"latency":12.3,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":836603.7735849057},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":143.0,"latency":33.5,"recall":0.9818,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":3237735.8490566034},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":106.0,"latency":20.7,"recall":0.9887,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":2400000.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":189.0,"latency":11.6,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":4279245.2830188675},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2023.6","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":379.9721,"latency":12.4,"recall":0.982,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":4301570.9433962265},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":71.74,"latency":50.8,"recall":0.9883,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":812150.9433962264},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":287.0,"latency":14.9,"recall":0.9865,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":3249056.603773585},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":34.6654,"latency":64.69999999999999,"recall":0.9961,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":392438.4905660377},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":412.0,"latency":10.3,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":4664150.9433962265},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2023.6","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":42.169,"latency":46.800000000000004,"recall":1.0,"label":1,"note":"","version":"v2023.6","test_time":"2023-08","qp$":477384.9056603773},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":5115.5303,"latency":8.7,"recall":0.9469,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":14477915.943396227},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":3685.0767,"latency":9.1,"recall":0.9736,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":10429462.358490566},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":4742.1617,"latency":9.1,"recall":0.9936,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":13421212.358490566},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":6054.4428,"latency":8.1,"recall":0.9155,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":17135215.471698113},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":4104.2598,"latency":10.6,"recall":0.9506,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":11615829.62264151},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":4252.1267,"latency":9.2,"recall":0.9964,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":12034320.849056603},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":1685.3091,"latency":13.299999999999999,"recall":0.9718,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":4769742.735849056},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":769.8991,"latency":10.7,"recall":0.9884,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":2178959.716981132},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":945.4061,"latency":10.9,"recall":0.9941,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":2675677.641509434},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":2214.9028,"latency":8.4,"recall":0.9249,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":6268592.830188679},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":827.975,"latency":12.0,"recall":0.9692,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":2343325.4716981133},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":776.9454,"latency":11.4,"recall":0.9966,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":2198902.075471698},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":269.5464,"latency":9.799999999999999,"recall":0.9776,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":6102937.358490566},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":240.0363,"latency":10.6,"recall":0.9822,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":5434784.150943397},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":218.0627,"latency":11.4,"recall":0.9936,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":4937268.679245283},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":392.8825,"latency":6.8999999999999995,"recall":0.9581,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":8895452.830188679},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":343.8204,"latency":8.4,"recall":0.968,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":7784612.8301886795},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":216.6773,"latency":10.1,"recall":0.9968,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":4905901.132075472},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":503.2284,"latency":9.0,"recall":0.9677,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":5696925.2830188675},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":413.3232,"latency":9.6,"recall":0.981,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":4679130.566037735},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":425.5492,"latency":10.200000000000001,"recall":0.9936,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":4817538.113207547},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":789.1229,"latency":5.6,"recall":0.9396,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":8933466.792452829},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":571.4257,"latency":7.7,"recall":0.9668,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":6468970.188679245},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":411.7653,"latency":9.1,"recall":0.9968,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":4661493.96226415},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":98.0448,"latency":16.1,"recall":0.9803,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":1109941.1320754716},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":58.3152,"latency":28.0,"recall":0.9891,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":660172.0754716981},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":46.8304,"latency":28.6,"recall":0.9941,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":530155.4716981131},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":170.5693,"latency":8.9,"recall":0.9605,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":1930973.2075471696},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":94.7766,"latency":19.599999999999998,"recall":0.9843,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":1072942.641509434},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":44.8695,"latency":30.900000000000002,"recall":0.9966,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":507956.60377358494},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":722.0315,"latency":7.7,"recall":0.976,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":16347883.018867927},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":467.5795,"latency":8.8,"recall":0.9898,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":10586705.660377359},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":975.2503,"latency":8.200000000000001,"recall":0.9936,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":22081138.86792453},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":873.3712,"latency":6.7,"recall":0.9477,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":19774442.264150944},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":544.6203,"latency":8.4,"recall":0.977,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":12331025.660377359},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":930.9164,"latency":9.6,"recall":0.9968,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":21077352.452830188},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0}] \ No newline at end of file +[{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":15.2269,"latency":861.8,"recall":0.9888,"label":1,"note":"","version":"","test_time":"2023-08","qp$":114368.53745044857},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":15.1749,"latency":774.3,"recall":0.989,"label":1,"note":"","version":"","test_time":"2023-08","qp$":113977.96786981013},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":27.6181,"latency":305.5,"recall":0.9999,"label":1,"note":"","version":"","test_time":"2023-08","qp$":207438.26413519715},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":11.2945,"latency":3611.2000000000003,"recall":0.996,"label":1,"note":"","version":"","test_time":"2023-08","qp$":84832.4640100146},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":17.3271,"latency":3774.7999999999997,"recall":0.9961,"label":1,"note":"","version":"","test_time":"2023-08","qp$":130143.04193615691},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":26.26,"latency":556.1,"recall":0.9999,"label":1,"note":"","version":"","test_time":"2023-08","qp$":197237.6382224077},{"db":"ElasticCloud","db_label":"upTo2.5c8g","db_name":"ElasticCloud-upTo2.5c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":100.6667,"latency":21.1,"recall":0.9909,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":101.1399,"latency":19.7,"recall":0.9907,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":52.2606,"latency":18.3,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":61.0661,"latency":49.8,"recall":0.9911,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":58.9326,"latency":44.6,"recall":0.9911,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":42.5977,"latency":54.9,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":536.0726,"latency":8.200000000000001,"recall":0.9728,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":467.179,"latency":7.0,"recall":0.9697,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":431.7512,"latency":8.3,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":274.5407,"latency":4.8999999999999995,"recall":0.9807,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":236.5672,"latency":10.3,"recall":0.981,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":309.4833,"latency":4.3,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":178.6585,"latency":13.700000000000001,"recall":0.9843,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":178.3732,"latency":15.0,"recall":0.9844,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":229.3526,"latency":12.5,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":1258.7043,"latency":4.8999999999999995,"recall":0.9799,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":1075.8776,"latency":5.3,"recall":0.98,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1494.8493,"latency":4.7,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":37.432,"latency":75.0,"recall":0.9975,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":37.0696,"latency":73.5,"recall":0.9976,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":81.1915,"latency":53.0,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-disk","db_name":"Milvus-2c8g-disk-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":321.6048,"latency":13.4,"recall":0.989,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":22.1467,"latency":86.8,"recall":0.9972,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":303.2551,"latency":10.9,"recall":0.9876,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":21.5388,"latency":81.69999999999999,"recall":0.997,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":394.5418,"latency":8.3,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"4c16g-disk","db_name":"Milvus-4c16g-disk-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":37.878,"latency":45.0,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":180.2757,"latency":6.0,"recall":0.9942,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":179.0033,"latency":6.4,"recall":0.9943,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":526.8846,"latency":3.6,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"2c8g-hnsw","db_name":"Milvus-2c8g-hnsw-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":626.5243,"latency":6.2,"recall":0.9954,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":78.4227,"latency":25.3,"recall":0.9982,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":599.4213,"latency":6.6,"recall":0.9955,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":78.5351,"latency":26.3,"recall":0.9982,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":2098.2113,"latency":3.4,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"Milvus","db_label":"16c64g-hnsw","db_name":"Milvus-16c64g-hnsw-v2.2.12","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":275.6292,"latency":10.0,"recall":1.0,"label":1,"note":"standalone mode","version":"v2.2.12","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":10.6271,"latency":730.7,"recall":0.8898,"label":1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":10.8507,"latency":733.1999999999999,"recall":0.8897,"label":1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":75.7055,"latency":121.2,"recall":0.9999,"label":1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":0.8836,"latency":2523.0,"recall":0.8528,"label":1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":0.8937,"latency":3720.2000000000003,"recall":0.8525,"label":1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":1.2145,"latency":3622.3999999999996,"recall":0.7487,"label":1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"PgVector","db_label":"2c8g","db_name":"PgVector-2c8g","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-2,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":18.7634,"latency":153.70000000000002,"recall":0.8737,"label":1,"note":"","version":"","test_time":"2023-08","qp$":694226.515930113},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":18.3619,"latency":79.8,"recall":0.8741,"label":1,"note":"","version":"","test_time":"2023-08","qp$":679371.4285714285},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":25.2744,"latency":61.199999999999996,"recall":0.9979,"label":1,"note":"","version":"","test_time":"2023-08","qp$":935126.8242548818},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":261.798,"latency":23.099999999999998,"recall":0.9262,"label":1,"note":"","version":"","test_time":"2023-08","qp$":6455293.150684931},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":166.1851,"latency":23.900000000000002,"recall":0.9264,"label":1,"note":"","version":"","test_time":"2023-08","qp$":4097714.7945205485},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":121.7169,"latency":29.0,"recall":0.9693,"label":1,"note":"","version":"","test_time":"2023-08","qp$":3001238.6301369863},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":46.6189,"latency":43.1,"recall":0.8737,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1724851.3874614593},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":42.4856,"latency":44.0,"recall":0.8741,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1571923.5354573485},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":138.9479,"latency":26.200000000000003,"recall":0.9979,"label":1,"note":"","version":"","test_time":"2023-08","qp$":5140929.496402878},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":20.7437,"latency":75.80000000000001,"recall":0.9291,"label":1,"note":"","version":"","test_time":"2023-08","qp$":382960.6153846154},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":20.2993,"latency":76.5,"recall":0.9293,"label":1,"note":"","version":"","test_time":"2023-08","qp$":374756.30769230763},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":26.4719,"latency":67.0,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":488712.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":8.6675,"latency":180.2,"recall":0.8369,"label":1,"note":"","version":"","test_time":"2023-08","qp$":160015.38461538462},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":7.8121,"latency":167.7,"recall":0.8369,"label":1,"note":"","version":"","test_time":"2023-08","qp$":144223.3846153846},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":16.869,"latency":87.8,"recall":0.9814,"label":1,"note":"","version":"","test_time":"2023-08","qp$":311427.6923076923},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":365.0835,"latency":23.599999999999998,"recall":0.945,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1125257.3630136987},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":325.5271,"latency":25.1,"recall":0.9452,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1003336.9520547946},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":596.7942,"latency":24.2,"recall":0.9693,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1839434.1780821919},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":303.2044,"latency":27.400000000000002,"recall":0.9246,"label":1,"note":"","version":"","test_time":"2023-08","qp$":934534.1095890411},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":136.0345,"latency":31.9,"recall":0.9244,"label":1,"note":"","version":"","test_time":"2023-08","qp$":419284.4178082192},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":66.7221,"latency":42.1,"recall":0.963,"label":1,"note":"","version":"","test_time":"2023-08","qp$":205650.30821917808},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":131.2549,"latency":30.200000000000003,"recall":0.9867,"label":1,"note":"","version":"","test_time":"2023-08","qp$":606569.4993581515},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":127.9337,"latency":30.099999999999998,"recall":0.9869,"label":1,"note":"","version":"","test_time":"2023-08","qp$":591221.2066752247},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":595.8462,"latency":23.400000000000002,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":2753589.627727856},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":228.4,"latency":22.2,"recall":0.9348,"label":1,"note":"","version":"","test_time":"2023-08","qp$":5631780.821917809},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":181.5,"latency":26.1,"recall":0.9345,"label":1,"note":"","version":"","test_time":"2023-08","qp$":4475342.465753425},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":205.7,"latency":24.2,"recall":0.9586,"label":1,"note":"","version":"","test_time":"2023-08","qp$":5072054.794520548},{"db":"Pinecone","db_label":"p2.x1","db_name":"Pinecone-p2.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":67.63,"latency":36.0,"recall":0.8064,"label":1,"note":"","version":"","test_time":"2023-08","qp$":2502240.4933196297},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":63.35,"latency":38.4,"recall":0.8065,"label":1,"note":"","version":"","test_time":"2023-08","qp$":2343884.892086331},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":176.7,"latency":27.599999999999998,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":6537718.396711202},{"db":"Pinecone","db_label":"p1.x1","db_name":"Pinecone-p1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":15.33,"latency":84.9,"recall":0.8064,"label":1,"note":"","version":"","test_time":"2023-08","qp$":567194.2446043165},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":15.13,"latency":86.7,"recall":0.8065,"label":1,"note":"","version":"","test_time":"2023-08","qp$":559794.4501541625},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":17.41,"latency":74.30000000000001,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":644152.1068859198},{"db":"Pinecone","db_label":"s1.x1","db_name":"Pinecone-s1.x1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":16.34,"latency":88.7,"recall":0.879,"label":1,"note":"","version":"","test_time":"2023-08","qp$":301661.53846153844},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":10.45,"latency":126.8,"recall":0.8208,"label":1,"note":"","version":"","test_time":"2023-08","qp$":192923.0769230769},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":16.18,"latency":87.5,"recall":0.8793,"label":1,"note":"","version":"","test_time":"2023-08","qp$":298707.69230769225},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":9.8,"latency":130.89999999999998,"recall":0.8212,"label":1,"note":"","version":"","test_time":"2023-08","qp$":180923.0769230769},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":36.11,"latency":55.1,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":666646.1538461539},{"db":"Pinecone","db_label":"s1.x1-2node","db_name":"Pinecone-s1.x1-2node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":14.84,"latency":92.7,"recall":0.96,"label":1,"note":"","version":"","test_time":"2023-08","qp$":273969.23076923075},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":322.7,"latency":26.4,"recall":0.9478,"label":1,"note":"","version":"","test_time":"2023-08","qp$":994623.287671233},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":265.5,"latency":26.9,"recall":0.9332,"label":1,"note":"","version":"","test_time":"2023-08","qp$":818321.9178082192},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":303.8,"latency":27.3,"recall":0.9478,"label":1,"note":"","version":"","test_time":"2023-08","qp$":936369.8630136987},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":180.2,"latency":28.2,"recall":0.9335,"label":1,"note":"","version":"","test_time":"2023-08","qp$":555410.9589041097},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":730.7,"latency":24.6,"recall":0.9586,"label":1,"note":"","version":"","test_time":"2023-08","qp$":2252157.5342465756},{"db":"Pinecone","db_label":"p2.x1-8node","db_name":"Pinecone-p2.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":104.3,"latency":31.7,"recall":0.9563,"label":1,"note":"","version":"","test_time":"2023-08","qp$":321472.602739726},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":147.7,"latency":35.3,"recall":0.9707,"label":1,"note":"","version":"","test_time":"2023-08","qp$":682567.3940949935},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":782.5,"latency":25.9,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":3616174.5827984596},{"db":"Pinecone","db_label":"p1.x1-8node","db_name":"Pinecone-p1.x1-8node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":537.4975,"latency":18.9,"recall":0.8903,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1356936.1851332397},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":372.0466,"latency":17.8,"recall":0.8904,"label":1,"note":"","version":"","test_time":"2023-08","qp$":939248.0785413746},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":1156.2898,"latency":14.4,"recall":0.9989,"label":1,"note":"","version":"","test_time":"2023-08","qp$":2919104.684431978},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":110.248,"latency":69.0,"recall":0.898,"label":1,"note":"","version":"","test_time":"2023-08","qp$":278325.94670406735},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":87.2601,"latency":27.799999999999997,"recall":0.898,"label":1,"note":"","version":"","test_time":"2023-08","qp$":220291.97755960727},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":125.7846,"latency":23.099999999999998,"recall":0.975,"label":1,"note":"","version":"","test_time":"2023-08","qp$":317548.77980364655},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":240.7209,"latency":17.4,"recall":0.8887,"label":1,"note":"","version":"","test_time":"2023-08","qp$":3038552.734922861},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":189.4399,"latency":17.5,"recall":0.8889,"label":1,"note":"","version":"","test_time":"2023-08","qp$":2391246.98457223},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":313.5116,"latency":16.1,"recall":0.9999,"label":1,"note":"","version":"","test_time":"2023-08","qp$":3957369.424964937},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":78.7196,"latency":49.4,"recall":0.9203,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1707172.048192771},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":68.3111,"latency":35.5,"recall":0.9202,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1481445.5421686745},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":210.2147,"latency":26.8,"recall":0.9996,"label":1,"note":"","version":"","test_time":"2023-08","qp$":4558873.012048192},{"db":"QdrantCloud","db_label":"2c8g-1node","db_name":"QdrantCloud-2c8g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":188.6436,"latency":917.5,"recall":0.9175,"label":1,"note":"","version":"","test_time":"2023-08","qp$":2381195.5119214584},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":155.6991,"latency":917.1,"recall":0.9171,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1965346.283309958},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":445.3289,"latency":14.1,"recall":0.9999,"label":1,"note":"","version":"","test_time":"2023-08","qp$":5621262.412342216},{"db":"QdrantCloud","db_label":"4c16g-1node","db_name":"QdrantCloud-4c16g-1node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":633.6033,"latency":24.6,"recall":0.919,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1599559.5231416551},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":95.5682,"latency":58.7,"recall":0.9463,"label":1,"note":"","version":"","test_time":"2023-08","qp$":241266.14305750353},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":434.4062,"latency":17.4,"recall":0.9181,"label":1,"note":"","version":"","test_time":"2023-08","qp$":1096677.643758766},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":80.3192,"latency":26.599999999999998,"recall":0.9462,"label":1,"note":"","version":"","test_time":"2023-08","qp$":202769.3688639551},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":1509.3293,"latency":18.5,"recall":0.9995,"label":1,"note":"","version":"","test_time":"2023-08","qp$":3810368.4992987383},{"db":"QdrantCloud","db_label":"4c16g-5node","db_name":"QdrantCloud-4c16g-5node","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":166.7252,"latency":19.599999999999998,"recall":0.9988,"label":1,"note":"","version":"","test_time":"2023-08","qp$":420905.1332398317},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":67.9121,"latency":179.5,"recall":0.9909,"label":1,"note":"","version":"","test_time":"2023-08","qp$":7499.495705521471},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.7636,"latency":1921.3,"recall":0.9908,"label":1,"note":"","version":"","test_time":"2023-08","qp$":84.32392638036809},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":32.0,"latency":124.5,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":3533.7423312883434},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"sandbox","db_name":"WeaviateCloud-sandbox","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":63.1365,"latency":145.7,"recall":0.991,"label":1,"note":"","version":"","test_time":"2023-08","qp$":22504.09900990099},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":0.7512,"latency":1983.8,"recall":0.9908,"label":1,"note":"","version":"","test_time":"2023-08","qp$":267.7544554455445},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":30.1358,"latency":129.8,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":10741.473267326734},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":46.8622,"latency":123.0,"recall":0.9957,"label":1,"note":"","version":"","test_time":"2023-08","qp$":16703.358415841583},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1.8388,"latency":1063.5,"recall":0.9957,"label":1,"note":"","version":"","test_time":"2023-08","qp$":655.4138613861386},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":45.0666,"latency":109.2,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":16063.342574257427},{"db":"WeaviateCloud","db_label":"standard","db_name":"WeaviateCloud-standard","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":43.5017,"latency":228.7,"recall":0.9957,"label":1,"note":"","version":"","test_time":"2023-08","qp$":4803.8687116564415},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":1.5668,"latency":1114.4,"recall":0.9957,"label":1,"note":"","version":"","test_time":"2023-08","qp$":173.02085889570552},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":41.5443,"latency":159.0,"recall":1.0,"label":1,"note":"","version":"","test_time":"2023-08","qp$":4587.714110429448},{"db":"WeaviateCloud","db_label":"bus_crit","db_name":"WeaviateCloud-bus_crit","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"","test_time":"2023-08","qp$":0.0},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":5115.5303,"latency":8.7,"recall":0.9469,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":14477915.943396227},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":3685.0767,"latency":9.1,"recall":0.9736,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":10429462.358490566},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":4742.1617,"latency":9.1,"recall":0.9936,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":13421212.358490566},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":6054.4428,"latency":8.1,"recall":0.9155,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":17135215.471698113},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":4104.2598,"latency":10.6,"recall":0.9506,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":11615829.62264151},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":4252.1267,"latency":9.2,"recall":0.9964,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":12034320.849056603},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":1685.3091,"latency":13.299999999999999,"recall":0.9718,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":4769742.735849056},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":769.8991,"latency":10.7,"recall":0.9884,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":2178959.716981132},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":945.4061,"latency":10.9,"recall":0.9941,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":2675677.641509434},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":2214.9028,"latency":8.4,"recall":0.9249,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":6268592.830188679},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":827.975,"latency":12.0,"recall":0.9692,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":2343325.4716981133},{"db":"ZillizCloud","db_label":"8cu-perf","db_name":"ZillizCloud-8cu-perf-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":776.9454,"latency":11.4,"recall":0.9966,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":2198902.075471698},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":269.5464,"latency":9.799999999999999,"recall":0.9776,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":6102937.358490566},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":240.0363,"latency":10.6,"recall":0.9822,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":5434784.150943397},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":218.0627,"latency":11.4,"recall":0.9936,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":4937268.679245283},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":392.8825,"latency":6.8999999999999995,"recall":0.9581,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":8895452.830188679},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":343.8204,"latency":8.4,"recall":0.968,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":7784612.8301886795},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":216.6773,"latency":10.1,"recall":0.9968,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":4905901.132075472},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":503.2284,"latency":9.0,"recall":0.9677,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":5696925.2830188675},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":413.3232,"latency":9.6,"recall":0.981,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":4679130.566037735},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":425.5492,"latency":10.200000000000001,"recall":0.9936,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":4817538.113207547},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":789.1229,"latency":5.6,"recall":0.9396,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":8933466.792452829},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":571.4257,"latency":7.7,"recall":0.9668,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":6468970.188679245},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":411.7653,"latency":9.1,"recall":0.9968,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":4661493.96226415},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":98.0448,"latency":16.1,"recall":0.9803,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":1109941.1320754716},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":58.3152,"latency":28.0,"recall":0.9891,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":660172.0754716981},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":46.8304,"latency":28.6,"recall":0.9941,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":530155.4716981131},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":170.5693,"latency":8.9,"recall":0.9605,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":1930973.2075471696},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":94.7766,"latency":19.599999999999998,"recall":0.9843,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":1072942.641509434},{"db":"ZillizCloud","db_label":"2cu-cap","db_name":"ZillizCloud-2cu-cap-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":44.8695,"latency":30.900000000000002,"recall":0.9966,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":507956.60377358494},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Search Performance Test (500K Dataset, 1536 Dim)","qps":722.0315,"latency":7.7,"recall":0.976,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":16347883.018867927},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)","qps":467.5795,"latency":8.8,"recall":0.9898,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":10586705.660377359},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)","qps":975.2503,"latency":8.200000000000001,"recall":0.9936,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":22081138.86792453},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Search Performance Test (1M Dataset, 768 Dim)","qps":873.3712,"latency":6.7,"recall":0.9477,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":19774442.264150944},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)","qps":544.6203,"latency":8.4,"recall":0.977,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":12331025.660377359},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)","qps":930.9164,"latency":9.6,"recall":0.9968,"label":1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":21077352.452830188},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-perf","db_name":"ZillizCloud-1cu-perf-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Search Performance Test (10M Dataset, 768 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Search Performance Test (5M Dataset, 1536 Dim)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0},{"db":"ZillizCloud","db_label":"1cu-cap","db_name":"ZillizCloud-1cu-cap-v2024.1","case":"Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)","qps":0.0,"latency":0.0,"recall":0.0,"label":-1,"note":"","version":"v2024.1","test_time":"2024-01","qp$":0.0}] \ No newline at end of file From df184767b8f1ff23a1226ed422e55bd408d110b9 Mon Sep 17 00:00:00 2001 From: Sheharyar Ahmad Date: Tue, 5 Nov 2024 19:05:39 +0500 Subject: [PATCH 104/327] fixed pgvectorivvflat cli reranking key error bug --- vectordb_bench/backend/clients/pgvector/cli.py | 3 +++ vectordb_bench/backend/clients/pgvector/config.py | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/vectordb_bench/backend/clients/pgvector/cli.py b/vectordb_bench/backend/clients/pgvector/cli.py index cde125cc1..bf94a7272 100644 --- a/vectordb_bench/backend/clients/pgvector/cli.py +++ b/vectordb_bench/backend/clients/pgvector/cli.py @@ -136,6 +136,9 @@ def PgVectorIVFFlat( lists=parameters["lists"], probes=parameters["probes"], quantization_type=parameters["quantization_type"], + reranking=parameters["reranking"], + reranking_metric=parameters["reranking_metric"], + quantized_fetch_limit=parameters["quantized_fetch_limit"], ), **parameters, ) diff --git a/vectordb_bench/backend/clients/pgvector/config.py b/vectordb_bench/backend/clients/pgvector/config.py index 31d832f13..16d547445 100644 --- a/vectordb_bench/backend/clients/pgvector/config.py +++ b/vectordb_bench/backend/clients/pgvector/config.py @@ -168,6 +168,9 @@ class PgVectorIVFFlatConfig(PgVectorIndexConfig): maintenance_work_mem: Optional[str] = None max_parallel_workers: Optional[int] = None quantization_type: Optional[str] = None + reranking: Optional[bool] = None + quantized_fetch_limit: Optional[int] = None + reranking_metric: Optional[str] = None def index_param(self) -> PgVectorIndexParam: index_parameters = {"lists": self.lists} @@ -187,6 +190,9 @@ def index_param(self) -> PgVectorIndexParam: def search_param(self) -> PgVectorSearchParam: return { "metric_fun_op": self.parse_metric_fun_op(), + "reranking": self.reranking, + "reranking_metric_fun_op": self.parse_reranking_metric_fun_op(), + "quantized_fetch_limit": self.quantized_fetch_limit, } def session_param(self) -> PgVectorSessionCommands: From 653bd39672e944057bde9d2fc89f9417d11e639b Mon Sep 17 00:00:00 2001 From: Sheharyar Ahmad Date: Tue, 5 Nov 2024 23:05:36 +0500 Subject: [PATCH 105/327] set default value of quantized_fetch_limit to 100 in case of ivfflat, because ef_search is not a param of ivfflat. --- vectordb_bench/backend/clients/pgvector/cli.py | 5 +++-- vectordb_bench/cli/cli.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/vectordb_bench/backend/clients/pgvector/cli.py b/vectordb_bench/backend/clients/pgvector/cli.py index bf94a7272..ce98806a8 100644 --- a/vectordb_bench/backend/clients/pgvector/cli.py +++ b/vectordb_bench/backend/clients/pgvector/cli.py @@ -18,11 +18,12 @@ from vectordb_bench.backend.clients import DB - def set_default_quantized_fetch_limit(ctx, param, value): if ctx.params.get("reranking") and value is None: # ef_search is the default value for quantized_fetch_limit as it's bound by ef_search. - return ctx.params["ef_search"] + # 100 (arbitrary) is default value for quantized_fetch_limit for IVFFlat. + default_value = ctx.params["ef_search"] if ctx.command.name == "pgvectorhnsw" else 100 + return default_value return value class PgVectorTypedDict(CommonTypedDict): diff --git a/vectordb_bench/cli/cli.py b/vectordb_bench/cli/cli.py index 44c13400d..e5b9a5fe2 100644 --- a/vectordb_bench/cli/cli.py +++ b/vectordb_bench/cli/cli.py @@ -414,7 +414,7 @@ class HNSWBaseRequiredTypedDict(TypedDict): class HNSWFlavor1(HNSWBaseTypedDict): ef_search: Annotated[ - Optional[int], click.option("--ef-search", type=int, help="hnsw ef-search") + Optional[int], click.option("--ef-search", type=int, help="hnsw ef-search", is_eager=True) ] From f498e71d19a2288ff3255195da966934e62a5884 Mon Sep 17 00:00:00 2001 From: Sheharyar Ahmad Date: Tue, 5 Nov 2024 23:13:47 +0500 Subject: [PATCH 106/327] update comment --- vectordb_bench/backend/clients/pgvector/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vectordb_bench/backend/clients/pgvector/cli.py b/vectordb_bench/backend/clients/pgvector/cli.py index ce98806a8..43385ee6b 100644 --- a/vectordb_bench/backend/clients/pgvector/cli.py +++ b/vectordb_bench/backend/clients/pgvector/cli.py @@ -21,7 +21,7 @@ def set_default_quantized_fetch_limit(ctx, param, value): if ctx.params.get("reranking") and value is None: # ef_search is the default value for quantized_fetch_limit as it's bound by ef_search. - # 100 (arbitrary) is default value for quantized_fetch_limit for IVFFlat. + # 100 is default value for quantized_fetch_limit for IVFFlat. default_value = ctx.params["ef_search"] if ctx.command.name == "pgvectorhnsw" else 100 return default_value return value From 183f47ba54044f0cdc64a6d38566164fb49f2680 Mon Sep 17 00:00:00 2001 From: yangxuan Date: Sat, 14 Sep 2024 15:48:49 +0800 Subject: [PATCH 107/327] Add rate runner Signed-off-by: yangxuan --- tests/test_rate_runner.py | 88 ++++++++++++++ vectordb_bench/__init__.py | 2 +- .../backend/clients/milvus/milvus.py | 5 +- vectordb_bench/backend/dataset.py | 13 +- vectordb_bench/backend/runner/mp_runner.py | 89 +++++++++++++- vectordb_bench/backend/runner/rate_runner.py | 79 ++++++++++++ .../backend/runner/read_write_runner.py | 112 ++++++++++++++++++ vectordb_bench/backend/runner/util.py | 32 +++++ vectordb_bench/backend/task_runner.py | 4 +- 9 files changed, 413 insertions(+), 11 deletions(-) create mode 100644 tests/test_rate_runner.py create mode 100644 vectordb_bench/backend/runner/rate_runner.py create mode 100644 vectordb_bench/backend/runner/read_write_runner.py create mode 100644 vectordb_bench/backend/runner/util.py diff --git a/tests/test_rate_runner.py b/tests/test_rate_runner.py new file mode 100644 index 000000000..363b7a33c --- /dev/null +++ b/tests/test_rate_runner.py @@ -0,0 +1,88 @@ +from typing import Iterable +import argparse +from vectordb_bench.backend.dataset import Dataset, DatasetSource +from vectordb_bench.backend.runner.rate_runner import RatedMultiThreadingInsertRunner +from vectordb_bench.backend.runner.read_write_runner import ReadWriteRunner +from vectordb_bench.backend.clients import DB, VectorDB +from vectordb_bench.backend.clients.milvus.config import FLATConfig +from vectordb_bench.backend.clients.zilliz_cloud.config import AutoIndexConfig + +import logging + +log = logging.getLogger("vectordb_bench") +log.setLevel(logging.DEBUG) + +def get_rate_runner(db): + cohere = Dataset.COHERE.manager(100_000) + prepared = cohere.prepare(DatasetSource.AliyunOSS) + assert prepared + runner = RatedMultiThreadingInsertRunner( + rate = 10, + db = db, + dataset = cohere, + ) + + return runner + +def test_rate_runner(db, insert_rate): + runner = get_rate_runner(db) + + _, t = runner.run_with_rate() + log.info(f"insert run done, time={t}") + +def test_read_write_runner(db, insert_rate, conc: list, search_stage: Iterable[float], read_dur_after_write: int, local: bool=False): + cohere = Dataset.COHERE.manager(1_000_000) + if local is True: + source = DatasetSource.AliyunOSS + else: + source = DatasetSource.S3 + prepared = cohere.prepare(source) + assert prepared + + rw_runner = ReadWriteRunner( + db=db, + dataset=cohere, + insert_rate=insert_rate, + search_stage=search_stage, + read_dur_after_write=read_dur_after_write, + concurrencies=conc + ) + rw_runner.run_read_write() + + +def get_db(db: str, config: dict) -> VectorDB: + if db == DB.Milvus.name: + return DB.Milvus.init_cls(dim=768, db_config=config, db_case_config=FLATConfig(metric_type="COSINE"), drop_old=True, pre_load=True) + elif db == DB.ZillizCloud.name: + return DB.ZillizCloud.init_cls(dim=768, db_config=config, db_case_config=AutoIndexConfig(metric_type="COSINE"), drop_old=True, pre_load=True) + else: + raise ValueError(f"unknown db: {db}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("-r", "--insert_rate", type=int, default="1000", help="insert entity row count per seconds, cps") + parser.add_argument("-d", "--db", type=str, default=DB.Milvus.name, help="db name") + parser.add_argument("-t", "--duration", type=int, default=300, help="stage search duration in seconds") + parser.add_argument("--use_s3", action='store_true', help="whether to use S3 dataset") + + flags = parser.parse_args() + + # TODO read uri, user, password from .env + config = { + "uri": "http://localhost:19530", + "user": "", + "password": "", + } + + conc = (1, 15, 50) + search_stage = (0.5, 0.6, 0.7, 0.8, 0.9, 1.0) + + db = get_db(flags.db, config) + test_read_write_runner( + db=db, + insert_rate=flags.insert_rate, + conc=conc, + search_stage=search_stage, + read_dur_after_write=flags.duration, + local=flags.use_s3) diff --git a/vectordb_bench/__init__.py b/vectordb_bench/__init__.py index f81c9bb3f..568e97705 100644 --- a/vectordb_bench/__init__.py +++ b/vectordb_bench/__init__.py @@ -17,7 +17,7 @@ class config: DEFAULT_DATASET_URL = env.str("DEFAULT_DATASET_URL", AWS_S3_URL) DATASET_LOCAL_DIR = env.path("DATASET_LOCAL_DIR", "/tmp/vectordb_bench/dataset") - NUM_PER_BATCH = env.int("NUM_PER_BATCH", 5000) + NUM_PER_BATCH = env.int("NUM_PER_BATCH", 100) DROP_OLD = env.bool("DROP_OLD", True) USE_SHUFFLED_DATA = env.bool("USE_SHUFFLED_DATA", True) diff --git a/vectordb_bench/backend/clients/milvus/milvus.py b/vectordb_bench/backend/clients/milvus/milvus.py index 4590265ae..8f8571aa3 100644 --- a/vectordb_bench/backend/clients/milvus/milvus.py +++ b/vectordb_bench/backend/clients/milvus/milvus.py @@ -66,7 +66,8 @@ def __init__( self.case_config.index_param(), index_name=self._index_name, ) - # self._pre_load(coll) + if kwargs.get("pre_load") is True: + self._pre_load(col) connections.disconnect("default") @@ -92,7 +93,7 @@ def _optimize(self): self._post_insert() log.info(f"{self.name} optimizing before search") try: - self.col.load() + self.col.load(refresh=True) except Exception as e: log.warning(f"{self.name} optimize error: {e}") raise e from None diff --git a/vectordb_bench/backend/dataset.py b/vectordb_bench/backend/dataset.py index d559eb6be..d62d96684 100644 --- a/vectordb_bench/backend/dataset.py +++ b/vectordb_bench/backend/dataset.py @@ -57,11 +57,11 @@ class CustomDataset(BaseDataset): dir: str file_num: int isCustom: bool = True - + @validator("size") def verify_size(cls, v): return v - + @property def label(self) -> str: return "Custom" @@ -73,7 +73,8 @@ def dir_name(self) -> str: @property def file_count(self) -> int: return self.file_num - + + class LAION(BaseDataset): name: str = "LAION" dim: int = 768 @@ -242,13 +243,15 @@ def __init__(self, dataset: DatasetManager): self._cur = None self._sub_idx = [0 for i in range(len(self._ds.train_files))] # iter num for each file + def __iter__(self): + return self + def _get_iter(self, file_name: str): p = pathlib.Path(self._ds.data_dir, file_name) log.info(f"Get iterator for {p.name}") if not p.exists(): raise IndexError(f"No such file {p}") - log.warning(f"No such file: {p}") - return ParquetFile(p).iter_batches(config.NUM_PER_BATCH) + return ParquetFile(p, memory_map=True, pre_buffer=True).iter_batches(config.NUM_PER_BATCH) def __next__(self) -> pd.DataFrame: """return the data in the next file of the training list""" diff --git a/vectordb_bench/backend/runner/mp_runner.py b/vectordb_bench/backend/runner/mp_runner.py index 6a9f7c979..596865b42 100644 --- a/vectordb_bench/backend/runner/mp_runner.py +++ b/vectordb_bench/backend/runner/mp_runner.py @@ -64,7 +64,7 @@ def search(self, test_data: list[list[float]], q: mp.Queue, cond: mp.Condition) log.warning(f"VectorDB search_embedding error: {e}") traceback.print_exc(chain=True) raise e from None - + latencies.append(time.perf_counter() - s) count += 1 # loop through the test data @@ -87,6 +87,8 @@ def get_mp_context(): log.debug(f"MultiProcessingSearchRunner get multiprocessing start method: {mp_start_method}") return mp.get_context(mp_start_method) + + def _run_all_concurrencies_mem_efficient(self) -> float: max_qps = 0 conc_num_list = [] @@ -145,3 +147,88 @@ def run(self) -> float: def stop(self) -> None: pass + + def run_by_dur(self, duration: int) -> float: + return self._run_by_dur(duration) + + def _run_by_dur(self, duration: int) -> float: + max_qps = 0 + try: + for conc in self.concurrencies: + with mp.Manager() as m: + q, cond = m.Queue(), m.Condition() + with concurrent.futures.ProcessPoolExecutor(mp_context=self.get_mp_context(), max_workers=conc) as executor: + log.info(f"Start search_by_dur {duration}s in concurrency {conc}, filters: {self.filters}") + future_iter = [executor.submit(self.search_by_dur, duration, self.test_data, q, cond) for i in range(conc)] + # Sync all processes + while q.qsize() < conc: + sleep_t = conc if conc < 10 else 10 + time.sleep(sleep_t) + + with cond: + cond.notify_all() + log.info(f"Syncing all process and start concurrency search, concurrency={conc}") + + start = time.perf_counter() + all_count = sum([r.result() for r in future_iter]) + cost = time.perf_counter() - start + + qps = round(all_count / cost, 4) + log.info(f"End search in concurrency {conc}: dur={cost}s, total_count={all_count}, qps={qps}") + + if qps > max_qps: + max_qps = qps + log.info(f"Update largest qps with concurrency {conc}: current max_qps={max_qps}") + except Exception as e: + log.warning(f"Fail to search all concurrencies: {self.concurrencies}, max_qps before failure={max_qps}, reason={e}") + traceback.print_exc() + + # No results available, raise exception + if max_qps == 0.0: + raise e from None + + finally: + self.stop() + + return max_qps + + + def search_by_dur(self, dur: int, test_data: list[list[float]], q: mp.Queue, cond: mp.Condition) -> int: + # sync all process + q.put(1) + with cond: + cond.wait() + + with self.db.init(): + num, idx = len(test_data), random.randint(0, len(test_data) - 1) + + start_time = time.perf_counter() + count = 0 + while time.perf_counter() < start_time + dur: + s = time.perf_counter() + try: + self.db.search_embedding( + test_data[idx], + self.k, + self.filters, + ) + except Exception as e: + log.warning(f"VectorDB search_embedding error: {e}") + traceback.print_exc(chain=True) + raise e from None + + count += 1 + # loop through the test data + idx = idx + 1 if idx < num - 1 else 0 + + if count % 500 == 0: + log.debug(f"({mp.current_process().name:16}) search_count: {count}, latest_latency={time.perf_counter()-s}") + + total_dur = round(time.perf_counter() - start_time, 4) + log.debug( + f"{mp.current_process().name:16} search {self.duration}s: " + f"actual_dur={total_dur}s, count={count}, qps in this process: {round(count / total_dur, 4):3}" + ) + + return count + diff --git a/vectordb_bench/backend/runner/rate_runner.py b/vectordb_bench/backend/runner/rate_runner.py new file mode 100644 index 000000000..4b6d7f6cf --- /dev/null +++ b/vectordb_bench/backend/runner/rate_runner.py @@ -0,0 +1,79 @@ +import logging +import time +from concurrent.futures import ThreadPoolExecutor +import multiprocessing as mp + + +from vectordb_bench.backend.clients import api +from vectordb_bench.backend.dataset import DataSetIterator +from vectordb_bench.backend.utils import time_it +from vectordb_bench import config + +from .util import get_data, is_futures_completed, get_future_exceptions +log = logging.getLogger(__name__) + + +class RatedMultiThreadingInsertRunner: + def __init__( + self, + rate: int, # numRows per second + db: api.VectorDB, + dataset_iter: DataSetIterator, + normalize: bool = False, + timeout: float | None = None, + ): + self.timeout = timeout if isinstance(timeout, (int, float)) else None + self.dataset = dataset_iter + self.db = db + self.normalize = normalize + self.insert_rate = rate + self.batch_rate = rate // config.NUM_PER_BATCH + + def send_insert_task(self, db, emb: list[list[float]], metadata: list[str]): + db.insert_embeddings(emb, metadata) + + @time_it + def run_with_rate(self, q: mp.Queue): + with ThreadPoolExecutor(max_workers=mp.cpu_count()) as executor: + executing_futures = [] + + @time_it + def submit_by_rate() -> bool: + rate = self.batch_rate + for data in self.dataset: + emb, metadata = get_data(data, self.normalize) + executing_futures.append(executor.submit(self.send_insert_task, self.db, emb, metadata)) + rate -= 1 + + if rate == 0: + return False + return rate == self.batch_rate + + with self.db.init(): + while True: + start_time = time.perf_counter() + finished, elapsed_time = submit_by_rate() + if finished is True: + q.put(None, block=True) + log.info(f"End of dataset, left unfinished={len(executing_futures)}") + return + + q.put(True, block=False) + wait_interval = 1 - elapsed_time if elapsed_time < 1 else 0.001 + + e, completed = is_futures_completed(executing_futures, wait_interval) + if completed is True: + ex = get_future_exceptions(executing_futures) + if ex is not None: + log.warn(f"task error, terminating, err={ex}") + q.put(None) + executor.shutdown(wait=True, cancel_futures=True) + raise ex + else: + log.debug(f"Finished {len(executing_futures)} insert-{config.NUM_PER_BATCH} task in 1s, wait_interval={wait_interval:.2f}") + executing_futures = [] + else: + log.warning(f"Failed to finish tasks in 1s, {e}, waited={wait_interval:.2f}, try to check the next round") + dur = time.perf_counter() - start_time + if dur < 1: + time.sleep(1 - dur) diff --git a/vectordb_bench/backend/runner/read_write_runner.py b/vectordb_bench/backend/runner/read_write_runner.py new file mode 100644 index 000000000..6e043dceb --- /dev/null +++ b/vectordb_bench/backend/runner/read_write_runner.py @@ -0,0 +1,112 @@ +import logging +from typing import Iterable +import multiprocessing as mp +import concurrent +import numpy as np +import math + +from .mp_runner import MultiProcessingSearchRunner +from .serial_runner import SerialSearchRunner +from .rate_runner import RatedMultiThreadingInsertRunner +from vectordb_bench.backend.clients import api +from vectordb_bench.backend.dataset import DatasetManager + +log = logging.getLogger(__name__) + + +class ReadWriteRunner(MultiProcessingSearchRunner, RatedMultiThreadingInsertRunner): + def __init__( + self, + db: api.VectorDB, + dataset: DatasetManager, + insert_rate: int = 1000, + normalize: bool = False, + k: int = 100, + filters: dict | None = None, + concurrencies: Iterable[int] = (1, 15, 50), + search_stage: Iterable[float] = (0.5, 0.6, 0.7, 0.8, 0.9, 1.0), # search in any insert portion, 0.0 means search from the start + read_dur_after_write: int = 300, # seconds, search duration when insertion is done + timeout: float | None = None, + ): + self.insert_rate = insert_rate + self.data_volume = dataset.data.size + + for stage in search_stage: + assert 0.0 <= stage <= 1.0, "each search stage should be in [0.0, 1.0]" + self.search_stage = sorted(search_stage) + self.read_dur_after_write = read_dur_after_write + + log.info(f"Init runner, concurencys={concurrencies}, search_stage={search_stage}, stage_search_dur={read_dur_after_write}") + + test_emb = np.stack(dataset.test_data["emb"]) + if normalize: + test_emb = test_emb / np.linalg.norm(test_emb, axis=1)[:, np.newaxis] + test_emb = test_emb.tolist() + + MultiProcessingSearchRunner.__init__( + self, + db=db, + test_data=test_emb, + k=k, + filters=filters, + concurrencies=concurrencies, + ) + RatedMultiThreadingInsertRunner.__init__( + self, + rate=insert_rate, + db=db, + dataset_iter=iter(dataset), + normalize=normalize, + ) + self.serial_search_runner = SerialSearchRunner( + db=db, + test_data=test_emb, + ground_truth=dataset.gt_data, + k=k, + ) + + def run_read_write(self): + futures = [] + with mp.Manager() as m: + q = m.Queue() + with concurrent.futures.ProcessPoolExecutor(mp_context=mp.get_context("spawn"), max_workers=2) as executor: + futures.append(executor.submit(self.run_with_rate, q)) + futures.append(executor.submit(self.run_search_by_sig, q)) + + for future in concurrent.futures.as_completed(futures): + res = future.result() + log.info(f"Result = {res}") + + log.info("Concurrent read write all done") + + + def run_search_by_sig(self, q): + res = [] + total_batch = math.ceil(self.data_volume / self.insert_rate) + batch = 0 + recall = 'x' + + for idx, stage in enumerate(self.search_stage): + target_batch = int(total_batch * stage) + while q.get(block=True): + batch += 1 + if batch >= target_batch: + perc = int(stage * 100) + log.info(f"Insert {perc}% done, total batch={total_batch}") + log.info(f"[{batch}/{total_batch}] Serial search - {perc}% start") + recall, ndcg, p99 =self.serial_search_runner.run() + + if idx < len(self.search_stage) - 1: + stage_search_dur = (self.data_volume * (self.search_stage[idx + 1] - stage) // self.insert_rate) // len(self.concurrencies) + if stage_search_dur < 30: + log.warning(f"Search duration too short, please reduce concurrency count or insert rate, or increase dataset volume: dur={stage_search_dur}, concurrencies={len(self.concurrencies)}, insert_rate={self.insert_rate}") + log.info(f"[{batch}/{total_batch}] Conc search - {perc}% start, dur for each conc={stage_search_dur}s") + else: + last_search_dur = self.data_volume * (1.0 - stage) // self.insert_rate + stage_search_dur = last_search_dur + self.read_dur_after_write + log.info(f"[{batch}/{total_batch}] Last conc search - {perc}% start, [read_until_write|read_after_write|total] =[{last_search_dur}s|{self.read_dur_after_write}s|{stage_search_dur}s]") + + max_qps = self.run_by_dur(stage_search_dur) + res.append((perc, max_qps, recall)) + break + return res diff --git a/vectordb_bench/backend/runner/util.py b/vectordb_bench/backend/runner/util.py new file mode 100644 index 000000000..0dfd9d0c4 --- /dev/null +++ b/vectordb_bench/backend/runner/util.py @@ -0,0 +1,32 @@ +import logging +import concurrent +from typing import Iterable + +from pandas import DataFrame +import numpy as np + +log = logging.getLogger(__name__) + +def get_data(data_df: DataFrame, normalize: bool) -> tuple[list[list[float]], list[str]]: + all_metadata = data_df['id'].tolist() + emb_np = np.stack(data_df['emb']) + if normalize: + log.debug("normalize the 100k train data") + all_embeddings = (emb_np / np.linalg.norm(emb_np, axis=1)[:, np.newaxis]).tolist() + else: + all_embeddings = emb_np.tolist() + return all_embeddings, all_metadata + +def is_futures_completed(futures: Iterable[concurrent.futures.Future], interval) -> (Exception, bool): + try: + list(concurrent.futures.as_completed(futures, timeout=interval)) + except TimeoutError as e: + return e, False + return None, True + + +def get_future_exceptions(futures: Iterable[concurrent.futures.Future]) -> BaseException | None: + for f in futures: + if f.exception() is not None: + return f.exception() + return diff --git a/vectordb_bench/backend/task_runner.py b/vectordb_bench/backend/task_runner.py index a6d94f186..c275ebe92 100644 --- a/vectordb_bench/backend/task_runner.py +++ b/vectordb_bench/backend/task_runner.py @@ -150,7 +150,7 @@ def _run_perf_case(self, drop_old: bool = True) -> Metric: ) self._init_search_runner() - + m.qps, m.conc_num_list, m.conc_qps_list, m.conc_latency_p99_list = self._conc_search() m.recall, m.serial_latency_p99 = self._serial_search() ''' @@ -186,7 +186,7 @@ def _run_perf_case(self, drop_old: bool = True) -> Metric: if TaskStage.SEARCH_CONCURRENT in self.config.stages: search_results = self._conc_search() m.qps, m.conc_num_list, m.conc_qps_list, m.conc_latency_p99_list = search_results - + except Exception as e: log.warning(f"Failed to run performance case, reason = {e}") traceback.print_exc() From 983ea4cc4dbd271fe90bdcae413141dc525246cd Mon Sep 17 00:00:00 2001 From: "min.tian" Date: Mon, 25 Nov 2024 10:42:53 +0800 Subject: [PATCH 108/327] fix conc_latency_p99 calculation; add conc_latency_avg metric; conc_test first Signed-off-by: min.tian --- vectordb_bench/backend/runner/mp_runner.py | 9 +++-- vectordb_bench/backend/task_runner.py | 6 ++-- .../frontend/components/concurrent/charts.py | 34 ++++++++++++++----- vectordb_bench/frontend/pages/concurrent.py | 6 +++- vectordb_bench/metric.py | 1 + 5 files changed, 40 insertions(+), 16 deletions(-) diff --git a/vectordb_bench/backend/runner/mp_runner.py b/vectordb_bench/backend/runner/mp_runner.py index 596865b42..8f35dcd8d 100644 --- a/vectordb_bench/backend/runner/mp_runner.py +++ b/vectordb_bench/backend/runner/mp_runner.py @@ -89,11 +89,12 @@ def get_mp_context(): - def _run_all_concurrencies_mem_efficient(self) -> float: + def _run_all_concurrencies_mem_efficient(self): max_qps = 0 conc_num_list = [] conc_qps_list = [] conc_latency_p99_list = [] + conc_latency_avg_list = [] try: for conc in self.concurrencies: with mp.Manager() as m: @@ -113,13 +114,15 @@ def _run_all_concurrencies_mem_efficient(self) -> float: start = time.perf_counter() all_count = sum([r.result()[0] for r in future_iter]) latencies = sum([r.result()[2] for r in future_iter], start=[]) - latency_p99 = np.percentile(latencies, 0.99) + latency_p99 = np.percentile(latencies, 99) + latency_avg = np.mean(latencies) cost = time.perf_counter() - start qps = round(all_count / cost, 4) conc_num_list.append(conc) conc_qps_list.append(qps) conc_latency_p99_list.append(latency_p99) + conc_latency_avg_list.append(latency_avg) log.info(f"End search in concurrency {conc}: dur={cost}s, total_count={all_count}, qps={qps}") if qps > max_qps: @@ -136,7 +139,7 @@ def _run_all_concurrencies_mem_efficient(self) -> float: finally: self.stop() - return max_qps, conc_num_list, conc_qps_list, conc_latency_p99_list + return max_qps, conc_num_list, conc_qps_list, conc_latency_p99_list, conc_latency_avg_list def run(self) -> float: """ diff --git a/vectordb_bench/backend/task_runner.py b/vectordb_bench/backend/task_runner.py index c275ebe92..0b115bba7 100644 --- a/vectordb_bench/backend/task_runner.py +++ b/vectordb_bench/backend/task_runner.py @@ -176,6 +176,9 @@ def _run_perf_case(self, drop_old: bool = True) -> Metric: or TaskStage.SEARCH_CONCURRENT in self.config.stages ): self._init_search_runner() + if TaskStage.SEARCH_CONCURRENT in self.config.stages: + search_results = self._conc_search() + m.qps, m.conc_num_list, m.conc_qps_list, m.conc_latency_p99_list, m.conc_latency_avg_list = search_results if TaskStage.SEARCH_SERIAL in self.config.stages: search_results = self._serial_search() ''' @@ -183,9 +186,6 @@ def _run_perf_case(self, drop_old: bool = True) -> Metric: m.serial_latencies = search_results.serial_latencies ''' m.recall, m.ndcg, m.serial_latency_p99 = search_results - if TaskStage.SEARCH_CONCURRENT in self.config.stages: - search_results = self._conc_search() - m.qps, m.conc_num_list, m.conc_qps_list, m.conc_latency_p99_list = search_results except Exception as e: log.warning(f"Failed to run performance case, reason = {e}") diff --git a/vectordb_bench/frontend/components/concurrent/charts.py b/vectordb_bench/frontend/components/concurrent/charts.py index 68e7f3210..11379c4b3 100644 --- a/vectordb_bench/frontend/components/concurrent/charts.py +++ b/vectordb_bench/frontend/components/concurrent/charts.py @@ -6,7 +6,7 @@ from vectordb_bench.frontend.config.styles import COLOR_MAP -def drawChartsByCase(allData, showCaseNames: list[str], st): +def drawChartsByCase(allData, showCaseNames: list[str], st, latency_type: str): initMainExpanderStyle(st) for caseName in showCaseNames: chartContainer = st.expander(caseName, True) @@ -14,15 +14,24 @@ def drawChartsByCase(allData, showCaseNames: list[str], st): data = [ { "conc_num": caseData["conc_num_list"][i], - "qps": caseData["conc_qps_list"][i], - "latency_p99": caseData["conc_latency_p99_list"][i] * 1000, + "qps": caseData["conc_qps_list"][i] + if 0 <= i < len(caseData["conc_qps_list"]) + else 0, + "latency_p99": caseData["conc_latency_p99_list"][i] * 1000 + if 0 <= i < len(caseData["conc_latency_p99_list"]) + else 0, + "latency_avg": caseData["conc_latency_avg_list"][i] * 1000 + if 0 <= i < len(caseData["conc_latency_avg_list"]) + else 0, "db_name": caseData["db_name"], "db": caseData["db"], } for caseData in caseDataList for i in range(len(caseData["conc_num_list"])) ] - drawChart(data, chartContainer, key=f"{caseName}-qps-p99") + drawChart( + data, chartContainer, key=f"{caseName}-qps-p99", x_metric=latency_type + ) def getRange(metric, data, padding_multipliers): @@ -36,14 +45,21 @@ def getRange(metric, data, padding_multipliers): return rangeV -def drawChart(data, st, key: str): +def gen_title(s: str) -> str: + if "latency" in s: + return f'{s.replace("_", " ").title()} (ms)' + else: + return s.upper() + + +def drawChart(data, st, key: str, x_metric: str = "latency_p99", y_metric: str = "qps"): if len(data) == 0: return - x = "latency_p99" + x = x_metric xrange = getRange(x, data, [0.05, 0.1]) - y = "qps" + y = y_metric yrange = getRange(y, data, [0.2, 0.1]) color = "db" @@ -69,8 +85,8 @@ def drawChart(data, st, key: str): }, height=720, ) - fig.update_xaxes(range=xrange, title_text="Latency P99 (ms)") - fig.update_yaxes(range=yrange, title_text="QPS") + fig.update_xaxes(range=xrange, title_text=gen_title(x_metric)) + fig.update_yaxes(range=yrange, title_text=gen_title(y_metric)) fig.update_traces(textposition="bottom right", texttemplate="conc-%{text:,.4~r}") st.plotly_chart(fig, use_container_width=True, key=key) diff --git a/vectordb_bench/frontend/pages/concurrent.py b/vectordb_bench/frontend/pages/concurrent.py index b4eae339c..941675436 100644 --- a/vectordb_bench/frontend/pages/concurrent.py +++ b/vectordb_bench/frontend/pages/concurrent.py @@ -55,7 +55,11 @@ def check_conc_data(res: TestResult): resultesContainer = st.sidebar.container() getResults(resultesContainer, "vectordb_bench_concurrent") - drawChartsByCase(shownData, showCaseNames, st.container()) + # main + latency_type = st.radio("Latency Type", options=["latency_p99", "latency_avg"]) + drawChartsByCase( + shownData, showCaseNames, st.container(), latency_type=latency_type + ) # footer footer(st.container()) diff --git a/vectordb_bench/metric.py b/vectordb_bench/metric.py index 5c23072e3..9f083a5c6 100644 --- a/vectordb_bench/metric.py +++ b/vectordb_bench/metric.py @@ -23,6 +23,7 @@ class Metric: conc_num_list: list[int] = field(default_factory=list) conc_qps_list: list[float] = field(default_factory=list) conc_latency_p99_list: list[float] = field(default_factory=list) + conc_latency_avg_list: list[float] = field(default_factory=list) QURIES_PER_DOLLAR_METRIC = "QP$ (Quries per Dollar)" From 02301842d5d1aae17329ac391bbb1ef968d454af Mon Sep 17 00:00:00 2001 From: Sheharyar Ahmad Date: Mon, 25 Nov 2024 22:16:17 +0500 Subject: [PATCH 109/327] Added AlloyDB client --- README.md | 3 +- pyproject.toml | 1 + vectordb_bench/backend/clients/__init__.py | 13 + .../backend/clients/alloydb/alloydb.py | 377 ++++++++++++++++++ vectordb_bench/backend/clients/alloydb/cli.py | 147 +++++++ .../backend/clients/alloydb/config.py | 168 ++++++++ vectordb_bench/backend/clients/api.py | 2 + vectordb_bench/cli/vectordbbench.py | 2 + .../frontend/config/dbCaseConfigs.py | 166 ++++++++ vectordb_bench/models.py | 9 + 10 files changed, 887 insertions(+), 1 deletion(-) create mode 100644 vectordb_bench/backend/clients/alloydb/alloydb.py create mode 100644 vectordb_bench/backend/clients/alloydb/cli.py create mode 100644 vectordb_bench/backend/clients/alloydb/config.py diff --git a/README.md b/README.md index bc550a7f2..9a109146e 100644 --- a/README.md +++ b/README.md @@ -38,11 +38,12 @@ All the database client supported | pgvector | `pip install vectordb-bench[pgvector]` | | pgvecto.rs | `pip install vectordb-bench[pgvecto_rs]` | | pgvectorscale | `pip install vectordb-bench[pgvectorscale]` | -| pgdiskann | `pip install vectordb-bench[pgdiskann]` | +| pgdiskann | `pip install vectordb-bench[pgdiskann]` | | redis | `pip install vectordb-bench[redis]` | | memorydb | `pip install vectordb-bench[memorydb]` | | chromadb | `pip install vectordb-bench[chromadb]` | | awsopensearch | `pip install vectordb-bench[awsopensearch]` | +| alloydb | `pip install vectordb-bench[alloydb]` | ### Run diff --git a/pyproject.toml b/pyproject.toml index 000800389..760404b1d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,6 +73,7 @@ elastic = [ "elasticsearch" ] pgvector = [ "psycopg", "psycopg-binary", "pgvector" ] pgvectorscale = [ "psycopg", "psycopg-binary", "pgvector" ] pgdiskann = [ "psycopg", "psycopg-binary", "pgvector" ] +alloydb = [ "psycopg", "psycopg-binary", "pgvector"] pgvecto_rs = [ "pgvecto_rs[psycopg3]>=0.2.2" ] redis = [ "redis" ] memorydb = [ "memorydb" ] diff --git a/vectordb_bench/backend/clients/__init__.py b/vectordb_bench/backend/clients/__init__.py index c26aa3d6d..8859381b5 100644 --- a/vectordb_bench/backend/clients/__init__.py +++ b/vectordb_bench/backend/clients/__init__.py @@ -32,6 +32,7 @@ class DB(Enum): PgVectoRS = "PgVectoRS" PgVectorScale = "PgVectorScale" PgDiskANN = "PgDiskANN" + AlloyDB = "AlloyDB" Redis = "Redis" MemoryDB = "MemoryDB" Chroma = "Chroma" @@ -97,6 +98,10 @@ def init_cls(self) -> Type[VectorDB]: if self == DB.AWSOpenSearch: from .aws_opensearch.aws_opensearch import AWSOpenSearch return AWSOpenSearch + + if self == DB.AlloyDB: + from .alloydb.alloydb import AlloyDB + return AlloyDB @property def config_cls(self) -> Type[DBConfig]: @@ -156,6 +161,10 @@ def config_cls(self) -> Type[DBConfig]: if self == DB.AWSOpenSearch: from .aws_opensearch.config import AWSOpenSearchConfig return AWSOpenSearchConfig + + if self == DB.AlloyDB: + from .alloydb.config import AlloyDBConfig + return AlloyDBConfig def case_config_cls(self, index_type: IndexType | None = None) -> Type[DBCaseConfig]: if self == DB.Milvus: @@ -197,6 +206,10 @@ def case_config_cls(self, index_type: IndexType | None = None) -> Type[DBCaseCon if self == DB.PgDiskANN: from .pgdiskann.config import _pgdiskann_case_config return _pgdiskann_case_config.get(index_type) + + if self == DB.AlloyDB: + from .alloydb.config import _alloydb_case_config + return _alloydb_case_config.get(index_type) # DB.Pinecone, DB.Chroma, DB.Redis return EmptyDBCaseConfig diff --git a/vectordb_bench/backend/clients/alloydb/alloydb.py b/vectordb_bench/backend/clients/alloydb/alloydb.py new file mode 100644 index 000000000..c053c2fad --- /dev/null +++ b/vectordb_bench/backend/clients/alloydb/alloydb.py @@ -0,0 +1,377 @@ +"""Wrapper around the alloydb vector database over VectorDB""" + +import logging +import pprint +from contextlib import contextmanager +from typing import Any, Generator, Optional, Tuple, Sequence + +import numpy as np +import psycopg +from pgvector.psycopg import register_vector +from psycopg import Connection, Cursor, sql + +from ..api import VectorDB +from .config import AlloyDBConfigDict, AlloyDBIndexConfig, AlloyDBScaNNConfig + +log = logging.getLogger(__name__) + + +class AlloyDB(VectorDB): + """Use psycopg instructions""" + + conn: psycopg.Connection[Any] | None = None + cursor: psycopg.Cursor[Any] | None = None + + _filtered_search: sql.Composed + _unfiltered_search: sql.Composed + + def __init__( + self, + dim: int, + db_config: AlloyDBConfigDict, + db_case_config: AlloyDBIndexConfig, + collection_name: str = "alloydb_collection", + drop_old: bool = False, + **kwargs, + ): + self.name = "AlloyDB" + self.db_config = db_config + self.case_config = db_case_config + self.table_name = collection_name + self.dim = dim + + self._index_name = "alloydb_index" + self._primary_field = "id" + self._vector_field = "embedding" + + # construct basic units + self.conn, self.cursor = self._create_connection(**self.db_config) + + # create vector extension + self.cursor.execute("CREATE EXTENSION IF NOT EXISTS alloydb_scann CASCADE") + self.conn.commit() + + log.info(f"{self.name} config values: {self.db_config}\n{self.case_config}") + if not any( + ( + self.case_config.create_index_before_load, + self.case_config.create_index_after_load, + ) + ): + err = f"{self.name} config must create an index using create_index_before_load or create_index_after_load" + log.error(err) + raise RuntimeError( + f"{err}\n{pprint.pformat(self.db_config)}\n{pprint.pformat(self.case_config)}" + ) + + if drop_old: + self._drop_index() + self._drop_table() + self._create_table(dim) + if self.case_config.create_index_before_load: + self._create_index() + + self.cursor.close() + self.conn.close() + self.cursor = None + self.conn = None + + @staticmethod + def _create_connection(**kwargs) -> Tuple[Connection, Cursor]: + conn = psycopg.connect(**kwargs) + register_vector(conn) + conn.autocommit = False + cursor = conn.cursor() + + assert conn is not None, "Connection is not initialized" + assert cursor is not None, "Cursor is not initialized" + return conn, cursor + + def _generate_search_query(self, filtered: bool=False) -> sql.Composed: + search_query = sql.Composed( + [ + sql.SQL( + "SELECT id FROM public.{table_name} {where_clause} ORDER BY embedding " + ).format( + table_name=sql.Identifier(self.table_name), + where_clause=sql.SQL("WHERE id >= %s") if filtered else sql.SQL(""), + ), + sql.SQL(self.case_config.search_param()["metric_fun_op"]), + sql.SQL(" %s::vector LIMIT %s::int"), + ] + ) + return search_query + + @contextmanager + def init(self) -> Generator[None, None, None]: + """ + Examples: + >>> with self.init(): + >>> self.insert_embeddings() + >>> self.search_embedding() + """ + + self.conn, self.cursor = self._create_connection(**self.db_config) + + # index configuration may have commands defined that we should set during each client session + session_options: Sequence[dict[str, Any]] = self.case_config.session_param()["session_options"] + + if len(session_options) > 0: + for setting in session_options: + command = sql.SQL("SET {setting_name} " + "= {val};").format( + setting_name=sql.Identifier(setting['parameter']['setting_name']), + val=sql.Identifier(str(setting['parameter']['val'])), + ) + log.debug(command.as_string(self.cursor)) + self.cursor.execute(command) + self.conn.commit() + + self._filtered_search = self._generate_search_query(filtered=True) + self._unfiltered_search = self._generate_search_query() + + try: + yield + finally: + self.cursor.close() + self.conn.close() + self.cursor = None + self.conn = None + + def _drop_table(self): + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + log.info(f"{self.name} client drop table : {self.table_name}") + + self.cursor.execute( + sql.SQL("DROP TABLE IF EXISTS public.{table_name}").format( + table_name=sql.Identifier(self.table_name) + ) + ) + self.conn.commit() + + def ready_to_load(self): + pass + + def optimize(self): + self._post_insert() + + def _post_insert(self): + log.info(f"{self.name} post insert before optimize") + if self.case_config.create_index_after_load: + self._drop_index() + self._create_index() + + def _drop_index(self): + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + log.info(f"{self.name} client drop index : {self._index_name}") + + drop_index_sql = sql.SQL("DROP INDEX IF EXISTS {index_name}").format( + index_name=sql.Identifier(self._index_name) + ) + log.debug(drop_index_sql.as_string(self.cursor)) + self.cursor.execute(drop_index_sql) + self.conn.commit() + + def _set_parallel_index_build_param(self): + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + index_param = self.case_config.index_param() + + if index_param["enable_pca"] is not None: + self.cursor.execute( + sql.SQL("SET scann.enable_pca TO {};").format( + index_param["enable_pca"] + ) + ) + self.cursor.execute( + sql.SQL("ALTER USER {} SET scann.enable_pca TO {};").format( + sql.Identifier(self.db_config["user"]), + index_param["enable_pca"], + ) + ) + self.conn.commit() + + if index_param["maintenance_work_mem"] is not None: + self.cursor.execute( + sql.SQL("SET maintenance_work_mem TO {};").format( + index_param["maintenance_work_mem"] + ) + ) + self.cursor.execute( + sql.SQL("ALTER USER {} SET maintenance_work_mem TO {};").format( + sql.Identifier(self.db_config["user"]), + index_param["maintenance_work_mem"], + ) + ) + self.conn.commit() + + if index_param["max_parallel_workers"] is not None: + self.cursor.execute( + sql.SQL("SET max_parallel_maintenance_workers TO '{}';").format( + index_param["max_parallel_workers"] + ) + ) + self.cursor.execute( + sql.SQL( + "ALTER USER {} SET max_parallel_maintenance_workers TO '{}';" + ).format( + sql.Identifier(self.db_config["user"]), + index_param["max_parallel_workers"], + ) + ) + self.cursor.execute( + sql.SQL("SET max_parallel_workers TO '{}';").format( + index_param["max_parallel_workers"] + ) + ) + self.cursor.execute( + sql.SQL( + "ALTER USER {} SET max_parallel_workers TO '{}';" + ).format( + sql.Identifier(self.db_config["user"]), + index_param["max_parallel_workers"], + ) + ) + self.cursor.execute( + sql.SQL( + "ALTER TABLE {} SET (parallel_workers = {});" + ).format( + sql.Identifier(self.table_name), + index_param["max_parallel_workers"], + ) + ) + self.conn.commit() + + results = self.cursor.execute( + sql.SQL("SHOW max_parallel_maintenance_workers;") + ).fetchall() + results.extend( + self.cursor.execute(sql.SQL("SHOW max_parallel_workers;")).fetchall() + ) + results.extend( + self.cursor.execute(sql.SQL("SHOW maintenance_work_mem;")).fetchall() + ) + log.info(f"{self.name} parallel index creation parameters: {results}") + + def _create_index(self): + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + log.info(f"{self.name} client create index : {self._index_name}") + + index_param = self.case_config.index_param() + self._set_parallel_index_build_param() + options = [] + for option in index_param["index_creation_with_options"]: + if option['val'] is not None: + options.append( + sql.SQL("{option_name} = {val}").format( + option_name=sql.Identifier(option['option_name']), + val=sql.Identifier(str(option['val'])), + ) + ) + if any(options): + with_clause = sql.SQL("WITH ({});").format(sql.SQL(", ").join(options)) + else: + with_clause = sql.Composed(()) + + index_create_sql = sql.SQL( + """ + CREATE INDEX IF NOT EXISTS {index_name} ON public.{table_name} + USING {index_type} (embedding {embedding_metric}) + """ + ).format( + index_name=sql.Identifier(self._index_name), + table_name=sql.Identifier(self.table_name), + index_type=sql.Identifier(index_param["index_type"]), + embedding_metric=sql.Identifier(index_param["metric"]), + ) + + index_create_sql_with_with_clause = ( + index_create_sql + with_clause + ).join(" ") + log.debug(index_create_sql_with_with_clause.as_string(self.cursor)) + self.cursor.execute(index_create_sql_with_with_clause) + self.conn.commit() + + def _create_table(self, dim: int): + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + try: + log.info(f"{self.name} client create table : {self.table_name}") + + # create table + self.cursor.execute( + sql.SQL( + "CREATE TABLE IF NOT EXISTS public.{table_name} (id BIGINT PRIMARY KEY, embedding vector({dim}));" + ).format(table_name=sql.Identifier(self.table_name), dim=dim) + ) + self.cursor.execute( + sql.SQL( + "ALTER TABLE public.{table_name} ALTER COLUMN embedding SET STORAGE PLAIN;" + ).format(table_name=sql.Identifier(self.table_name)) + ) + self.conn.commit() + except Exception as e: + log.warning( + f"Failed to create alloydb table: {self.table_name} error: {e}" + ) + raise e from None + + def insert_embeddings( + self, + embeddings: list[list[float]], + metadata: list[int], + **kwargs: Any, + ) -> Tuple[int, Optional[Exception]]: + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + try: + metadata_arr = np.array(metadata) + embeddings_arr = np.array(embeddings) + + with self.cursor.copy( + sql.SQL("COPY public.{table_name} FROM STDIN (FORMAT BINARY)").format( + table_name=sql.Identifier(self.table_name) + ) + ) as copy: + copy.set_types(["bigint", "vector"]) + for i, row in enumerate(metadata_arr): + copy.write_row((row, embeddings_arr[i])) + self.conn.commit() + + if kwargs.get("last_batch"): + self._post_insert() + + return len(metadata), None + except Exception as e: + log.warning( + f"Failed to insert data into alloydb table ({self.table_name}), error: {e}" + ) + return 0, e + + def search_embedding( + self, + query: list[float], + k: int = 100, + filters: dict | None = None, + timeout: int | None = None, + ) -> list[int]: + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + q = np.asarray(query) + if filters: + gt = filters.get("id") + result = self.cursor.execute( + self._filtered_search, (gt, q, k), prepare=True, binary=True + ) + else: + result = self.cursor.execute( + self._unfiltered_search, (q, k), prepare=True, binary=True + ) + + return [int(i[0]) for i in result.fetchall()] diff --git a/vectordb_bench/backend/clients/alloydb/cli.py b/vectordb_bench/backend/clients/alloydb/cli.py new file mode 100644 index 000000000..ab49da955 --- /dev/null +++ b/vectordb_bench/backend/clients/alloydb/cli.py @@ -0,0 +1,147 @@ +from typing import Annotated, Optional, TypedDict, Unpack + +import click +import os +from pydantic import SecretStr + +from vectordb_bench.backend.clients.api import MetricType + +from ....cli.cli import ( + CommonTypedDict, + cli, + click_parameter_decorators_from_typed_dict, + get_custom_case_config, + run, +) +from vectordb_bench.backend.clients import DB + + +class AlloyDBTypedDict(CommonTypedDict): + user_name: Annotated[ + str, click.option("--user-name", type=str, help="Db username", required=True) + ] + password: Annotated[ + str, + click.option("--password", + type=str, + help="Postgres database password", + default=lambda: os.environ.get("POSTGRES_PASSWORD", ""), + show_default="$POSTGRES_PASSWORD", + ), + ] + + host: Annotated[ + str, click.option("--host", type=str, help="Db host", required=True) + ] + db_name: Annotated[ + str, click.option("--db-name", type=str, help="Db name", required=True) + ] + maintenance_work_mem: Annotated[ + Optional[str], + click.option( + "--maintenance-work-mem", + type=str, + help="Sets the maximum memory to be used for maintenance operations (index creation). " + "Can be entered as string with unit like '64GB' or as an integer number of KB." + "This will set the parameters: max_parallel_maintenance_workers," + " max_parallel_workers & table(parallel_workers)", + required=False, + ), + ] + max_parallel_workers: Annotated[ + Optional[int], + click.option( + "--max-parallel-workers", + type=int, + help="Sets the maximum number of parallel processes per maintenance operation (index creation)", + required=False, + ), + ] + + + +class AlloyDBScaNNTypedDict(AlloyDBTypedDict): + num_leaves: Annotated[ + int, + click.option("--num-leaves", type=int, help="Number of leaves", required=True) + ] + num_leaves_to_search: Annotated[ + int, + click.option("--num-leaves-to-search", type=int, help="Number of leaves to search", required=True) + ] + pre_reordering_num_neighbors: Annotated[ + Optional[int], + click.option("--pre-reordering-num-neighbors", type=int, help="Pre-reordering number of neighbors",) + ] + max_top_neighbors_buffer_size: Annotated[ + int, + click.option("--max-top-neighbors-buffer-size", type=int, help="Maximum top neighbors buffer size", default=20_000) + ] + num_search_threads: Annotated[ + int, + click.option("--num-search-threads", type=int, help="Number of search threads", default=2) + ] + max_num_prefetch_datasets: Annotated[ + int, + click.option("--max-num-prefetch-datasets", type=int, help="Maximum number of prefetch datasets", default=100) + ] + quantizer: Annotated[ + str, + click.option( + "--quantizer", + type=click.Choice(["SQ8", "FLAT"]), + help="Quantizer type", + default="SQ8" + ) + ] + enable_pca: Annotated[ + bool, click.option( + "--enable-pca", + type=click.Choice(["on", "off"]), + help="Enable PCA", + default="on" + ) + ] + max_num_levels: Annotated[ + int, + click.option( + "--max-num-levels", + type=click.Choice([1, 2]), + help="Maximum number of levels", + default=1 + ) + ] + + +@cli.command() +@click_parameter_decorators_from_typed_dict(AlloyDBScaNNTypedDict) +def AlloyDBScaNN( + **parameters: Unpack[AlloyDBScaNNTypedDict], +): + from .config import AlloyDBConfig, AlloyDBScaNNConfig + + parameters["custom_case"] = get_custom_case_config(parameters) + run( + db=DB.AlloyDB, + db_config=AlloyDBConfig( + db_label=parameters["db_label"], + user_name=SecretStr(parameters["user_name"]), + password=SecretStr(parameters["password"]), + host=parameters["host"], + db_name=parameters["db_name"], + ), + db_case_config=AlloyDBScaNNConfig( + num_leaves=parameters["num_leaves"], + quantizer=parameters["quantizer"], + enable_pca=parameters["enable_pca"], + max_num_levels=parameters["max_num_levels"], + num_leaves_to_search=parameters["num_leaves_to_search"], + max_top_neighbors_buffer_size=parameters["max_top_neighbors_buffer_size"], + pre_reordering_num_neighbors=parameters["pre_reordering_num_neighbors"], + num_search_threads=parameters["num_search_threads"], + max_num_prefetch_datasets=parameters["max_num_prefetch_datasets"], + max_parallel_workers=parameters["max_parallel_workers"], + maintenance_work_mem=parameters["maintenance_work_mem"], + ), + **parameters, + ) \ No newline at end of file diff --git a/vectordb_bench/backend/clients/alloydb/config.py b/vectordb_bench/backend/clients/alloydb/config.py new file mode 100644 index 000000000..1d5dde519 --- /dev/null +++ b/vectordb_bench/backend/clients/alloydb/config.py @@ -0,0 +1,168 @@ +from abc import abstractmethod +from typing import Any, Mapping, Optional, Sequence, TypedDict +from pydantic import BaseModel, SecretStr +from typing_extensions import LiteralString +from ..api import DBCaseConfig, DBConfig, IndexType, MetricType + +POSTGRE_URL_PLACEHOLDER = "postgresql://%s:%s@%s/%s" + + +class AlloyDBConfigDict(TypedDict): + """These keys will be directly used as kwargs in psycopg connection string, + so the names must match exactly psycopg API""" + + user: str + password: str + host: str + port: int + dbname: str + + +class AlloyDBConfig(DBConfig): + user_name: SecretStr = SecretStr("postgres") + password: SecretStr + host: str = "localhost" + port: int = 5432 + db_name: str + + def to_dict(self) -> AlloyDBConfigDict: + user_str = self.user_name.get_secret_value() + pwd_str = self.password.get_secret_value() + return { + "host": self.host, + "port": self.port, + "dbname": self.db_name, + "user": user_str, + "password": pwd_str, + } + + +class AlloyDBIndexParam(TypedDict): + metric: str + index_type: str + index_creation_with_options: Sequence[dict[str, Any]] + maintenance_work_mem: Optional[str] + max_parallel_workers: Optional[int] + + +class AlloyDBSearchParam(TypedDict): + metric_fun_op: LiteralString + + +class AlloyDBSessionCommands(TypedDict): + session_options: Sequence[dict[str, Any]] + + +class AlloyDBIndexConfig(BaseModel, DBCaseConfig): + metric_type: MetricType | None = None + create_index_before_load: bool = False + create_index_after_load: bool = True + + def parse_metric(self) -> str: + if self.metric_type == MetricType.L2: + return "l2" + elif self.metric_type == MetricType.DP: + return "dot_product" + return "cosine" + + def parse_metric_fun_op(self) -> LiteralString: + if self.metric_type == MetricType.L2: + return "<->" + elif self.metric_type == MetricType.IP: + return "<#>" + return "<=>" + + @abstractmethod + def index_param(self) -> AlloyDBIndexParam: + ... + + @abstractmethod + def search_param(self) -> AlloyDBSearchParam: + ... + + @abstractmethod + def session_param(self) -> AlloyDBSessionCommands: + ... + + @staticmethod + def _optionally_build_with_options(with_options: Mapping[str, Any]) -> Sequence[dict[str, Any]]: + """Walk through mappings, creating a List of {key1 = value} pairs. That will be used to build a where clause""" + options = [] + for option_name, value in with_options.items(): + if value is not None: + options.append( + { + "option_name": option_name, + "val": str(value), + } + ) + return options + + @staticmethod + def _optionally_build_set_options( + set_mapping: Mapping[str, Any] + ) -> Sequence[dict[str, Any]]: + """Walk through options, creating 'SET 'key1 = "value1";' list""" + session_options = [] + for setting_name, value in set_mapping.items(): + if value: + session_options.append( + {"parameter": { + "setting_name": setting_name, + "val": str(value), + }, + } + ) + return session_options + + +class AlloyDBScaNNConfig(AlloyDBIndexConfig): + index: IndexType = IndexType.SCANN + num_leaves: int | None + quantizer: str | None + enable_pca: str | None + max_num_levels: int | None + num_leaves_to_search: int | None + max_top_neighbors_buffer_size: int | None + pre_reordering_num_neighbors: int | None + num_search_threads: int | None + max_num_prefetch_datasets: int | None + maintenance_work_mem: Optional[str] = None + max_parallel_workers: Optional[int] = None + + def index_param(self) -> AlloyDBIndexParam: + index_parameters = { + "num_leaves": self.num_leaves, "max_num_levels": self.max_num_levels, "quantizer": self.quantizer, + } + return { + "metric": self.parse_metric(), + "index_type": self.index.value, + "index_creation_with_options": self._optionally_build_with_options( + index_parameters + ), + "maintenance_work_mem": self.maintenance_work_mem, + "max_parallel_workers": self.max_parallel_workers, + "enable_pca": self.enable_pca, + } + + def search_param(self) -> AlloyDBSearchParam: + return { + "metric_fun_op": self.parse_metric_fun_op(), + } + + def session_param(self) -> AlloyDBSessionCommands: + session_parameters = { + "scann.num_leaves_to_search": self.num_leaves_to_search, + "scann.max_top_neighbors_buffer_size": self.max_top_neighbors_buffer_size, + "scann.pre_reordering_num_neighbors": self.pre_reordering_num_neighbors, + "scann.num_search_threads": self.num_search_threads, + "scann.max_num_prefetch_datasets": self.max_num_prefetch_datasets, + } + return { + "session_options": self._optionally_build_set_options(session_parameters) + } + + +_alloydb_case_config = { + IndexType.SCANN: AlloyDBScaNNConfig, +} diff --git a/vectordb_bench/backend/clients/api.py b/vectordb_bench/backend/clients/api.py index 0c26fdd3b..da2bed089 100644 --- a/vectordb_bench/backend/clients/api.py +++ b/vectordb_bench/backend/clients/api.py @@ -10,6 +10,7 @@ class MetricType(str, Enum): L2 = "L2" COSINE = "COSINE" IP = "IP" + DP = "DP" HAMMING = "HAMMING" JACCARD = "JACCARD" @@ -27,6 +28,7 @@ class IndexType(str, Enum): GPU_IVF_FLAT = "GPU_IVF_FLAT" GPU_IVF_PQ = "GPU_IVF_PQ" GPU_CAGRA = "GPU_CAGRA" + SCANN = "scann" class DBConfig(ABC, BaseModel): diff --git a/vectordb_bench/cli/vectordbbench.py b/vectordb_bench/cli/vectordbbench.py index 4d23ed952..f9ad69ceb 100644 --- a/vectordb_bench/cli/vectordbbench.py +++ b/vectordb_bench/cli/vectordbbench.py @@ -9,6 +9,7 @@ from ..backend.clients.zilliz_cloud.cli import ZillizAutoIndex from ..backend.clients.milvus.cli import MilvusAutoIndex from ..backend.clients.aws_opensearch.cli import AWSOpenSearch +from ..backend.clients.alloydb.cli import AlloyDBScaNN from .cli import cli @@ -24,6 +25,7 @@ cli.add_command(AWSOpenSearch) cli.add_command(PgVectorScaleDiskAnn) cli.add_command(PgDiskAnn) +cli.add_command(AlloyDBScaNN) if __name__ == "__main__": diff --git a/vectordb_bench/frontend/config/dbCaseConfigs.py b/vectordb_bench/frontend/config/dbCaseConfigs.py index 3db547fef..f8632105e 100644 --- a/vectordb_bench/frontend/config/dbCaseConfigs.py +++ b/vectordb_bench/frontend/config/dbCaseConfigs.py @@ -906,6 +906,141 @@ class CaseConfigInput(BaseModel): == "bit" and config.get(CaseConfigParamType.reranking, False) ) + +CaseConfigParamInput_IndexType_AlloyDB = CaseConfigInput( + label=CaseConfigParamType.IndexType, + inputHelp="Select Index Type", + inputType=InputType.Option, + inputConfig={ + "options": [ + IndexType.SCANN.value, + ], + }, +) + +CaseConfigParamInput_num_leaves_AlloyDB = CaseConfigInput( + label=CaseConfigParamType.numLeaves, + displayLabel="Num Leaves", + inputHelp="The number of partition to apply to this index", + inputType=InputType.Number, + inputConfig={ + "min": 1, + "max": 1048576, + "value": 200, + }, +) + +CaseConfigParamInput_quantizer_AlloyDB = CaseConfigInput( + label=CaseConfigParamType.quantizer, + inputType=InputType.Option, + inputConfig={ + "options": ["SQ8", "Flat"], + }, +) + +CaseConfigParamInput_max_num_levels_AlloyDB = CaseConfigInput( + label=CaseConfigParamType.maxNumLevels, + inputType=InputType.Option, + inputConfig={ + "options": [1, 2], + }, +) + +CaseConfigParamInput_enable_pca_AlloyDB = CaseConfigInput( + label=CaseConfigParamType.enablePca, + inputType=InputType.Option, + inputConfig={ + "options": ["on", "off"], + }, +) + +CaseConfigParamInput_num_leaves_to_search_AlloyDB = CaseConfigInput( + label=CaseConfigParamType.numLeavesToSearch, + displayLabel="Num leaves to search", + inputHelp="The database flag controls the trade off between recall and QPS", + inputType=InputType.Number, + inputConfig={ + "min": 20, + "max": 10486, + "value": 20, + }, +) + +CaseConfigParamInput_max_top_neighbors_buffer_size_AlloyDB = CaseConfigInput( + label=CaseConfigParamType.maxTopNeighborsBufferSize, + displayLabel="Max top neighbors buffer size", + inputHelp="The database flag specifies the size of cache used to improve the \ + performance for filtered queries by scoring or ranking the scanned candidate \ + neighbors in memory instead of the disk", + inputType=InputType.Number, + inputConfig={ + "min": 10000, + "max": 60000, + "value": 20000, + }, +) + +CaseConfigParamInput_pre_reordering_num_neighbors_AlloyDB = CaseConfigInput( + label=CaseConfigParamType.preReorderingNumNeigbors, + displayLabel="Pre reordering num neighbors", + inputHelp="Specifies the number of candidate neighbors to consider during the reordering \ + stages after initial search identifies a set of candidates", + inputType=InputType.Number, + inputConfig={ + "min": 20, + "max": 10486, + "value": 80, + }, +) + +CaseConfigParamInput_num_search_threads_AlloyDB = CaseConfigInput( + label=CaseConfigParamType.numSearchThreads, + displayLabel="Num of searcher threads", + inputHelp="The number of searcher threads for multi-thread search.", + inputType=InputType.Number, + inputConfig={ + "min": 1, + "max": 100, + "value": 2, + }, +) + +CaseConfigParamInput_max_num_prefetch_datasets_AlloyDB = CaseConfigInput( + label=CaseConfigParamType.maxNumPrefetchDatasets, + displayLabel="Max num prefetch datasets", + inputHelp="The maximum number of data batches to prefetch during index search, where batch is a group of buffer pages", + inputType=InputType.Number, + inputConfig={ + "min": 10, + "max": 150, + "value": 100, + }, +) + +CaseConfigParamInput_maintenance_work_mem_AlloyDB = CaseConfigInput( + label=CaseConfigParamType.maintenance_work_mem, + inputHelp="Recommended value: 1.33x the index size, not to exceed the available free memory." + "Specify in gigabytes. e.g. 8GB", + inputType=InputType.Text, + inputConfig={ + "value": "8GB", + }, +) + +CaseConfigParamInput_max_parallel_workers_AlloyDB = CaseConfigInput( + label=CaseConfigParamType.max_parallel_workers, + displayLabel="Max parallel workers", + inputHelp="Recommended value: (cpu cores - 1). This will set the parameters: max_parallel_maintenance_workers," + " max_parallel_workers & table(parallel_workers)", + inputType=InputType.Number, + inputConfig={ + "min": 0, + "max": 1024, + "value": 7, + }, +) + + MilvusLoadConfig = [ CaseConfigParamInput_IndexType, CaseConfigParamInput_M, @@ -1045,6 +1180,33 @@ class CaseConfigInput(BaseModel): CaseConfigParamInput_l_value_is, ] + +AlloyDBLoadConfig = [ + CaseConfigParamInput_IndexType_AlloyDB, + CaseConfigParamInput_num_leaves_AlloyDB, + CaseConfigParamInput_max_num_levels_AlloyDB, + CaseConfigParamInput_enable_pca_AlloyDB, + CaseConfigParamInput_quantizer_AlloyDB, + CaseConfigParamInput_maintenance_work_mem_AlloyDB, + CaseConfigParamInput_max_parallel_workers_AlloyDB, +] + +AlloyDBPerformanceConfig = [ + CaseConfigParamInput_IndexType_AlloyDB, + CaseConfigParamInput_num_leaves_AlloyDB, + CaseConfigParamInput_max_num_levels_AlloyDB, + CaseConfigParamInput_enable_pca_AlloyDB, + CaseConfigParamInput_quantizer_AlloyDB, + CaseConfigParamInput_num_search_threads_AlloyDB, + CaseConfigParamInput_num_leaves_to_search_AlloyDB, + CaseConfigParamInput_max_num_prefetch_datasets_AlloyDB, + CaseConfigParamInput_max_top_neighbors_buffer_size_AlloyDB, + CaseConfigParamInput_pre_reordering_num_neighbors_AlloyDB, + CaseConfigParamInput_maintenance_work_mem_AlloyDB, + CaseConfigParamInput_max_parallel_workers_AlloyDB, +] + + CASE_CONFIG_MAP = { DB.Milvus: { CaseLabel.Load: MilvusLoadConfig, @@ -1081,4 +1243,8 @@ class CaseConfigInput(BaseModel): CaseLabel.Load: PgDiskANNLoadConfig, CaseLabel.Performance: PgDiskANNPerformanceConfig, }, + DB.AlloyDB: { + CaseLabel.Load: AlloyDBLoadConfig, + CaseLabel.Performance: AlloyDBPerformanceConfig, + }, } diff --git a/vectordb_bench/models.py b/vectordb_bench/models.py index 9e064c564..648fb1727 100644 --- a/vectordb_bench/models.py +++ b/vectordb_bench/models.py @@ -76,6 +76,15 @@ class CaseConfigParamType(Enum): num_bits_per_dimension = "num_bits_per_dimension" query_search_list_size = "query_search_list_size" query_rescore = "query_rescore" + numLeaves = "num_leaves" + quantizer = "quantizer" + enablePca = "enable_pca" + maxNumLevels = "max_num_levels" + numLeavesToSearch = "num_leaves_to_search" + maxTopNeighborsBufferSize = "max_top_neighbors_buffer_size" + preReorderingNumNeigbors = "pre_reordering_num_neighbors" + numSearchThreads = "num_search_threads" + maxNumPrefetchDatasets = "max_num_prefetch_datasets" class CustomizedCase(BaseModel): From a64124421333b736b4a652a507a0e3fdd0d39b32 Mon Sep 17 00:00:00 2001 From: Sheharyar Ahmad Date: Mon, 25 Nov 2024 22:24:29 +0500 Subject: [PATCH 110/327] Remove query that set storage to plain. --- vectordb_bench/backend/clients/alloydb/alloydb.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/vectordb_bench/backend/clients/alloydb/alloydb.py b/vectordb_bench/backend/clients/alloydb/alloydb.py index c053c2fad..5b275b30f 100644 --- a/vectordb_bench/backend/clients/alloydb/alloydb.py +++ b/vectordb_bench/backend/clients/alloydb/alloydb.py @@ -308,11 +308,6 @@ def _create_table(self, dim: int): "CREATE TABLE IF NOT EXISTS public.{table_name} (id BIGINT PRIMARY KEY, embedding vector({dim}));" ).format(table_name=sql.Identifier(self.table_name), dim=dim) ) - self.cursor.execute( - sql.SQL( - "ALTER TABLE public.{table_name} ALTER COLUMN embedding SET STORAGE PLAIN;" - ).format(table_name=sql.Identifier(self.table_name)) - ) self.conn.commit() except Exception as e: log.warning( From 6a1477a5c79182e9174450c5e67c9814081134b9 Mon Sep 17 00:00:00 2001 From: Sheharyar Ahmad Date: Mon, 25 Nov 2024 22:29:11 +0500 Subject: [PATCH 111/327] Add default value for pre_reordering_num_neighbors in cli options. --- vectordb_bench/backend/clients/alloydb/cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vectordb_bench/backend/clients/alloydb/cli.py b/vectordb_bench/backend/clients/alloydb/cli.py index ab49da955..aa3a5cbe8 100644 --- a/vectordb_bench/backend/clients/alloydb/cli.py +++ b/vectordb_bench/backend/clients/alloydb/cli.py @@ -70,8 +70,8 @@ class AlloyDBScaNNTypedDict(AlloyDBTypedDict): click.option("--num-leaves-to-search", type=int, help="Number of leaves to search", required=True) ] pre_reordering_num_neighbors: Annotated[ - Optional[int], - click.option("--pre-reordering-num-neighbors", type=int, help="Pre-reordering number of neighbors",) + int, + click.option("--pre-reordering-num-neighbors", type=int, help="Pre-reordering number of neighbors", default=200) ] max_top_neighbors_buffer_size: Annotated[ int, From b0553bc078b416da960cb660081f656662da4876 Mon Sep 17 00:00:00 2001 From: yangxuan Date: Thu, 28 Nov 2024 16:28:17 +0800 Subject: [PATCH 112/327] fix: Donot refresh load Signed-off-by: yangxuan --- vectordb_bench/backend/clients/milvus/milvus.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vectordb_bench/backend/clients/milvus/milvus.py b/vectordb_bench/backend/clients/milvus/milvus.py index 8f8571aa3..7267e06ee 100644 --- a/vectordb_bench/backend/clients/milvus/milvus.py +++ b/vectordb_bench/backend/clients/milvus/milvus.py @@ -93,7 +93,7 @@ def _optimize(self): self._post_insert() log.info(f"{self.name} optimizing before search") try: - self.col.load(refresh=True) + self.col.load() except Exception as e: log.warning(f"{self.name} optimize error: {e}") raise e from None From c6a2e792fb7ae5d769e64c291258d6ee06dd65fc Mon Sep 17 00:00:00 2001 From: Sheharyar Ahmad Date: Fri, 29 Nov 2024 18:59:44 +0500 Subject: [PATCH 113/327] fix: invalid value for --max-num-levels when using CLI. --- vectordb_bench/backend/clients/alloydb/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vectordb_bench/backend/clients/alloydb/cli.py b/vectordb_bench/backend/clients/alloydb/cli.py index aa3a5cbe8..54d9b9fa3 100644 --- a/vectordb_bench/backend/clients/alloydb/cli.py +++ b/vectordb_bench/backend/clients/alloydb/cli.py @@ -106,7 +106,7 @@ class AlloyDBScaNNTypedDict(AlloyDBTypedDict): int, click.option( "--max-num-levels", - type=click.Choice([1, 2]), + type=click.Choice(["1", "2"]), help="Maximum number of levels", default=1 ) From 2250e62b5aa3e77390ef16a65e01ea6881248bb2 Mon Sep 17 00:00:00 2001 From: Teynar <97400690+teynar@users.noreply.github.com> Date: Wed, 4 Dec 2024 03:09:57 +0100 Subject: [PATCH 114/327] Add Milvus auth support through user_name and password fields (#416) * tweak(milvus): add auth support through user_name and password fields (like zilliz cloud) * tweak(frontend): make username and password for milvus optional fields --- vectordb_bench/backend/clients/milvus/cli.py | 26 ++++++++++++++++++- .../backend/clients/milvus/config.py | 18 +++++++++++-- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/vectordb_bench/backend/clients/milvus/cli.py b/vectordb_bench/backend/clients/milvus/cli.py index 05ff95418..885995def 100644 --- a/vectordb_bench/backend/clients/milvus/cli.py +++ b/vectordb_bench/backend/clients/milvus/cli.py @@ -1,4 +1,4 @@ -from typing import Annotated, TypedDict, Unpack +from typing import Annotated, TypedDict, Unpack, Optional import click from pydantic import SecretStr @@ -21,6 +21,12 @@ class MilvusTypedDict(TypedDict): uri: Annotated[ str, click.option("--uri", type=str, help="uri connection string", required=True) ] + user_name: Annotated[ + Optional[str], click.option("--user-name", type=str, help="Db username", required=False) + ] + password: Annotated[ + Optional[str], click.option("--password", type=str, help="Db password", required=False) + ] class MilvusAutoIndexTypedDict(CommonTypedDict, MilvusTypedDict): @@ -37,6 +43,8 @@ def MilvusAutoIndex(**parameters: Unpack[MilvusAutoIndexTypedDict]): db_config=MilvusConfig( db_label=parameters["db_label"], uri=SecretStr(parameters["uri"]), + user=parameters["user_name"], + password=SecretStr(parameters["password"]), ), db_case_config=AutoIndexConfig(), **parameters, @@ -53,6 +61,8 @@ def MilvusFlat(**parameters: Unpack[MilvusAutoIndexTypedDict]): db_config=MilvusConfig( db_label=parameters["db_label"], uri=SecretStr(parameters["uri"]), + user=parameters["user_name"], + password=SecretStr(parameters["password"]), ), db_case_config=FLATConfig(), **parameters, @@ -73,6 +83,8 @@ def MilvusHNSW(**parameters: Unpack[MilvusHNSWTypedDict]): db_config=MilvusConfig( db_label=parameters["db_label"], uri=SecretStr(parameters["uri"]), + user=parameters["user_name"], + password=SecretStr(parameters["password"]) if parameters["password"] else None, ), db_case_config=HNSWConfig( M=parameters["m"], @@ -97,6 +109,8 @@ def MilvusIVFFlat(**parameters: Unpack[MilvusIVFFlatTypedDict]): db_config=MilvusConfig( db_label=parameters["db_label"], uri=SecretStr(parameters["uri"]), + user=parameters["user_name"], + password=SecretStr(parameters["password"]), ), db_case_config=IVFFlatConfig( nlist=parameters["nlist"], @@ -116,6 +130,8 @@ def MilvusIVFSQ8(**parameters: Unpack[MilvusIVFFlatTypedDict]): db_config=MilvusConfig( db_label=parameters["db_label"], uri=SecretStr(parameters["uri"]), + user=parameters["user_name"], + password=SecretStr(parameters["password"]), ), db_case_config=IVFSQ8Config( nlist=parameters["nlist"], @@ -143,6 +159,8 @@ def MilvusDISKANN(**parameters: Unpack[MilvusDISKANNTypedDict]): db_config=MilvusConfig( db_label=parameters["db_label"], uri=SecretStr(parameters["uri"]), + user=parameters["user_name"], + password=SecretStr(parameters["password"]), ), db_case_config=DISKANNConfig( search_list=parameters["search_list"], @@ -174,6 +192,8 @@ def MilvusGPUIVFFlat(**parameters: Unpack[MilvusGPUIVFTypedDict]): db_config=MilvusConfig( db_label=parameters["db_label"], uri=SecretStr(parameters["uri"]), + user=parameters["user_name"], + password=SecretStr(parameters["password"]), ), db_case_config=GPUIVFFlatConfig( nlist=parameters["nlist"], @@ -208,6 +228,8 @@ def MilvusGPUIVFPQ(**parameters: Unpack[MilvusGPUIVFPQTypedDict]): db_config=MilvusConfig( db_label=parameters["db_label"], uri=SecretStr(parameters["uri"]), + user=parameters["user_name"], + password=SecretStr(parameters["password"]), ), db_case_config=GPUIVFPQConfig( nlist=parameters["nlist"], @@ -274,6 +296,8 @@ def MilvusGPUCAGRA(**parameters: Unpack[MilvusGPUCAGRATypedDict]): db_config=MilvusConfig( db_label=parameters["db_label"], uri=SecretStr(parameters["uri"]), + user=parameters["user_name"], + password=SecretStr(parameters["password"]), ), db_case_config=GPUCAGRAConfig( intermediate_graph_degree=parameters["intermediate_graph_degree"], diff --git a/vectordb_bench/backend/clients/milvus/config.py b/vectordb_bench/backend/clients/milvus/config.py index eea7b11f5..059ef0461 100644 --- a/vectordb_bench/backend/clients/milvus/config.py +++ b/vectordb_bench/backend/clients/milvus/config.py @@ -1,12 +1,26 @@ -from pydantic import BaseModel, SecretStr +from pydantic import BaseModel, SecretStr, validator from ..api import DBConfig, DBCaseConfig, MetricType, IndexType class MilvusConfig(DBConfig): uri: SecretStr = "http://localhost:19530" + user: str | None = None + password: SecretStr | None = None def to_dict(self) -> dict: - return {"uri": self.uri.get_secret_value()} + return { + "uri": self.uri.get_secret_value(), + "user": self.user if self.user else None, + "password": self.password.get_secret_value() if self.password else None, + } + + @validator("*") + def not_empty_field(cls, v, field): + if field.name in cls.common_short_configs() or field.name in cls.common_long_configs() or field.name in ["user", "password"]: + return v + if isinstance(v, (str, SecretStr)) and len(v) == 0: + raise ValueError("Empty string!") + return v class MilvusIndexConfig(BaseModel): From fa14f04b737884d4c5c8e831b72cc9fd66a055e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=A2=E5=B0=91=E6=95=8F?= Date: Wed, 11 Dec 2024 09:45:39 +0800 Subject: [PATCH 115/327] support alibaba cloud elasticsearch (#418) * add aliyun elasticsearch * code reuse --- vectordb_bench/backend/clients/__init__.py | 13 +++++++ .../aliyun_elasticsearch.py | 27 +++++++++++++ .../clients/aliyun_elasticsearch/config.py | 19 +++++++++ .../frontend/config/dbCaseConfigs.py | 39 +++++++++++++++++++ 4 files changed, 98 insertions(+) create mode 100644 vectordb_bench/backend/clients/aliyun_elasticsearch/aliyun_elasticsearch.py create mode 100644 vectordb_bench/backend/clients/aliyun_elasticsearch/config.py diff --git a/vectordb_bench/backend/clients/__init__.py b/vectordb_bench/backend/clients/__init__.py index 8859381b5..ba78c35cc 100644 --- a/vectordb_bench/backend/clients/__init__.py +++ b/vectordb_bench/backend/clients/__init__.py @@ -37,6 +37,7 @@ class DB(Enum): MemoryDB = "MemoryDB" Chroma = "Chroma" AWSOpenSearch = "OpenSearch" + AliyunElasticsearch = "AliyunElasticsearch" Test = "test" @@ -103,6 +104,10 @@ def init_cls(self) -> Type[VectorDB]: from .alloydb.alloydb import AlloyDB return AlloyDB + if self == DB.AliyunElasticsearch: + from .aliyun_elasticsearch.aliyun_elasticsearch import AliyunElasticsearch + return AliyunElasticsearch + @property def config_cls(self) -> Type[DBConfig]: """Import while in use""" @@ -166,6 +171,10 @@ def config_cls(self) -> Type[DBConfig]: from .alloydb.config import AlloyDBConfig return AlloyDBConfig + if self == DB.AliyunElasticsearch: + from .aliyun_elasticsearch.config import AliyunElasticsearchConfig + return AliyunElasticsearchConfig + def case_config_cls(self, index_type: IndexType | None = None) -> Type[DBCaseConfig]: if self == DB.Milvus: from .milvus.config import _milvus_case_config @@ -211,6 +220,10 @@ def case_config_cls(self, index_type: IndexType | None = None) -> Type[DBCaseCon from .alloydb.config import _alloydb_case_config return _alloydb_case_config.get(index_type) + if self == DB.AliyunElasticsearch: + from .elastic_cloud.config import ElasticCloudIndexConfig + return ElasticCloudIndexConfig + # DB.Pinecone, DB.Chroma, DB.Redis return EmptyDBCaseConfig diff --git a/vectordb_bench/backend/clients/aliyun_elasticsearch/aliyun_elasticsearch.py b/vectordb_bench/backend/clients/aliyun_elasticsearch/aliyun_elasticsearch.py new file mode 100644 index 000000000..41253ca1e --- /dev/null +++ b/vectordb_bench/backend/clients/aliyun_elasticsearch/aliyun_elasticsearch.py @@ -0,0 +1,27 @@ +from ..elastic_cloud.elastic_cloud import ElasticCloud +from ..elastic_cloud.config import ElasticCloudIndexConfig + + +class AliyunElasticsearch(ElasticCloud): + def __init__( + self, + dim: int, + db_config: dict, + db_case_config: ElasticCloudIndexConfig, + indice: str = "vdb_bench_indice", # must be lowercase + id_col_name: str = "id", + vector_col_name: str = "vector", + drop_old: bool = False, + **kwargs, + ): + super().__init__( + dim=dim, + db_config=db_config, + db_case_config=db_case_config, + indice=indice, + id_col_name=id_col_name, + vector_col_name=vector_col_name, + drop_old=drop_old, + **kwargs, + ) + diff --git a/vectordb_bench/backend/clients/aliyun_elasticsearch/config.py b/vectordb_bench/backend/clients/aliyun_elasticsearch/config.py new file mode 100644 index 000000000..a2de4dc75 --- /dev/null +++ b/vectordb_bench/backend/clients/aliyun_elasticsearch/config.py @@ -0,0 +1,19 @@ +from enum import Enum +from pydantic import SecretStr, BaseModel + +from ..api import DBConfig, DBCaseConfig, MetricType, IndexType + + +class AliyunElasticsearchConfig(DBConfig, BaseModel): + #: Protocol in use to connect to the node + scheme: str = "http" + host: str = "" + port: int = 9200 + user: str = "elastic" + password: SecretStr + + def to_dict(self) -> dict: + return { + "hosts": [{'scheme': self.scheme, 'host': self.host, 'port': self.port}], + "basic_auth": (self.user, self.password.get_secret_value()), + } diff --git a/vectordb_bench/frontend/config/dbCaseConfigs.py b/vectordb_bench/frontend/config/dbCaseConfigs.py index f8632105e..88794e30f 100644 --- a/vectordb_bench/frontend/config/dbCaseConfigs.py +++ b/vectordb_bench/frontend/config/dbCaseConfigs.py @@ -1040,6 +1040,35 @@ class CaseConfigInput(BaseModel): }, ) +CaseConfigParamInput_EFConstruction_AliES = CaseConfigInput( + label=CaseConfigParamType.EFConstruction, + inputType=InputType.Number, + inputConfig={ + "min": 8, + "max": 512, + "value": 360, + }, +) + +CaseConfigParamInput_M_AliES = CaseConfigInput( + label=CaseConfigParamType.M, + inputType=InputType.Number, + inputConfig={ + "min": 4, + "max": 64, + "value": 30, + }, +) +CaseConfigParamInput_NumCandidates_AliES = CaseConfigInput( + label=CaseConfigParamType.numCandidates, + inputType=InputType.Number, + inputConfig={ + "min": 1, + "max": 10000, + "value": 100, + }, +) + MilvusLoadConfig = [ CaseConfigParamInput_IndexType, @@ -1206,6 +1235,12 @@ class CaseConfigInput(BaseModel): CaseConfigParamInput_max_parallel_workers_AlloyDB, ] +AliyunElasticsearchLoadingConfig = [CaseConfigParamInput_EFConstruction_AliES, CaseConfigParamInput_M_AliES] +AliyunElasticsearchPerformanceConfig = [ + CaseConfigParamInput_EFConstruction_AliES, + CaseConfigParamInput_M_AliES, + CaseConfigParamInput_NumCandidates_AliES, +] CASE_CONFIG_MAP = { DB.Milvus: { @@ -1247,4 +1282,8 @@ class CaseConfigInput(BaseModel): CaseLabel.Load: AlloyDBLoadConfig, CaseLabel.Performance: AlloyDBPerformanceConfig, }, + DB.AliyunElasticsearch: { + CaseLabel.Load: AliyunElasticsearchLoadingConfig, + CaseLabel.Performance: AliyunElasticsearchPerformanceConfig, + }, } From 0f9d9c8021926cd4cd95e53e1b59ae8e910b621f Mon Sep 17 00:00:00 2001 From: yangxuan Date: Tue, 10 Dec 2024 16:39:41 +0800 Subject: [PATCH 116/327] enhance: refine read write cases 1. Control search time during 2 insert stages, make sure search donesn't shift away from insert proportions. 2. Collect ndcg metric 3. Optimize, serial search and conc search after insertion Signed-off-by: yangxuan --- tests/test_rate_runner.py | 6 +- .../backend/clients/milvus/milvus.py | 12 +- vectordb_bench/backend/runner/rate_runner.py | 47 ++++-- .../backend/runner/read_write_runner.py | 138 +++++++++++++----- .../backend/runner/serial_runner.py | 10 +- vectordb_bench/backend/runner/util.py | 16 -- vectordb_bench/backend/utils.py | 1 + 7 files changed, 151 insertions(+), 79 deletions(-) diff --git a/tests/test_rate_runner.py b/tests/test_rate_runner.py index 363b7a33c..df92b0dd7 100644 --- a/tests/test_rate_runner.py +++ b/tests/test_rate_runner.py @@ -52,9 +52,9 @@ def test_read_write_runner(db, insert_rate, conc: list, search_stage: Iterable[f def get_db(db: str, config: dict) -> VectorDB: if db == DB.Milvus.name: - return DB.Milvus.init_cls(dim=768, db_config=config, db_case_config=FLATConfig(metric_type="COSINE"), drop_old=True, pre_load=True) + return DB.Milvus.init_cls(dim=768, db_config=config, db_case_config=FLATConfig(metric_type="COSINE"), drop_old=True) elif db == DB.ZillizCloud.name: - return DB.ZillizCloud.init_cls(dim=768, db_config=config, db_case_config=AutoIndexConfig(metric_type="COSINE"), drop_old=True, pre_load=True) + return DB.ZillizCloud.init_cls(dim=768, db_config=config, db_case_config=AutoIndexConfig(metric_type="COSINE"), drop_old=True) else: raise ValueError(f"unknown db: {db}") @@ -76,7 +76,7 @@ def get_db(db: str, config: dict) -> VectorDB: } conc = (1, 15, 50) - search_stage = (0.5, 0.6, 0.7, 0.8, 0.9, 1.0) + search_stage = (0.5, 0.6, 0.7, 0.8, 0.9) db = get_db(flags.db, config) test_read_write_runner( diff --git a/vectordb_bench/backend/clients/milvus/milvus.py b/vectordb_bench/backend/clients/milvus/milvus.py index 7267e06ee..251dee8ad 100644 --- a/vectordb_bench/backend/clients/milvus/milvus.py +++ b/vectordb_bench/backend/clients/milvus/milvus.py @@ -8,7 +8,7 @@ from pymilvus import Collection, utility from pymilvus import CollectionSchema, DataType, FieldSchema, MilvusException -from ..api import VectorDB, IndexType +from ..api import VectorDB from .config import MilvusIndexConfig @@ -66,8 +66,7 @@ def __init__( self.case_config.index_param(), index_name=self._index_name, ) - if kwargs.get("pre_load") is True: - self._pre_load(col) + col.load() connections.disconnect("default") @@ -90,16 +89,15 @@ def init(self) -> None: connections.disconnect("default") def _optimize(self): - self._post_insert() log.info(f"{self.name} optimizing before search") + self._post_insert() try: - self.col.load() + self.col.load(refresh=True) except Exception as e: log.warning(f"{self.name} optimize error: {e}") raise e from None def _post_insert(self): - log.info(f"{self.name} post insert before optimize") try: self.col.flush() # wait for index done and load refresh @@ -130,7 +128,7 @@ def wait_index(): log.warning(f"{self.name} compact error: {e}") if hasattr(e, 'code'): if e.code().name == 'PERMISSION_DENIED': - log.warning(f"Skip compact due to permission denied.") + log.warning("Skip compact due to permission denied.") pass else: raise e diff --git a/vectordb_bench/backend/runner/rate_runner.py b/vectordb_bench/backend/runner/rate_runner.py index 4b6d7f6cf..d77c0fd15 100644 --- a/vectordb_bench/backend/runner/rate_runner.py +++ b/vectordb_bench/backend/runner/rate_runner.py @@ -1,5 +1,6 @@ import logging import time +import concurrent from concurrent.futures import ThreadPoolExecutor import multiprocessing as mp @@ -9,7 +10,7 @@ from vectordb_bench.backend.utils import time_it from vectordb_bench import config -from .util import get_data, is_futures_completed, get_future_exceptions +from .util import get_data log = logging.getLogger(__name__) @@ -54,26 +55,42 @@ def submit_by_rate() -> bool: start_time = time.perf_counter() finished, elapsed_time = submit_by_rate() if finished is True: - q.put(None, block=True) + q.put(True, block=True) log.info(f"End of dataset, left unfinished={len(executing_futures)}") - return + break - q.put(True, block=False) + q.put(False, block=False) wait_interval = 1 - elapsed_time if elapsed_time < 1 else 0.001 - e, completed = is_futures_completed(executing_futures, wait_interval) - if completed is True: - ex = get_future_exceptions(executing_futures) - if ex is not None: - log.warn(f"task error, terminating, err={ex}") - q.put(None) - executor.shutdown(wait=True, cancel_futures=True) - raise ex + try: + done, not_done = concurrent.futures.wait( + executing_futures, + timeout=wait_interval, + return_when=concurrent.futures.FIRST_EXCEPTION) + + if len(not_done) > 0: + log.warning(f"Failed to finish all tasks in 1s, [{len(not_done)}/{len(executing_futures)}] tasks are not done, waited={wait_interval:.2f}, trying to wait in the next round") + executing_futures = list(not_done) else: log.debug(f"Finished {len(executing_futures)} insert-{config.NUM_PER_BATCH} task in 1s, wait_interval={wait_interval:.2f}") - executing_futures = [] - else: - log.warning(f"Failed to finish tasks in 1s, {e}, waited={wait_interval:.2f}, try to check the next round") + executing_futures = [] + except Exception as e: + log.warn(f"task error, terminating, err={e}") + q.put(None, block=True) + executor.shutdown(wait=True, cancel_futures=True) + raise e + dur = time.perf_counter() - start_time if dur < 1: time.sleep(1 - dur) + + # wait for all tasks in executing_futures to complete + if len(executing_futures) > 0: + try: + done, _ = concurrent.futures.wait(executing_futures, + return_when=concurrent.futures.FIRST_EXCEPTION) + except Exception as e: + log.warn(f"task error, terminating, err={e}") + q.put(None, block=True) + executor.shutdown(wait=True, cancel_futures=True) + raise e diff --git a/vectordb_bench/backend/runner/read_write_runner.py b/vectordb_bench/backend/runner/read_write_runner.py index 6e043dceb..fd425b227 100644 --- a/vectordb_bench/backend/runner/read_write_runner.py +++ b/vectordb_bench/backend/runner/read_write_runner.py @@ -24,7 +24,7 @@ def __init__( k: int = 100, filters: dict | None = None, concurrencies: Iterable[int] = (1, 15, 50), - search_stage: Iterable[float] = (0.5, 0.6, 0.7, 0.8, 0.9, 1.0), # search in any insert portion, 0.0 means search from the start + search_stage: Iterable[float] = (0.5, 0.6, 0.7, 0.8, 0.9), # search from insert portion, 0.0 means search from the start read_dur_after_write: int = 300, # seconds, search duration when insertion is done timeout: float | None = None, ): @@ -32,7 +32,7 @@ def __init__( self.data_volume = dataset.data.size for stage in search_stage: - assert 0.0 <= stage <= 1.0, "each search stage should be in [0.0, 1.0]" + assert 0.0 <= stage < 1.0, "each search stage should be in [0.0, 1.0)" self.search_stage = sorted(search_stage) self.read_dur_after_write = read_dur_after_write @@ -65,48 +65,114 @@ def __init__( k=k, ) + def run_optimize(self): + """Optimize needs to run in differenct process for pymilvus schema recursion problem""" + with self.db.init(): + log.info("Search after write - Optimize start") + self.db.optimize() + log.info("Search after write - Optimize finished") + + def run_search(self): + log.info("Search after write - Serial search start") + res, ssearch_dur = self.serial_search_runner.run() + recall, ndcg, p99_latency = res + log.info(f"Search after write - Serial search - recall={recall}, ndcg={ndcg}, p99={p99_latency}, dur={ssearch_dur:.4f}") + log.info(f"Search after wirte - Conc search start, dur for each conc={self.read_dur_after_write}") + max_qps = self.run_by_dur(self.read_dur_after_write) + log.info(f"Search after wirte - Conc search finished, max_qps={max_qps}") + + return (max_qps, recall, ndcg, p99_latency) + def run_read_write(self): - futures = [] with mp.Manager() as m: q = m.Queue() with concurrent.futures.ProcessPoolExecutor(mp_context=mp.get_context("spawn"), max_workers=2) as executor: - futures.append(executor.submit(self.run_with_rate, q)) - futures.append(executor.submit(self.run_search_by_sig, q)) - - for future in concurrent.futures.as_completed(futures): - res = future.result() - log.info(f"Result = {res}") - + read_write_futures = [] + read_write_futures.append(executor.submit(self.run_with_rate, q)) + read_write_futures.append(executor.submit(self.run_search_by_sig, q)) + + try: + for f in concurrent.futures.as_completed(read_write_futures): + res = f.result() + log.info(f"Result = {res}") + + # Wait for read_write_futures finishing and do optimize and search + op_future = executor.submit(self.run_optimize) + op_future.result() + + search_future = executor.submit(self.run_search) + last_res = search_future.result() + + log.info(f"Max QPS after optimze and search: {last_res}") + except Exception as e: + log.warning(f"Read and write error: {e}") + executor.shutdown(wait=True, cancel_futures=True) + raise e log.info("Concurrent read write all done") - def run_search_by_sig(self, q): - res = [] + """ + Args: + q: multiprocessing queue + (None) means abnormal exit + (False) means updating progress + (True) means normal exit + """ + result, start_batch = [], 0 total_batch = math.ceil(self.data_volume / self.insert_rate) - batch = 0 - recall = 'x' + recall, ndcg, p99_latency = None, None, None + + def wait_next_target(start, target_batch) -> bool: + """Return False when receive True or None""" + while start < target_batch: + sig = q.get(block=True) + + if sig is None or sig is True: + return False + else: + start += 1 + return True for idx, stage in enumerate(self.search_stage): target_batch = int(total_batch * stage) - while q.get(block=True): - batch += 1 - if batch >= target_batch: - perc = int(stage * 100) - log.info(f"Insert {perc}% done, total batch={total_batch}") - log.info(f"[{batch}/{total_batch}] Serial search - {perc}% start") - recall, ndcg, p99 =self.serial_search_runner.run() - - if idx < len(self.search_stage) - 1: - stage_search_dur = (self.data_volume * (self.search_stage[idx + 1] - stage) // self.insert_rate) // len(self.concurrencies) - if stage_search_dur < 30: - log.warning(f"Search duration too short, please reduce concurrency count or insert rate, or increase dataset volume: dur={stage_search_dur}, concurrencies={len(self.concurrencies)}, insert_rate={self.insert_rate}") - log.info(f"[{batch}/{total_batch}] Conc search - {perc}% start, dur for each conc={stage_search_dur}s") - else: - last_search_dur = self.data_volume * (1.0 - stage) // self.insert_rate - stage_search_dur = last_search_dur + self.read_dur_after_write - log.info(f"[{batch}/{total_batch}] Last conc search - {perc}% start, [read_until_write|read_after_write|total] =[{last_search_dur}s|{self.read_dur_after_write}s|{stage_search_dur}s]") - - max_qps = self.run_by_dur(stage_search_dur) - res.append((perc, max_qps, recall)) - break - return res + perc = int(stage * 100) + + got = wait_next_target(start_batch, target_batch) + if got is False: + log.warning(f"Abnormal exit, target_batch={target_batch}, start_batch={start_batch}") + return + + log.info(f"Insert {perc}% done, total batch={total_batch}") + log.info(f"[{target_batch}/{total_batch}] Serial search - {perc}% start") + res, ssearch_dur = self.serial_search_runner.run() + recall, ndcg, p99_latency = res + log.info(f"[{target_batch}/{total_batch}] Serial search - {perc}% done, recall={recall}, ndcg={ndcg}, p99={p99_latency}, dur={ssearch_dur:.4f}") + + # Search duration for non-last search stage is carefully calculated. + # If duration for each concurrency is less than 30s, runner will raise error. + if idx < len(self.search_stage) - 1: + total_dur_between_stages = self.data_volume * (self.search_stage[idx + 1] - stage) // self.insert_rate + csearch_dur = total_dur_between_stages - ssearch_dur + + # Try to leave room for init process executors + csearch_dur = csearch_dur - 30 if csearch_dur > 60 else csearch_dur + + each_conc_search_dur = csearch_dur / len(self.concurrencies) + if each_conc_search_dur < 30: + warning_msg = f"Results might be inaccurate, duration[{csearch_dur:.4f}] left for conc-search is too short, total available dur={total_dur_between_stages}, serial_search_cost={ssearch_dur}." + log.warning(warning_msg) + + # The last stage + else: + each_conc_search_dur = 60 + + log.info(f"[{target_batch}/{total_batch}] Concurrent search - {perc}% start, dur={each_conc_search_dur:.4f}") + max_qps = self.run_by_dur(each_conc_search_dur) + result.append((perc, max_qps, recall, ndcg, p99_latency)) + + start_batch = target_batch + + # Drain the queue + while q.empty() is False: + q.get(block=True) + return result diff --git a/vectordb_bench/backend/runner/serial_runner.py b/vectordb_bench/backend/runner/serial_runner.py index 9e6818443..13270e21a 100644 --- a/vectordb_bench/backend/runner/serial_runner.py +++ b/vectordb_bench/backend/runner/serial_runner.py @@ -167,7 +167,7 @@ def __init__( self.test_data = test_data self.ground_truth = ground_truth - def search(self, args: tuple[list, pd.DataFrame]): + def search(self, args: tuple[list, pd.DataFrame]) -> tuple[float, float, float]: log.info(f"{mp.current_process().name:14} start search the entire test_data to get recall and latency") with self.db.init(): test_data, ground_truth = args @@ -224,5 +224,11 @@ def _run_in_subprocess(self) -> tuple[float, float]: result = future.result() return result - def run(self) -> tuple[float, float]: + @utils.time_it + def run(self) -> tuple[float, float, float]: + """ + Returns: + tuple[tuple[float, float, float], float]: (avg_recall, avg_ndcg, p99_latency), cost + + """ return self._run_in_subprocess() diff --git a/vectordb_bench/backend/runner/util.py b/vectordb_bench/backend/runner/util.py index 0dfd9d0c4..ba1888167 100644 --- a/vectordb_bench/backend/runner/util.py +++ b/vectordb_bench/backend/runner/util.py @@ -1,6 +1,4 @@ import logging -import concurrent -from typing import Iterable from pandas import DataFrame import numpy as np @@ -16,17 +14,3 @@ def get_data(data_df: DataFrame, normalize: bool) -> tuple[list[list[float]], li else: all_embeddings = emb_np.tolist() return all_embeddings, all_metadata - -def is_futures_completed(futures: Iterable[concurrent.futures.Future], interval) -> (Exception, bool): - try: - list(concurrent.futures.as_completed(futures, timeout=interval)) - except TimeoutError as e: - return e, False - return None, True - - -def get_future_exceptions(futures: Iterable[concurrent.futures.Future]) -> BaseException | None: - for f in futures: - if f.exception() is not None: - return f.exception() - return diff --git a/vectordb_bench/backend/utils.py b/vectordb_bench/backend/utils.py index 690cff3e8..ea11e6461 100644 --- a/vectordb_bench/backend/utils.py +++ b/vectordb_bench/backend/utils.py @@ -35,6 +35,7 @@ def numerize(n) -> str: def time_it(func): + """ returns result and elapsed time""" @wraps(func) def inner(*args, **kwargs): pref = time.perf_counter() From 185541aa8ee4e3e81f1484e3f1b991dc5bff614f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=85=9A=E5=8F=82?= Date: Tue, 19 Nov 2024 20:19:30 +0800 Subject: [PATCH 117/327] add aliyun opensearch client --- vectordb_bench/backend/cases.py | 2 +- vectordb_bench/backend/clients/__init__.py | 13 + .../aliyun_opensearch/aliyun_opensearch.py | 304 ++++++++++++++++++ .../clients/aliyun_opensearch/config.py | 48 +++ vectordb_bench/backend/clients/api.py | 3 + vectordb_bench/backend/task_runner.py | 2 +- .../frontend/config/dbCaseConfigs.py | 19 ++ 7 files changed, 389 insertions(+), 2 deletions(-) create mode 100644 vectordb_bench/backend/clients/aliyun_opensearch/aliyun_opensearch.py create mode 100644 vectordb_bench/backend/clients/aliyun_opensearch/config.py diff --git a/vectordb_bench/backend/cases.py b/vectordb_bench/backend/cases.py index 6c43bb910..8b643b66b 100644 --- a/vectordb_bench/backend/cases.py +++ b/vectordb_bench/backend/cases.py @@ -289,7 +289,7 @@ class Performance1536D50K(PerformanceCase): description: str = """This case tests the search performance of a vector database with a medium 50K dataset (OpenAI 50K vectors, 1536 dimensions), at varying parallel levels. Results will show index building time, recall, and maximum QPS.""" load_timeout: float | int = 3600 - optimize_timeout: float | int | None = 15 * 60 + optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_DEFAULT def metric_type_map(s: str) -> MetricType: diff --git a/vectordb_bench/backend/clients/__init__.py b/vectordb_bench/backend/clients/__init__.py index ba78c35cc..e1b66a81d 100644 --- a/vectordb_bench/backend/clients/__init__.py +++ b/vectordb_bench/backend/clients/__init__.py @@ -39,6 +39,7 @@ class DB(Enum): AWSOpenSearch = "OpenSearch" AliyunElasticsearch = "AliyunElasticsearch" Test = "test" + AliyunOpenSearch = "AliyunOpenSearch" @property @@ -108,6 +109,10 @@ def init_cls(self) -> Type[VectorDB]: from .aliyun_elasticsearch.aliyun_elasticsearch import AliyunElasticsearch return AliyunElasticsearch + if self == DB.AliyunOpenSearch: + from .aliyun_opensearch.aliyun_opensearch import AliyunOpenSearch + return AliyunOpenSearch + @property def config_cls(self) -> Type[DBConfig]: """Import while in use""" @@ -175,6 +180,10 @@ def config_cls(self) -> Type[DBConfig]: from .aliyun_elasticsearch.config import AliyunElasticsearchConfig return AliyunElasticsearchConfig + if self == DB.AliyunOpenSearch: + from .aliyun_opensearch.config import AliyunOpenSearchConfig + return AliyunOpenSearchConfig + def case_config_cls(self, index_type: IndexType | None = None) -> Type[DBCaseConfig]: if self == DB.Milvus: from .milvus.config import _milvus_case_config @@ -224,6 +233,10 @@ def case_config_cls(self, index_type: IndexType | None = None) -> Type[DBCaseCon from .elastic_cloud.config import ElasticCloudIndexConfig return ElasticCloudIndexConfig + if self == DB.AliyunOpenSearch: + from .aliyun_opensearch.config import AliyunOpenSearchIndexConfig + return AliyunOpenSearchIndexConfig + # DB.Pinecone, DB.Chroma, DB.Redis return EmptyDBCaseConfig diff --git a/vectordb_bench/backend/clients/aliyun_opensearch/aliyun_opensearch.py b/vectordb_bench/backend/clients/aliyun_opensearch/aliyun_opensearch.py new file mode 100644 index 000000000..5d4dcbfa6 --- /dev/null +++ b/vectordb_bench/backend/clients/aliyun_opensearch/aliyun_opensearch.py @@ -0,0 +1,304 @@ +import json +import logging +from contextlib import contextmanager +import time + +from alibabacloud_ha3engine_vector.models import QueryRequest + +from ..api import VectorDB, MetricType +from .config import AliyunOpenSearchIndexConfig + +from alibabacloud_searchengine20211025.client import Client as searchengineClient +from alibabacloud_searchengine20211025 import models as searchengine_models +from alibabacloud_tea_openapi import models as open_api_models +from alibabacloud_ha3engine_vector import models, client + +log = logging.getLogger(__name__) + +ALIYUN_OPENSEARCH_MAX_SIZE_PER_BATCH = 2 * 1024 * 1024 # 2MB +ALIYUN_OPENSEARCH_MAX_NUM_PER_BATCH = 100 + +class AliyunOpenSearch(VectorDB): + def __init__( + self, + dim: int, + db_config: dict, + db_case_config: AliyunOpenSearchIndexConfig, + collection_name: str = "VectorDBBenchCollection", + drop_old: bool = False, + **kwargs, + ): + self.control_client = None + self.dim = dim + self.db_config = db_config + self.case_config = db_case_config + self.collection_name = collection_name + self.instance_id = db_config["host"].split(".")[0].replace("http://", "").replace("https://", "") + + self._primary_field = "id" + self._scalar_field = "int_id" + self._vector_field = "vector" + self._index_name = "vector_idx" + + self.batch_size = int( + min(ALIYUN_OPENSEARCH_MAX_SIZE_PER_BATCH / (dim * 25), ALIYUN_OPENSEARCH_MAX_NUM_PER_BATCH) + ) + + log.info(f"Aliyun_OpenSearch client config: {self.db_config}") + control_config = open_api_models.Config( + access_key_id=self.db_config["ak"], + access_key_secret=self.db_config["sk"], + endpoint=self.db_config["control_host"] + ) + self.control_client = searchengineClient(control_config) + + if drop_old: + log.info(f"aliyun_OpenSearch client drop old index: {self.collection_name}") + if self._index_exists(self.control_client): + self._modify_index(self.control_client) + else: + self._create_index(self.control_client) + + def _create_index(self, client: searchengineClient): + create_table_request = searchengine_models.CreateTableRequest() + create_table_request.name = self.collection_name + create_table_request.primary_key = self._primary_field + create_table_request.partition_count = 1 + create_table_request.field_schema = { + self._primary_field: "INT64", + self._vector_field: "MULTI_FLOAT", + self._scalar_field: "INT64" + } + vector_index = searchengine_models.ModifyTableRequestVectorIndex() + vector_index.index_name = self._index_name + vector_index.dimension = self.dim + vector_index.distance_type = self.case_config.distance_type() + vector_index.vector_field = self._vector_field + vector_index.vector_index_type = "HNSW" + + advance_params = searchengine_models.ModifyTableRequestVectorIndexAdvanceParams() + advance_params.build_index_params = "{\"proxima.hnsw.builder.max_neighbor_count\":" + str(self.case_config.M) + ",\"proxima.hnsw.builder.efconstruction\":" + str(self.case_config.efConstruction) + ",\"proxima.hnsw.builder.enable_adsampling\":true,\"proxima.hnsw.builder.slack_pruning_factor\":1.1,\"proxima.hnsw.builder.thread_count\":16}" + advance_params.search_index_params = "{\"proxima.hnsw.searcher.ef\":400,\"proxima.hnsw.searcher.dynamic_termination.prob_threshold\":0.7}" + vector_index.advance_params = advance_params + create_table_request.vector_index = [vector_index] + + try: + response = client.create_table(self.instance_id, create_table_request) + log.info(f"create table success: {response.body}") + except Exception as error: + log.info(error.message) + log.info(error.data.get("Recommend")) + log.info(f"Failed to create index: error: {str(error)}") + raise error from None + + # check if index create success + self._active_index(client) + + # check if index create success + def _active_index(self, client: searchengineClient) -> None: + retry_times = 0 + while True: + time.sleep(10) + log.info(f"begin to {retry_times} times get table") + retry_times += 1 + response = client.get_table(self.instance_id, self.collection_name) + if response.body.result.status == 'IN_USE': + log.info(f"{self.collection_name} table begin to use.") + return + + def _index_exists(self, client: searchengineClient) -> bool: + try: + client.get_table(self.instance_id, self.collection_name) + return True + except Exception as error: + log.info(f'get table from searchengine error') + log.info(error.message) + return False + + # check if index build success, Insert the embeddings to the vector database after index build success + def _index_build_success(self, client: searchengineClient) -> None: + log.info(f"begin to check if table build success.") + time.sleep(50) + + retry_times = 0 + while True: + time.sleep(10) + log.info(f"begin to {retry_times} times get table fsm") + retry_times += 1 + request = searchengine_models.ListTasksRequest() + request.start = (int(time.time()) - 3600) * 1000 + request.end = int(time.time()) * 1000 + response = client.list_tasks(self.instance_id, request) + fsms = response.body.result + cur_fsm = None + for fsm in fsms: + if fsm["type"] != "datasource_flow_fsm": + continue + if self.collection_name not in fsm["fsmId"]: + continue + cur_fsm = fsm + break + if cur_fsm is None: + print("no build index fsm") + return + if "success" == cur_fsm["status"]: + return + + def _modify_index(self, client: searchengineClient) -> None: + # check if index create success + self._active_index(client) + + modify_table_request = searchengine_models.ModifyTableRequest() + modify_table_request.partition_count = 1 + modify_table_request.primary_key = self._primary_field + modify_table_request.field_schema = { + self._primary_field: "INT64", + self._vector_field: "MULTI_FLOAT", + self._scalar_field: "INT64" + } + vector_index = searchengine_models.ModifyTableRequestVectorIndex() + vector_index.index_name = self._index_name + vector_index.dimension = self.dim + vector_index.distance_type = self.case_config.distance_type() + vector_index.vector_field = self._vector_field + vector_index.vector_index_type = "HNSW" + advance_params = searchengine_models.ModifyTableRequestVectorIndexAdvanceParams() + advance_params.build_index_params = "{\"proxima.hnsw.builder.max_neighbor_count\":" + str(self.case_config.M) + ",\"proxima.hnsw.builder.efconstruction\":" + str(self.case_config.efConstruction) + ",\"proxima.hnsw.builder.enable_adsampling\":true,\"proxima.hnsw.builder.slack_pruning_factor\":1.1,\"proxima.hnsw.builder.thread_count\":16}" + advance_params.search_index_params = "{\"proxima.hnsw.searcher.ef\":400,\"proxima.hnsw.searcher.dynamic_termination.prob_threshold\":0.7}" + vector_index.advance_params = advance_params + + modify_table_request.vector_index = [vector_index] + + try: + response = client.modify_table(self.instance_id, self.collection_name, modify_table_request) + log.info(f"modify table success: {response.body}") + except Exception as error: + log.info(error.message) + log.info(error.data.get("Recommend")) + log.info(f"Failed to modify index: error: {str(error)}") + raise error from None + + # check if modify index & delete data fsm success + self._index_build_success(client) + + # get collection records total count + def _get_total_count(self): + try: + response = self.client.stats(self.collection_name) + body = json.loads(response.body) + log.info(f"stats info: {response.body}") + + if "result" in body and "totalDocCount" in body.get("result"): + return body.get("result").get("totalDocCount") + else: + return 0 + except Exception as e: + print(f"Error querying index: {e}") + return 0 + + @contextmanager + def init(self) -> None: + """connect to aliyun opensearch""" + config = models.Config( + endpoint=self.db_config["host"], + protocol="http", + access_user_name=self.db_config["user"], + access_pass_word=self.db_config["password"] + ) + + self.client = client.Client(config) + + yield + # self.client.transport.close() + self.client = None + del self.client + + def insert_embeddings( + self, + embeddings: list[list[float]], + metadata: list[int], + **kwargs, + ) -> tuple[int, Exception]: + """Insert the embeddings to the opensearch.""" + assert self.client is not None, "should self.init() first" + assert len(embeddings) == len(metadata) + insert_count = 0 + + try: + for batch_start_offset in range(0, len(embeddings), self.batch_size): + batch_end_offset = min( + batch_start_offset + self.batch_size, len(embeddings) + ) + documents = [] + for i in range(batch_start_offset, batch_end_offset): + documentFields = { + self._primary_field: metadata[i], + self._vector_field: embeddings[i], + self._scalar_field: metadata[i], + "ops_build_channel": "inc" + } + document = { + "fields": documentFields, + "cmd": "add" + } + documents.append(document) + + pushDocumentsRequest = models.PushDocumentsRequest({}, documents) + self.client.push_documents(self.collection_name, self._primary_field, pushDocumentsRequest) + insert_count += batch_end_offset - batch_start_offset + except Exception as e: + log.info(f"Failed to insert data: {e}") + return (insert_count, e) + return (insert_count, None) + + def search_embedding( + self, + query: list[float], + k: int = 100, + filters: dict | None = None, + ) -> list[int]: + assert self.client is not None, "should self.init() first" + search_params = "{\"proxima.hnsw.searcher.ef\":"+ str(self.case_config.ef_search) +"}" + + os_filter = f"{self._scalar_field} {filters.get('metadata')}" if filters else "" + + try: + request = QueryRequest(table_name=self.collection_name, + vector=query, + top_k=k, + search_params=search_params, filter=os_filter) + result = self.client.query(request) + except Exception as e: + log.info(f"Error querying index: {e}") + raise e + res = json.loads(result.body) + id_res = [one_res["id"] for one_res in res["result"]] + return id_res + + def need_normalize_cosine(self) -> bool: + """Wheather this database need to normalize dataset to support COSINE""" + if self.case_config.metric_type == MetricType.COSINE: + log.info(f"cosine dataset need normalize.") + return True + + return False + + def optimize(self): + pass + + def optimize_with_size(self, data_size: int): + log.info(f"optimize count: {data_size}") + retry_times = 0 + while True: + time.sleep(10) + log.info(f"begin to {retry_times} times get optimize table") + retry_times += 1 + total_count = self._get_total_count() + # check if the data is inserted + if total_count == data_size: + log.info(f"optimize table finish.") + return + + def ready_to_load(self): + """ready_to_load will be called before load in load cases.""" + pass diff --git a/vectordb_bench/backend/clients/aliyun_opensearch/config.py b/vectordb_bench/backend/clients/aliyun_opensearch/config.py new file mode 100644 index 000000000..7b2b9ad13 --- /dev/null +++ b/vectordb_bench/backend/clients/aliyun_opensearch/config.py @@ -0,0 +1,48 @@ +import logging +from enum import Enum +from pydantic import SecretStr, BaseModel + +from ..api import DBConfig, DBCaseConfig, MetricType, IndexType + +log = logging.getLogger(__name__) + + +class AliyunOpenSearchConfig(DBConfig, BaseModel): + host: str = "" + user: str = "" + password: SecretStr = "" + + ak: str = "" + sk: SecretStr = "" + control_host: str = "searchengine.cn-hangzhou.aliyuncs.com" + + def to_dict(self) -> dict: + return { + "host": self.host, + "user": self.user, + "password": self.password.get_secret_value(), + "ak": self.ak, + "sk": self.sk.get_secret_value(), + "control_host": self.control_host, + } + +class AliyunOpenSearchIndexConfig(BaseModel, DBCaseConfig): + metric_type: MetricType = MetricType.L2 + efConstruction: int = 500 + M: int = 100 + ef_search: int = 40 + + def distance_type(self) -> str: + if self.metric_type == MetricType.L2: + return "SquaredEuclidean" + elif self.metric_type == MetricType.IP: + return "InnerProduct" + elif self.metric_type == MetricType.COSINE: + return "InnerProduct" + return "SquaredEuclidean" + + def index_param(self) -> dict: + return {} + + def search_param(self) -> dict: + return {} diff --git a/vectordb_bench/backend/clients/api.py b/vectordb_bench/backend/clients/api.py index da2bed089..fe2e554f3 100644 --- a/vectordb_bench/backend/clients/api.py +++ b/vectordb_bench/backend/clients/api.py @@ -204,6 +204,9 @@ def optimize(self): """ raise NotImplementedError + def optimize_with_size(self, data_size: int): + self.optimize() + # TODO: remove @abstractmethod def ready_to_load(self): diff --git a/vectordb_bench/backend/task_runner.py b/vectordb_bench/backend/task_runner.py index 0b115bba7..9de42d647 100644 --- a/vectordb_bench/backend/task_runner.py +++ b/vectordb_bench/backend/task_runner.py @@ -238,7 +238,7 @@ def _conc_search(self): @utils.time_it def _task(self) -> None: with self.db.init(): - self.db.optimize() + self.db.optimize_with_size(data_size=self.ca.dataset.data.size) def _optimize(self) -> float: with concurrent.futures.ProcessPoolExecutor(max_workers=1) as executor: diff --git a/vectordb_bench/frontend/config/dbCaseConfigs.py b/vectordb_bench/frontend/config/dbCaseConfigs.py index 88794e30f..7f076a2dd 100644 --- a/vectordb_bench/frontend/config/dbCaseConfigs.py +++ b/vectordb_bench/frontend/config/dbCaseConfigs.py @@ -437,6 +437,16 @@ class CaseConfigInput(BaseModel): }, ) +CaseConfigParamInput_EF_SEARCH_AliyunOpensearch = CaseConfigInput( + label=CaseConfigParamType.ef_search, + inputType=InputType.Number, + inputConfig={ + "min": 1, + "max": 1000000, + "value": 40, + }, +) + CaseConfigParamInput_maintenance_work_mem_PgVector = CaseConfigInput( label=CaseConfigParamType.maintenance_work_mem, @@ -1128,6 +1138,11 @@ class CaseConfigInput(BaseModel): CaseConfigParamInput_EF_SEARCH_AWSOpensearch, ] +AliyunOpensearchLoadingConfig = [] +AliyunOpenSearchPerformanceConfig = [ + CaseConfigParamInput_EF_SEARCH_AliyunOpensearch, +] + PgVectorLoadingConfig = [ CaseConfigParamInput_IndexType_PgVector, CaseConfigParamInput_Lists_PgVector, @@ -1286,4 +1301,8 @@ class CaseConfigInput(BaseModel): CaseLabel.Load: AliyunElasticsearchLoadingConfig, CaseLabel.Performance: AliyunElasticsearchPerformanceConfig, }, + DB.AliyunOpenSearch: { + CaseLabel.Load: AliyunOpensearchLoadingConfig, + CaseLabel.Performance: AliyunOpenSearchPerformanceConfig, + }, } From 8a1e18ec0ea1a866e6a91385ea3b9b139a10993f Mon Sep 17 00:00:00 2001 From: "min.tian" Date: Thu, 12 Dec 2024 16:59:03 +0800 Subject: [PATCH 118/327] bug fix: cost time should be removed from the results of the serial_search Signed-off-by: min.tian --- vectordb_bench/backend/task_runner.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/vectordb_bench/backend/task_runner.py b/vectordb_bench/backend/task_runner.py index 9de42d647..568152ae0 100644 --- a/vectordb_bench/backend/task_runner.py +++ b/vectordb_bench/backend/task_runner.py @@ -206,7 +206,7 @@ def _load_train_data(self): finally: runner = None - def _serial_search(self) -> tuple[float, float]: + def _serial_search(self) -> tuple[float, float, float]: """Performance serial tests, search the entire test data once, calculate the recall, serial_latency_p99 @@ -214,7 +214,8 @@ def _serial_search(self) -> tuple[float, float]: tuple[float, float]: recall, serial_latency_p99 """ try: - return self.serial_search_runner.run() + results, _ = self.serial_search_runner.run() + return results except Exception as e: log.warning(f"search error: {str(e)}, {e}") self.stop() From 0b524d9fc721c40032debc47fcb5c573ac1eecc7 Mon Sep 17 00:00:00 2001 From: yangxuan Date: Fri, 13 Dec 2024 12:12:08 +0800 Subject: [PATCH 119/327] fix: Opensearch requirements Also remove some dup sub-packages Signed-off-by: yangxuan --- README.md | 13 +++++-------- pyproject.toml | 37 +++++++++++++++++++------------------ 2 files changed, 24 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 9a109146e..893c0497a 100644 --- a/README.md +++ b/README.md @@ -29,21 +29,18 @@ All the database client supported | Optional database client | install command | |--------------------------|---------------------------------------------| -| pymilvus(*default*) | `pip install vectordb-bench` | -| all | `pip install vectordb-bench[all]` | +| pymilvus, zilliz_cloud (*default*) | `pip install vectordb-bench` | +| all (*clients requirements might be conflict with each other*) | `pip install vectordb-bench[all]` | | qdrant | `pip install vectordb-bench[qdrant]` | | pinecone | `pip install vectordb-bench[pinecone]` | | weaviate | `pip install vectordb-bench[weaviate]` | -| elastic | `pip install vectordb-bench[elastic]` | -| pgvector | `pip install vectordb-bench[pgvector]` | +| elastic, aliyun_elasticsearch| `pip install vectordb-bench[elastic]` | +| pgvector, pgvectorscale, pgdiskann, alloydb | `pip install vectordb-bench[pgvector]` | | pgvecto.rs | `pip install vectordb-bench[pgvecto_rs]` | -| pgvectorscale | `pip install vectordb-bench[pgvectorscale]` | -| pgdiskann | `pip install vectordb-bench[pgdiskann]` | | redis | `pip install vectordb-bench[redis]` | | memorydb | `pip install vectordb-bench[memorydb]` | | chromadb | `pip install vectordb-bench[chromadb]` | -| awsopensearch | `pip install vectordb-bench[awsopensearch]` | -| alloydb | `pip install vectordb-bench[alloydb]` | +| awsopensearch | `pip install vectordb-bench[opensearch]` | ### Run diff --git a/pyproject.toml b/pyproject.toml index 760404b1d..6ad8e23ec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,31 +55,32 @@ all = [ "pinecone-client", "weaviate-client", "elasticsearch", - "pgvector", - "pgvecto_rs[psycopg3]>=0.2.2", "sqlalchemy", "redis", "chromadb", + "pgvector", "psycopg", "psycopg-binary", - "opensearch-dsl==2.1.0", - "opensearch-py==2.6.0", + "pgvecto_rs[psycopg3]>=0.2.2", + "opensearch-dsl", + "opensearch-py", + "memorydb", ] -qdrant = [ "qdrant-client" ] -pinecone = [ "pinecone-client" ] -weaviate = [ "weaviate-client" ] -elastic = [ "elasticsearch" ] -pgvector = [ "psycopg", "psycopg-binary", "pgvector" ] -pgvectorscale = [ "psycopg", "psycopg-binary", "pgvector" ] -pgdiskann = [ "psycopg", "psycopg-binary", "pgvector" ] -alloydb = [ "psycopg", "psycopg-binary", "pgvector"] -pgvecto_rs = [ "pgvecto_rs[psycopg3]>=0.2.2" ] -redis = [ "redis" ] -memorydb = [ "memorydb" ] -chromadb = [ "chromadb" ] -awsopensearch = [ "awsopensearch" ] -zilliz_cloud = [] +qdrant = [ "qdrant-client" ] +pinecone = [ "pinecone-client" ] +weaviate = [ "weaviate-client" ] +elastic = [ "elasticsearch" ] +# For elastic and aliyun_elasticsearch + +pgvector = [ "psycopg", "psycopg-binary", "pgvector" ] +# for pgvector, pgvectorscale, pgdiskann, and, alloydb + +pgvecto_rs = [ "pgvecto_rs[psycopg3]>=0.2.2" ] +redis = [ "redis" ] +memorydb = [ "memorydb" ] +chromadb = [ "chromadb" ] +opensearch = [ "opensearch-py" ] [project.urls] "repository" = "https://github.com/zilliztech/VectorDBBench" From 91d1eed67de36777d10309585d85cc3473b1413d Mon Sep 17 00:00:00 2001 From: hust-xing Date: Mon, 16 Dec 2024 14:32:51 +0800 Subject: [PATCH 120/327] add aliyun Opensearch requirements --- README.md | 1 + pyproject.toml | 3 +++ 2 files changed, 4 insertions(+) diff --git a/README.md b/README.md index 893c0497a..a642abf06 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,7 @@ All the database client supported | memorydb | `pip install vectordb-bench[memorydb]` | | chromadb | `pip install vectordb-bench[chromadb]` | | awsopensearch | `pip install vectordb-bench[opensearch]` | +| aliyun_opensearch | `pip install vectordb-bench[aliyun_opensearch]` | ### Run diff --git a/pyproject.toml b/pyproject.toml index 6ad8e23ec..8363aa4fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,6 +65,8 @@ all = [ "opensearch-dsl", "opensearch-py", "memorydb", + "alibabacloud_ha3engine_vector", + "alibabacloud_searchengine20211025", ] qdrant = [ "qdrant-client" ] @@ -81,6 +83,7 @@ redis = [ "redis" ] memorydb = [ "memorydb" ] chromadb = [ "chromadb" ] opensearch = [ "opensearch-py" ] +aliyun_opensearch = [ "alibabacloud_ha3engine_vector", "alibabacloud_searchengine20211025"] [project.urls] "repository" = "https://github.com/zilliztech/VectorDBBench" From 9aa58f0382599c7591f22a4d6aa58330562ed091 Mon Sep 17 00:00:00 2001 From: "min.tian" Date: Tue, 31 Dec 2024 10:15:29 +0800 Subject: [PATCH 121/327] update readme Signed-off-by: min.tian --- README.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index a642abf06..56d54c49d 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,16 @@ [![version](https://img.shields.io/pypi/v/vectordb-bench.svg?color=blue)](https://pypi.org/project/vectordb-bench/) [![Downloads](https://pepy.tech/badge/vectordb-bench)](https://pepy.tech/project/vectordb-bench) +## What is VectorDBBench +VectorDBBench is not just an offering of benchmark results for mainstream vector databases and cloud services, it's your go-to tool for the ultimate performance and cost-effectiveness comparison. Designed with ease-of-use in mind, VectorDBBench is devised to help users, even non-professionals, reproduce results or test new systems, making the hunt for the optimal choice amongst a plethora of cloud services and open-source vector databases a breeze. + +Understanding the importance of user experience, we provide an intuitive visual interface. This not only empowers users to initiate benchmarks at ease, but also to view comparative result reports, thereby reproducing benchmark results effortlessly. +To add more relevance and practicality, we provide cost-effectiveness reports particularly for cloud services. This allows for a more realistic and applicable benchmarking process. + +Closely mimicking real-world production environments, we've set up diverse testing scenarios including insertion, searching, and filtered searching. To provide you with credible and reliable data, we've included public datasets from actual production scenarios, such as [SIFT](http://corpus-texmex.irisa.fr/), [GIST](http://corpus-texmex.irisa.fr/), [Cohere](https://huggingface.co/datasets/Cohere/wikipedia-22-12/tree/main/en), and a dataset generated by OpenAI from an opensource [raw dataset](https://huggingface.co/datasets/allenai/c4). It's fascinating to discover how a relatively unknown open-source database might excel in certain circumstances! + +Prepare to delve into the world of VectorDBBench, and let it guide you in uncovering your perfect vector database match. + **Leaderboard:** https://zilliz.com/benchmark ## Quick Start ### Prerequirement @@ -180,16 +190,6 @@ milvushnsw: > - Options passed on the command line will override the configuration file* > - Parameter names use an _ not - -## What is VectorDBBench -VectorDBBench is not just an offering of benchmark results for mainstream vector databases and cloud services, it's your go-to tool for the ultimate performance and cost-effectiveness comparison. Designed with ease-of-use in mind, VectorDBBench is devised to help users, even non-professionals, reproduce results or test new systems, making the hunt for the optimal choice amongst a plethora of cloud services and open-source vector databases a breeze. - -Understanding the importance of user experience, we provide an intuitive visual interface. This not only empowers users to initiate benchmarks at ease, but also to view comparative result reports, thereby reproducing benchmark results effortlessly. -To add more relevance and practicality, we provide cost-effectiveness reports particularly for cloud services. This allows for a more realistic and applicable benchmarking process. - -Closely mimicking real-world production environments, we've set up diverse testing scenarios including insertion, searching, and filtered searching. To provide you with credible and reliable data, we've included public datasets from actual production scenarios, such as [SIFT](http://corpus-texmex.irisa.fr/), [GIST](http://corpus-texmex.irisa.fr/), [Cohere](https://huggingface.co/datasets/Cohere/wikipedia-22-12/tree/main/en), and a dataset generated by OpenAI from an opensource [raw dataset](https://huggingface.co/datasets/allenai/c4). It's fascinating to discover how a relatively unknown open-source database might excel in certain circumstances! - -Prepare to delve into the world of VectorDBBench, and let it guide you in uncovering your perfect vector database match. - ## Leaderboard ### Introduction To facilitate the presentation of test results and provide a comprehensive performance analysis report, we offer a [leaderboard page](https://zilliz.com/benchmark). It allows us to choose from QPS, QP$, and latency metrics, and provides a comprehensive assessment of a system's performance based on the test results of various cases and a set of scoring mechanisms (to be introduced later). On this leaderboard, we can select the systems and models to be compared, and filter out cases we do not want to consider. Comprehensive scores are always ranked from best to worst, and the specific test results of each query will be presented in the list below. From 4481615aeaf2c8a87959fc5b77b8b35478b7feaa Mon Sep 17 00:00:00 2001 From: xavrathi Date: Mon, 6 Jan 2025 18:53:52 -0800 Subject: [PATCH 122/327] Removed the Filter Path from Search, so we can get the full response --- vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py b/vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py index a27eb01fc..5fe67a36c 100644 --- a/vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py +++ b/vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py @@ -139,7 +139,7 @@ def search_embedding( **({"filter": {"range": {self.id_col_name: {"gt": filters["id"]}}}} if filters else {}) } try: - resp = self.client.search(index=self.index_name, body=body,size=k,_source=False,docvalue_fields=[self.id_col_name],stored_fields="_none_",filter_path=[f"hits.hits.fields.{self.id_col_name}"],) + resp = self.client.search(index=self.index_name, body=body,size=k,_source=False,docvalue_fields=[self.id_col_name],stored_fields="_none_",) log.info(f'Search took: {resp["took"]}') log.info(f'Search shards: {resp["_shards"]}') log.info(f'Search hits total: {resp["hits"]["total"]}') From ba6bd9b32aa90cfda2cea85da07a91a16db141b6 Mon Sep 17 00:00:00 2001 From: shaharuk-yb Date: Tue, 7 Jan 2025 15:41:21 +0530 Subject: [PATCH 123/327] add support to provide custom port in pgvector --- vectordb_bench/backend/clients/pgvector/cli.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/vectordb_bench/backend/clients/pgvector/cli.py b/vectordb_bench/backend/clients/pgvector/cli.py index 43385ee6b..de001c2dd 100644 --- a/vectordb_bench/backend/clients/pgvector/cli.py +++ b/vectordb_bench/backend/clients/pgvector/cli.py @@ -43,6 +43,15 @@ class PgVectorTypedDict(CommonTypedDict): host: Annotated[ str, click.option("--host", type=str, help="Db host", required=True) ] + port: Annotated[ + int, + click.option("--port", + type=int, + help="Postgres database port", + default=5432, + required=False + ), + ] db_name: Annotated[ str, click.option("--db-name", type=str, help="Db name", required=True) ] @@ -130,6 +139,7 @@ def PgVectorIVFFlat( user_name=SecretStr(parameters["user_name"]), password=SecretStr(parameters["password"]), host=parameters["host"], + port=parameters["port"], db_name=parameters["db_name"], ), db_case_config=PgVectorIVFFlatConfig( @@ -164,6 +174,7 @@ def PgVectorHNSW( user_name=SecretStr(parameters["user_name"]), password=SecretStr(parameters["password"]), host=parameters["host"], + port=parameters["port"], db_name=parameters["db_name"], ), db_case_config=PgVectorHNSWConfig( From 834c440a15af734de5c1670831f45b399a817a54 Mon Sep 17 00:00:00 2001 From: shaharuk-yb Date: Tue, 7 Jan 2025 15:50:20 +0530 Subject: [PATCH 124/327] show_default=True --- vectordb_bench/backend/clients/pgvector/cli.py | 1 + 1 file changed, 1 insertion(+) diff --git a/vectordb_bench/backend/clients/pgvector/cli.py b/vectordb_bench/backend/clients/pgvector/cli.py index de001c2dd..ef8914be0 100644 --- a/vectordb_bench/backend/clients/pgvector/cli.py +++ b/vectordb_bench/backend/clients/pgvector/cli.py @@ -49,6 +49,7 @@ class PgVectorTypedDict(CommonTypedDict): type=int, help="Postgres database port", default=5432, + show_default=True, required=False ), ] From 378841fe930511290a8acb593d0b0d6d9dd2125c Mon Sep 17 00:00:00 2001 From: xavrathi Date: Tue, 7 Jan 2025 12:12:50 -0800 Subject: [PATCH 125/327] updated the opensearch to use column id instead of _id --- vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py b/vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py index 5fe67a36c..2abaf6048 100644 --- a/vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py +++ b/vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py @@ -102,7 +102,7 @@ def insert_embeddings( insert_data = [] for i in range(len(embeddings)): - insert_data.append({"index": {"_index": self.index_name, "_id": metadata[i]}}) + insert_data.append({"index": {"_index": self.index_name, self.id_col_name: metadata[i]}}) insert_data.append({self.vector_col_name: embeddings[i]}) try: resp = self.client.bulk(insert_data) From 370106fe806c515ab46520b1633e41d6afdd4549 Mon Sep 17 00:00:00 2001 From: xavrathi Date: Tue, 7 Jan 2025 12:55:25 -0800 Subject: [PATCH 126/327] Modified the code to use internal column _id instead of id. --- .../backend/clients/aws_opensearch/aws_opensearch.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py b/vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py index 2abaf6048..60e519d7a 100644 --- a/vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py +++ b/vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py @@ -17,7 +17,7 @@ def __init__( db_config: dict, db_case_config: AWSOpenSearchIndexConfig, index_name: str = "vdb_bench_index", # must be lowercase - id_col_name: str = "id", + id_col_name: str = "_id", vector_col_name: str = "embedding", drop_old: bool = False, **kwargs, @@ -61,7 +61,6 @@ def _create_index(self, client: OpenSearch): } mappings = { "properties": { - self.id_col_name: {"type": "integer"}, **{ categoryCol: {"type": "keyword"} for categoryCol in self.category_col_names @@ -143,7 +142,7 @@ def search_embedding( log.info(f'Search took: {resp["took"]}') log.info(f'Search shards: {resp["_shards"]}') log.info(f'Search hits total: {resp["hits"]["total"]}') - result = [h["fields"][self.id_col_name][0] for h in resp["hits"]["hits"]] + result = [int(h["fields"][self.id_col_name][0]) for h in resp["hits"]["hits"]] #result = [int(d["_id"]) for d in resp["hits"]["hits"]] # log.info(f'success! length={len(res)}') From c0be2c7b6957147414599c512bf1ce3289613d48 Mon Sep 17 00:00:00 2001 From: Luka958Pixion Date: Wed, 8 Jan 2025 10:31:57 +0100 Subject: [PATCH 127/327] added HNSW params to index creation and search --- vectordb_bench/backend/clients/redis/redis.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/vectordb_bench/backend/clients/redis/redis.py b/vectordb_bench/backend/clients/redis/redis.py index 8acf669d2..e95569873 100644 --- a/vectordb_bench/backend/clients/redis/redis.py +++ b/vectordb_bench/backend/clients/redis/redis.py @@ -1,8 +1,7 @@ import logging from contextlib import contextmanager -from typing import Any, Type -from ..api import VectorDB, DBConfig, DBCaseConfig, EmptyDBCaseConfig, IndexType -from .config import RedisConfig +from typing import Any +from ..api import VectorDB, DBCaseConfig import redis from redis.commands.search.field import TagField, VectorField, NumericField from redis.commands.search.indexDefinition import IndexDefinition, IndexType @@ -58,6 +57,8 @@ def make_index(self, vector_dimensions: int, conn: redis.Redis): "TYPE": "FLOAT32", # FLOAT32 or FLOAT64 "DIM": vector_dimensions, # Number of Vector Dimensions "DISTANCE_METRIC": "COSINE", # Vector Search Distance Metric + "M": self.case_config.index_param()["params"]["M"], + "EF_CONSTRUCTION": self.case_config.index_param()["params"]["efConstruction"], } ), ) @@ -138,21 +139,20 @@ def search_embedding( query_obj = Query(f"*=>[KNN {k} @vector $vec as score]").sort_by("score").return_fields("id", "score").paging(0, k).dialect(2) query_params = {"vec": query_vector} + ef_runtime = {self.case_config.search_param()["params"]["ef"]} + if filters: # benchmark test filters of format: {'metadata': '>=10000', 'id': 10000} # gets exact match for id, and range for metadata if they exist in filters id_value = filters.get("id") metadata_value = filters.get("metadata") if id_value and metadata_value: - query_obj = Query(f"(@metadata:[{metadata_value} +inf] @id:{ {id_value} })=>[KNN {k} @vector $vec as score]").sort_by("score").return_fields("id", "score").paging(0, k).dialect(2) + query_obj = Query(f"(@metadata:[{metadata_value} +inf] @id:{ {id_value} })=>[KNN {k} @vector $vec EF_RUNTIME {ef_runtime} as score]").sort_by("score").return_fields("id", "score").paging(0, k).dialect(2) elif id_value: #gets exact match for id - query_obj = Query(f"@id:{ {id_value} }=>[KNN {k} @vector $vec as score]").sort_by("score").return_fields("id", "score").paging(0, k).dialect(2) + query_obj = Query(f"@id:{ {id_value} }=>[KNN {k} @vector $vec EF_RUNTIME {ef_runtime} as score]").sort_by("score").return_fields("id", "score").paging(0, k).dialect(2) else: #metadata only case, greater than or equal to metadata value - query_obj = Query(f"@metadata:[{metadata_value} +inf]=>[KNN {k} @vector $vec as score]").sort_by("score").return_fields("id", "score").paging(0, k).dialect(2) + query_obj = Query(f"@metadata:[{metadata_value} +inf]=>[KNN {k} @vector $vec EF_RUNTIME {ef_runtime} as score]").sort_by("score").return_fields("id", "score").paging(0, k).dialect(2) res = self.conn.ft(INDEX_NAME).search(query_obj, query_params) # doc in res of format {'id': '9831', 'payload': None, 'score': '1.19209289551e-07'} return [int(doc["id"]) for doc in res.docs] - - - From 2a0c43bb56f485f2c9890e84804aefc2e41a294c Mon Sep 17 00:00:00 2001 From: Luka958Pixion Date: Wed, 8 Jan 2025 15:51:18 +0000 Subject: [PATCH 128/327] fixed types --- vectordb_bench/backend/clients/redis/redis.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vectordb_bench/backend/clients/redis/redis.py b/vectordb_bench/backend/clients/redis/redis.py index e95569873..ab61c3418 100644 --- a/vectordb_bench/backend/clients/redis/redis.py +++ b/vectordb_bench/backend/clients/redis/redis.py @@ -69,7 +69,7 @@ def make_index(self, vector_dimensions: int, conn: redis.Redis): rs.create_index(schema, definition=definition) @contextmanager - def init(self) -> None: + def init(self): """ create and destory connections to database. Examples: @@ -99,7 +99,7 @@ def insert_embeddings( embeddings: list[list[float]], metadata: list[int], **kwargs: Any, - ) -> (int, Exception): + ) -> tuple[int, Exception]: """Insert embeddings into the database. Should call self.init() first. """ From 8627b8f7051e6a3dca38c653257c507367d3ab69 Mon Sep 17 00:00:00 2001 From: Luka958Pixion Date: Wed, 8 Jan 2025 17:41:37 +0000 Subject: [PATCH 129/327] fixed query --- vectordb_bench/backend/clients/redis/redis.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vectordb_bench/backend/clients/redis/redis.py b/vectordb_bench/backend/clients/redis/redis.py index ab61c3418..49fef60cd 100644 --- a/vectordb_bench/backend/clients/redis/redis.py +++ b/vectordb_bench/backend/clients/redis/redis.py @@ -136,10 +136,10 @@ def search_embedding( assert self.conn is not None query_vector = np.array(query).astype(np.float32).tobytes() - query_obj = Query(f"*=>[KNN {k} @vector $vec as score]").sort_by("score").return_fields("id", "score").paging(0, k).dialect(2) + ef_runtime = {self.case_config.search_param()["params"]["ef"]} + query_obj = Query(f"*=>[KNN {k} @vector $vec EF_RUNTIME {ef_runtime} as score]").sort_by("score").return_fields("id", "score").paging(0, k).dialect(2) query_params = {"vec": query_vector} - ef_runtime = {self.case_config.search_param()["params"]["ef"]} if filters: # benchmark test filters of format: {'metadata': '>=10000', 'id': 10000} From c0027564c91f8609a60af5f147c9001492f0a42e Mon Sep 17 00:00:00 2001 From: Luka958Pixion Date: Wed, 8 Jan 2025 19:00:17 +0100 Subject: [PATCH 130/327] fixed ef_runtime --- vectordb_bench/backend/clients/redis/redis.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/vectordb_bench/backend/clients/redis/redis.py b/vectordb_bench/backend/clients/redis/redis.py index 49fef60cd..cf51fcc48 100644 --- a/vectordb_bench/backend/clients/redis/redis.py +++ b/vectordb_bench/backend/clients/redis/redis.py @@ -136,11 +136,10 @@ def search_embedding( assert self.conn is not None query_vector = np.array(query).astype(np.float32).tobytes() - ef_runtime = {self.case_config.search_param()["params"]["ef"]} + ef_runtime = self.case_config.search_param()["params"]["ef"] query_obj = Query(f"*=>[KNN {k} @vector $vec EF_RUNTIME {ef_runtime} as score]").sort_by("score").return_fields("id", "score").paging(0, k).dialect(2) query_params = {"vec": query_vector} - if filters: # benchmark test filters of format: {'metadata': '>=10000', 'id': 10000} # gets exact match for id, and range for metadata if they exist in filters From f9dec0854e2fd597d86e2a1c0a9faaf6289f1eea Mon Sep 17 00:00:00 2001 From: yangxuan Date: Wed, 8 Jan 2025 18:39:35 +0800 Subject: [PATCH 131/327] enhance: Refine the coding style and enable lint-action Signed-off-by: yangxuan --- .github/workflows/pull_request.yml | 4 + .ruff.toml | 49 ----- Makefile | 8 + README.md | 6 +- pyproject.toml | 114 ++++++++++ vectordb_bench/__init__.py | 73 ++++--- vectordb_bench/__main__.py | 7 +- vectordb_bench/backend/assembler.py | 25 ++- vectordb_bench/backend/cases.py | 100 +++++---- vectordb_bench/backend/clients/__init__.py | 89 ++++++-- .../aliyun_elasticsearch.py | 3 +- .../clients/aliyun_elasticsearch/config.py | 7 +- .../aliyun_opensearch/aliyun_opensearch.py | 181 +++++++++------- .../clients/aliyun_opensearch/config.py | 13 +- .../backend/clients/alloydb/alloydb.py | 138 ++++++------- vectordb_bench/backend/clients/alloydb/cli.py | 85 +++++--- .../backend/clients/alloydb/config.py | 60 +++--- vectordb_bench/backend/clients/api.py | 14 +- .../clients/aws_opensearch/aws_opensearch.py | 90 ++++---- .../backend/clients/aws_opensearch/cli.py | 11 +- .../backend/clients/aws_opensearch/config.py | 22 +- .../backend/clients/aws_opensearch/run.py | 128 ++++++------ .../backend/clients/chroma/chroma.py | 74 +++---- .../backend/clients/chroma/config.py | 6 +- .../backend/clients/elastic_cloud/config.py | 10 +- .../clients/elastic_cloud/elastic_cloud.py | 45 ++-- .../backend/clients/memorydb/cli.py | 16 +- .../backend/clients/memorydb/config.py | 4 +- .../backend/clients/memorydb/memorydb.py | 118 ++++++----- vectordb_bench/backend/clients/milvus/cli.py | 124 ++++------- .../backend/clients/milvus/config.py | 26 ++- .../backend/clients/milvus/milvus.py | 37 ++-- .../backend/clients/pgdiskann/cli.py | 51 +++-- .../backend/clients/pgdiskann/config.py | 55 ++--- .../backend/clients/pgdiskann/pgdiskann.py | 128 +++++------- .../backend/clients/pgvecto_rs/cli.py | 20 +- .../backend/clients/pgvecto_rs/config.py | 22 +- .../backend/clients/pgvecto_rs/pgvecto_rs.py | 67 +++--- .../backend/clients/pgvector/cli.py | 77 ++++--- .../backend/clients/pgvector/config.py | 136 ++++++------ .../backend/clients/pgvector/pgvector.py | 195 +++++++++--------- .../backend/clients/pgvectorscale/cli.py | 62 +++--- .../backend/clients/pgvectorscale/config.py | 29 ++- .../clients/pgvectorscale/pgvectorscale.py | 81 ++++---- .../backend/clients/pinecone/config.py | 1 + .../backend/clients/pinecone/pinecone.py | 35 ++-- .../backend/clients/qdrant_cloud/config.py | 21 +- .../clients/qdrant_cloud/qdrant_cloud.py | 71 ++++--- vectordb_bench/backend/clients/redis/cli.py | 18 +- .../backend/clients/redis/config.py | 12 +- vectordb_bench/backend/clients/redis/redis.py | 143 ++++++++----- vectordb_bench/backend/clients/test/cli.py | 3 +- vectordb_bench/backend/clients/test/config.py | 4 +- vectordb_bench/backend/clients/test/test.py | 9 +- .../backend/clients/weaviate_cloud/cli.py | 7 +- .../backend/clients/weaviate_cloud/config.py | 4 +- .../clients/weaviate_cloud/weaviate_cloud.py | 58 ++++-- .../backend/clients/zilliz_cloud/cli.py | 25 ++- .../backend/clients/zilliz_cloud/config.py | 6 +- .../clients/zilliz_cloud/zilliz_cloud.py | 2 +- vectordb_bench/backend/data_source.py | 48 +++-- vectordb_bench/backend/dataset.py | 74 ++++--- vectordb_bench/backend/result_collector.py | 5 +- vectordb_bench/backend/runner/__init__.py | 10 +- vectordb_bench/backend/runner/mp_runner.py | 119 ++++++++--- vectordb_bench/backend/runner/rate_runner.py | 49 +++-- .../backend/runner/read_write_runner.py | 74 ++++--- .../backend/runner/serial_runner.py | 139 ++++++++----- vectordb_bench/backend/runner/util.py | 7 +- vectordb_bench/backend/task_runner.py | 164 ++++++++------- vectordb_bench/backend/utils.py | 27 ++- vectordb_bench/base.py | 1 - vectordb_bench/cli/cli.py | 125 +++++------ vectordb_bench/cli/vectordbbench.py | 13 +- .../components/check_results/charts.py | 27 +-- .../frontend/components/check_results/data.py | 20 +- .../components/check_results/filters.py | 24 +-- .../frontend/components/check_results/nav.py | 8 +- .../components/check_results/priceTable.py | 4 +- .../components/check_results/stPageConfig.py | 3 +- .../frontend/components/concurrent/charts.py | 24 +-- .../components/custom/displayCustomCase.py | 28 ++- .../components/custom/displaypPrams.py | 6 +- .../components/custom/getCustomConfig.py | 3 +- .../frontend/components/custom/initStyle.py | 2 +- .../components/get_results/saveAsImage.py | 2 + .../components/run_test/caseSelector.py | 12 +- .../components/run_test/dbConfigSetting.py | 5 +- .../components/run_test/dbSelector.py | 2 +- .../components/run_test/generateTasks.py | 16 +- .../components/run_test/submitTask.py | 32 ++- .../frontend/components/tables/data.py | 9 +- .../frontend/config/dbCaseConfigs.py | 135 +++++------- vectordb_bench/frontend/pages/concurrent.py | 8 +- vectordb_bench/frontend/pages/custom.py | 39 +++- .../frontend/pages/quries_per_dollar.py | 6 +- vectordb_bench/frontend/pages/run_test.py | 10 +- vectordb_bench/frontend/utils.py | 2 +- vectordb_bench/frontend/vdb_benchmark.py | 10 +- vectordb_bench/interface.py | 82 +++++--- vectordb_bench/log_util.py | 123 ++++++----- vectordb_bench/metric.py | 21 +- vectordb_bench/models.py | 69 +++---- 103 files changed, 2490 insertions(+), 2126 deletions(-) delete mode 100644 .ruff.toml diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 7c133cd5f..ea346dcd0 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -31,6 +31,10 @@ jobs: python -m pip install --upgrade pip pip install -e ".[test]" + - name: Run coding checks + run: | + make lint + - name: Test with pytest run: | make unittest diff --git a/.ruff.toml b/.ruff.toml deleted file mode 100644 index fb6b19c3d..000000000 --- a/.ruff.toml +++ /dev/null @@ -1,49 +0,0 @@ -# Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default. -# Enable flake8-bugbear (`B`) rules. -select = ["E", "F", "B"] -ignore = [ - "E501", # (line length violations) -] - -# Allow autofix for all enabled rules (when `--fix`) is provided. -fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", - "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", - "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT", -] -unfixable = [] - -# Exclude a variety of commonly ignored directories. -exclude = [ - ".bzr", - ".direnv", - ".eggs", - ".git", - ".hg", - ".mypy_cache", - ".nox", - ".pants.d", - ".pytype", - ".ruff_cache", - ".svn", - ".tox", - ".venv", - "__pypackages__", - "_build", - "buck-out", - "build", - "dist", - "node_modules", - "venv", - "__pycache__", - "__init__.py", -] - -# Allow unused variables when underscore-prefixed. -dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" - -# Assume Python 3.11. -target-version = "py311" - -[mccabe] -# Unlike Flake8, default to a complexity level of 10. -max-complexity = 10 diff --git a/Makefile b/Makefile index 562615f6d..ef8207c55 100644 --- a/Makefile +++ b/Makefile @@ -1,2 +1,10 @@ unittest: PYTHONPATH=`pwd` python3 -m pytest tests/test_dataset.py::TestDataSet::test_download_small -svv + +format: + PYTHONPATH=`pwd` python3 -m black vectordb_bench + PYTHONPATH=`pwd` python3 -m ruff check vectordb_bench --fix + +lint: + PYTHONPATH=`pwd` python3 -m black vectordb_bench --check + PYTHONPATH=`pwd` python3 -m ruff check vectordb_bench diff --git a/README.md b/README.md index 56d54c49d..737fc6064 100644 --- a/README.md +++ b/README.md @@ -240,13 +240,13 @@ After reopen the repository in container, run `python -m vectordb_bench` in the ### Check coding styles ```shell -$ ruff check vectordb_bench +$ make lint ``` -Add `--fix` if you want to fix the coding styles automatically +To fix the coding styles automatically ```shell -$ ruff check vectordb_bench --fix +$ make format ``` ## How does it work? diff --git a/pyproject.toml b/pyproject.toml index 8363aa4fa..312940634 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,7 @@ dynamic = ["version"] [project.optional-dependencies] test = [ + "black", "ruff", "pytest", ] @@ -93,3 +94,116 @@ init_bench = "vectordb_bench.__main__:main" vectordbbench = "vectordb_bench.cli.vectordbbench:cli" [tool.setuptools_scm] + +[tool.black] +line-length = 120 +target-version = ['py311'] +include = '\.pyi?$' + +[tool.ruff] +lint.select = [ + "E", + "F", + "C90", + "I", + "N", + "B", "C", "G", + "A", + "ANN001", + "S", "T", "W", "ARG", "BLE", "COM", "DJ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT" +] +lint.ignore = [ + "BLE001", # blind-except (BLE001) + "SLF001", # SLF001 Private member accessed [E] + "TRY003", # [ruff] TRY003 Avoid specifying long messages outside the exception class [E] + "FBT001", "FBT002", "FBT003", + "G004", # [ruff] G004 Logging statement uses f-string [E] + "UP031", + "RUF012", + "EM101", + "N805", + "ARG002", + "ARG003", + "PIE796", # https://github.com/zilliztech/VectorDBBench/issues/438 + "INP001", # TODO + "TID252", # TODO + "N801", "N802", "N815", + "S101", "S108", "S603", "S311", + "PLR2004", + "RUF017", + "C416", + "PLW0603", +] + +# Allow autofix for all enabled rules (when `--fix`) is provided. +lint.fixable = [ + "A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", + "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", + "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", + "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", + "YTT", +] +lint.unfixable = [] + +show-fixes = true + +# Exclude a variety of commonly ignored directories. +exclude = [ + ".bzr", + ".direnv", + ".eggs", + ".git", + ".git-rewrite", + ".hg", + ".mypy_cache", + ".nox", + ".pants.d", + ".pytype", + ".ruff_cache", + ".svn", + ".tox", + ".venv", + "__pypackages__", + "_build", + "buck-out", + "build", + "dist", + "node_modules", + "venv", + "grpc_gen", + "__pycache__", + "frontend", # TODO + "tests", +] + +# Same as Black. +line-length = 120 + +# Allow unused variables when underscore-prefixed. +lint.dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" + +# Assume Python 3.11 +target-version = "py311" + +[tool.ruff.lint.mccabe] +# Unlike Flake8, default to a complexity level of 10. +max-complexity = 18 + +[tool.ruff.lint.pycodestyle] +max-line-length = 120 +max-doc-length = 120 + +[tool.ruff.lint.pylint] +max-args = 20 +max-branches = 15 + +[tool.ruff.lint.flake8-builtins] +builtins-ignorelist = [ + # "format", + # "next", + # "object", # TODO + # "id", + # "dict", # TODO + # "filter", +] + diff --git a/vectordb_bench/__init__.py b/vectordb_bench/__init__.py index 568e97705..c07fc855d 100644 --- a/vectordb_bench/__init__.py +++ b/vectordb_bench/__init__.py @@ -22,46 +22,71 @@ class config: DROP_OLD = env.bool("DROP_OLD", True) USE_SHUFFLED_DATA = env.bool("USE_SHUFFLED_DATA", True) - NUM_CONCURRENCY = env.list("NUM_CONCURRENCY", [1, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75, 80, 85, 90, 95, 100], subcast=int ) + NUM_CONCURRENCY = env.list( + "NUM_CONCURRENCY", + [ + 1, + 5, + 10, + 15, + 20, + 25, + 30, + 35, + 40, + 45, + 50, + 55, + 60, + 65, + 70, + 75, + 80, + 85, + 90, + 95, + 100, + ], + subcast=int, + ) CONCURRENCY_DURATION = 30 RESULTS_LOCAL_DIR = env.path( - "RESULTS_LOCAL_DIR", pathlib.Path(__file__).parent.joinpath("results") + "RESULTS_LOCAL_DIR", + pathlib.Path(__file__).parent.joinpath("results"), ) CONFIG_LOCAL_DIR = env.path( - "CONFIG_LOCAL_DIR", pathlib.Path(__file__).parent.joinpath("config-files") + "CONFIG_LOCAL_DIR", + pathlib.Path(__file__).parent.joinpath("config-files"), ) - K_DEFAULT = 100 # default return top k nearest neighbors during search CUSTOM_CONFIG_DIR = pathlib.Path(__file__).parent.joinpath("custom/custom_case.json") - CAPACITY_TIMEOUT_IN_SECONDS = 24 * 3600 # 24h - LOAD_TIMEOUT_DEFAULT = 24 * 3600 # 24h - LOAD_TIMEOUT_768D_1M = 24 * 3600 # 24h - LOAD_TIMEOUT_768D_10M = 240 * 3600 # 10d - LOAD_TIMEOUT_768D_100M = 2400 * 3600 # 100d + CAPACITY_TIMEOUT_IN_SECONDS = 24 * 3600 # 24h + LOAD_TIMEOUT_DEFAULT = 24 * 3600 # 24h + LOAD_TIMEOUT_768D_1M = 24 * 3600 # 24h + LOAD_TIMEOUT_768D_10M = 240 * 3600 # 10d + LOAD_TIMEOUT_768D_100M = 2400 * 3600 # 100d - LOAD_TIMEOUT_1536D_500K = 24 * 3600 # 24h - LOAD_TIMEOUT_1536D_5M = 240 * 3600 # 10d + LOAD_TIMEOUT_1536D_500K = 24 * 3600 # 24h + LOAD_TIMEOUT_1536D_5M = 240 * 3600 # 10d - OPTIMIZE_TIMEOUT_DEFAULT = 24 * 3600 # 24h - OPTIMIZE_TIMEOUT_768D_1M = 24 * 3600 # 24h - OPTIMIZE_TIMEOUT_768D_10M = 240 * 3600 # 10d - OPTIMIZE_TIMEOUT_768D_100M = 2400 * 3600 # 100d + OPTIMIZE_TIMEOUT_DEFAULT = 24 * 3600 # 24h + OPTIMIZE_TIMEOUT_768D_1M = 24 * 3600 # 24h + OPTIMIZE_TIMEOUT_768D_10M = 240 * 3600 # 10d + OPTIMIZE_TIMEOUT_768D_100M = 2400 * 3600 # 100d + OPTIMIZE_TIMEOUT_1536D_500K = 24 * 3600 # 24h + OPTIMIZE_TIMEOUT_1536D_5M = 240 * 3600 # 10d - OPTIMIZE_TIMEOUT_1536D_500K = 24 * 3600 # 24h - OPTIMIZE_TIMEOUT_1536D_5M = 240 * 3600 # 10d - def display(self) -> str: - tmp = [ - i for i in inspect.getmembers(self) - if not inspect.ismethod(i[1]) - and not i[0].startswith('_') - and "TIMEOUT" not in i[0] + return [ + i + for i in inspect.getmembers(self) + if not inspect.ismethod(i[1]) and not i[0].startswith("_") and "TIMEOUT" not in i[0] ] - return tmp + log_util.init(config.LOG_LEVEL) diff --git a/vectordb_bench/__main__.py b/vectordb_bench/__main__.py index a8b4436bc..6663731f5 100644 --- a/vectordb_bench/__main__.py +++ b/vectordb_bench/__main__.py @@ -1,7 +1,8 @@ -import traceback import logging +import pathlib import subprocess -import os +import traceback + from . import config log = logging.getLogger("vectordb_bench") @@ -16,7 +17,7 @@ def run_streamlit(): cmd = [ "streamlit", "run", - f"{os.path.dirname(__file__)}/frontend/vdb_benchmark.py", + f"{pathlib.Path(__file__).parent}/frontend/vdb_benchmark.py", "--logger.level", "info", "--theme.base", diff --git a/vectordb_bench/backend/assembler.py b/vectordb_bench/backend/assembler.py index e7da4d49f..b81d315c2 100644 --- a/vectordb_bench/backend/assembler.py +++ b/vectordb_bench/backend/assembler.py @@ -1,24 +1,25 @@ -from .cases import CaseLabel -from .task_runner import CaseRunner, RunningStatus, TaskRunner -from ..models import TaskConfig -from ..backend.clients import EmptyDBCaseConfig -from ..backend.data_source import DatasetSource import logging +from vectordb_bench.backend.clients import EmptyDBCaseConfig +from vectordb_bench.backend.data_source import DatasetSource +from vectordb_bench.models import TaskConfig + +from .cases import CaseLabel +from .task_runner import CaseRunner, RunningStatus, TaskRunner log = logging.getLogger(__name__) class Assembler: @classmethod - def assemble(cls, run_id , task: TaskConfig, source: DatasetSource) -> CaseRunner: + def assemble(cls, run_id: str, task: TaskConfig, source: DatasetSource) -> CaseRunner: c_cls = task.case_config.case_id.case_cls c = c_cls(task.case_config.custom_case) - if type(task.db_case_config) != EmptyDBCaseConfig: + if type(task.db_case_config) is not EmptyDBCaseConfig: task.db_case_config.metric_type = c.dataset.data.metric_type - runner = CaseRunner( + return CaseRunner( run_id=run_id, config=task, ca=c, @@ -26,8 +27,6 @@ def assemble(cls, run_id , task: TaskConfig, source: DatasetSource) -> CaseRunne dataset_source=source, ) - return runner - @classmethod def assemble_all( cls, @@ -50,12 +49,12 @@ def assemble_all( db2runner[db].append(r) # check dbclient installed - for k in db2runner.keys(): + for k in db2runner: _ = k.init_cls # sort by dataset size - for k in db2runner.keys(): - db2runner[k].sort(key=lambda x:x.ca.dataset.data.size) + for k, _ in db2runner: + db2runner[k].sort(key=lambda x: x.ca.dataset.data.size) all_runners = [] all_runners.extend(load_runners) diff --git a/vectordb_bench/backend/cases.py b/vectordb_bench/backend/cases.py index 8b643b66b..15fc069cc 100644 --- a/vectordb_bench/backend/cases.py +++ b/vectordb_bench/backend/cases.py @@ -1,7 +1,5 @@ -import typing import logging from enum import Enum, auto -from typing import Type from vectordb_bench import config from vectordb_bench.backend.clients.api import MetricType @@ -12,7 +10,6 @@ from .dataset import CustomDataset, Dataset, DatasetManager - log = logging.getLogger(__name__) @@ -50,11 +47,10 @@ class CaseType(Enum): Custom = 100 PerformanceCustomDataset = 101 - def case_cls(self, custom_configs: dict | None = None) -> Type["Case"]: + def case_cls(self, custom_configs: dict | None = None) -> type["Case"]: if custom_configs is None: return type2case.get(self)() - else: - return type2case.get(self)(**custom_configs) + return type2case.get(self)(**custom_configs) def case_name(self, custom_configs: dict | None = None) -> str: c = self.case_cls(custom_configs) @@ -99,10 +95,10 @@ class Case(BaseModel): @property def filters(self) -> dict | None: if self.filter_rate is not None: - ID = round(self.filter_rate * self.dataset.data.size) + target_id = round(self.filter_rate * self.dataset.data.size) return { - "metadata": f">={ID}", - "id": ID, + "metadata": f">={target_id}", + "id": target_id, } return None @@ -126,8 +122,8 @@ class CapacityDim960(CapacityCase): case_id: CaseType = CaseType.CapacityDim960 dataset: DatasetManager = Dataset.GIST.manager(100_000) name: str = "Capacity Test (960 Dim Repeated)" - description: str = """This case tests the vector database's loading capacity by repeatedly inserting large-dimension - vectors (GIST 100K vectors, 960 dimensions) until it is fully loaded. Number of inserted vectors will be + description: str = """This case tests the vector database's loading capacity by repeatedly inserting large-dimension + vectors (GIST 100K vectors, 960 dimensions) until it is fully loaded. Number of inserted vectors will be reported.""" @@ -136,7 +132,7 @@ class CapacityDim128(CapacityCase): dataset: DatasetManager = Dataset.SIFT.manager(500_000) name: str = "Capacity Test (128 Dim Repeated)" description: str = """This case tests the vector database's loading capacity by repeatedly inserting small-dimension - vectors (SIFT 100K vectors, 128 dimensions) until it is fully loaded. Number of inserted vectors will be + vectors (SIFT 100K vectors, 128 dimensions) until it is fully loaded. Number of inserted vectors will be reported.""" @@ -144,8 +140,9 @@ class Performance768D10M(PerformanceCase): case_id: CaseType = CaseType.Performance768D10M dataset: DatasetManager = Dataset.COHERE.manager(10_000_000) name: str = "Search Performance Test (10M Dataset, 768 Dim)" - description: str = """This case tests the search performance of a vector database with a large dataset (Cohere 10M vectors, 768 dimensions) at varying parallel levels. -Results will show index building time, recall, and maximum QPS.""" + description: str = """This case tests the search performance of a vector database with a large dataset + (Cohere 10M vectors, 768 dimensions) at varying parallel levels. + Results will show index building time, recall, and maximum QPS.""" load_timeout: float | int = config.LOAD_TIMEOUT_768D_10M optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_768D_10M @@ -154,8 +151,9 @@ class Performance768D1M(PerformanceCase): case_id: CaseType = CaseType.Performance768D1M dataset: DatasetManager = Dataset.COHERE.manager(1_000_000) name: str = "Search Performance Test (1M Dataset, 768 Dim)" - description: str = """This case tests the search performance of a vector database with a medium dataset (Cohere 1M vectors, 768 dimensions) at varying parallel levels. -Results will show index building time, recall, and maximum QPS.""" + description: str = """This case tests the search performance of a vector database with a medium dataset + (Cohere 1M vectors, 768 dimensions) at varying parallel levels. + Results will show index building time, recall, and maximum QPS.""" load_timeout: float | int = config.LOAD_TIMEOUT_768D_1M optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_768D_1M @@ -165,8 +163,9 @@ class Performance768D10M1P(PerformanceCase): filter_rate: float | int | None = 0.01 dataset: DatasetManager = Dataset.COHERE.manager(10_000_000) name: str = "Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 1%)" - description: str = """This case tests the search performance of a vector database with a large dataset (Cohere 10M vectors, 768 dimensions) under a low filtering rate (1% vectors), at varying parallel levels. -Results will show index building time, recall, and maximum QPS.""" + description: str = """This case tests the search performance of a vector database with a large dataset + (Cohere 10M vectors, 768 dimensions) under a low filtering rate (1% vectors), at varying parallel + levels. Results will show index building time, recall, and maximum QPS.""" load_timeout: float | int = config.LOAD_TIMEOUT_768D_10M optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_768D_10M @@ -176,8 +175,9 @@ class Performance768D1M1P(PerformanceCase): filter_rate: float | int | None = 0.01 dataset: DatasetManager = Dataset.COHERE.manager(1_000_000) name: str = "Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 1%)" - description: str = """This case tests the search performance of a vector database with a medium dataset (Cohere 1M vectors, 768 dimensions) under a low filtering rate (1% vectors), at varying parallel levels. -Results will show index building time, recall, and maximum QPS.""" + description: str = """This case tests the search performance of a vector database with a medium dataset + (Cohere 1M vectors, 768 dimensions) under a low filtering rate (1% vectors), + at varying parallel levels. Results will show index building time, recall, and maximum QPS.""" load_timeout: float | int = config.LOAD_TIMEOUT_768D_1M optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_768D_1M @@ -187,8 +187,9 @@ class Performance768D10M99P(PerformanceCase): filter_rate: float | int | None = 0.99 dataset: DatasetManager = Dataset.COHERE.manager(10_000_000) name: str = "Filtering Search Performance Test (10M Dataset, 768 Dim, Filter 99%)" - description: str = """This case tests the search performance of a vector database with a large dataset (Cohere 10M vectors, 768 dimensions) under a high filtering rate (99% vectors), at varying parallel levels. -Results will show index building time, recall, and maximum QPS.""" + description: str = """This case tests the search performance of a vector database with a large dataset + (Cohere 10M vectors, 768 dimensions) under a high filtering rate (99% vectors), + at varying parallel levels. Results will show index building time, recall, and maximum QPS.""" load_timeout: float | int = config.LOAD_TIMEOUT_768D_10M optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_768D_10M @@ -198,8 +199,9 @@ class Performance768D1M99P(PerformanceCase): filter_rate: float | int | None = 0.99 dataset: DatasetManager = Dataset.COHERE.manager(1_000_000) name: str = "Filtering Search Performance Test (1M Dataset, 768 Dim, Filter 99%)" - description: str = """This case tests the search performance of a vector database with a medium dataset (Cohere 1M vectors, 768 dimensions) under a high filtering rate (99% vectors), at varying parallel levels. -Results will show index building time, recall, and maximum QPS.""" + description: str = """This case tests the search performance of a vector database with a medium dataset + (Cohere 1M vectors, 768 dimensions) under a high filtering rate (99% vectors), + at varying parallel levels. Results will show index building time, recall, and maximum QPS.""" load_timeout: float | int = config.LOAD_TIMEOUT_768D_1M optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_768D_1M @@ -209,8 +211,9 @@ class Performance768D100M(PerformanceCase): filter_rate: float | int | None = None dataset: DatasetManager = Dataset.LAION.manager(100_000_000) name: str = "Search Performance Test (100M Dataset, 768 Dim)" - description: str = """This case tests the search performance of a vector database with a large 100M dataset (LAION 100M vectors, 768 dimensions), at varying parallel levels. -Results will show index building time, recall, and maximum QPS.""" + description: str = """This case tests the search performance of a vector database with a large 100M dataset + (LAION 100M vectors, 768 dimensions), at varying parallel levels. Results will show index building time, + recall, and maximum QPS.""" load_timeout: float | int = config.LOAD_TIMEOUT_768D_100M optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_768D_100M @@ -220,8 +223,9 @@ class Performance1536D500K(PerformanceCase): filter_rate: float | int | None = None dataset: DatasetManager = Dataset.OPENAI.manager(500_000) name: str = "Search Performance Test (500K Dataset, 1536 Dim)" - description: str = """This case tests the search performance of a vector database with a medium 500K dataset (OpenAI 500K vectors, 1536 dimensions), at varying parallel levels. -Results will show index building time, recall, and maximum QPS.""" + description: str = """This case tests the search performance of a vector database with a medium 500K dataset + (OpenAI 500K vectors, 1536 dimensions), at varying parallel levels. Results will show index building time, + recall, and maximum QPS.""" load_timeout: float | int = config.LOAD_TIMEOUT_1536D_500K optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_1536D_500K @@ -231,8 +235,9 @@ class Performance1536D5M(PerformanceCase): filter_rate: float | int | None = None dataset: DatasetManager = Dataset.OPENAI.manager(5_000_000) name: str = "Search Performance Test (5M Dataset, 1536 Dim)" - description: str = """This case tests the search performance of a vector database with a medium 5M dataset (OpenAI 5M vectors, 1536 dimensions), at varying parallel levels. -Results will show index building time, recall, and maximum QPS.""" + description: str = """This case tests the search performance of a vector database with a medium 5M dataset + (OpenAI 5M vectors, 1536 dimensions), at varying parallel levels. Results will show index building time, + recall, and maximum QPS.""" load_timeout: float | int = config.LOAD_TIMEOUT_1536D_5M optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_1536D_5M @@ -242,8 +247,9 @@ class Performance1536D500K1P(PerformanceCase): filter_rate: float | int | None = 0.01 dataset: DatasetManager = Dataset.OPENAI.manager(500_000) name: str = "Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 1%)" - description: str = """This case tests the search performance of a vector database with a large dataset (OpenAI 500K vectors, 1536 dimensions) under a low filtering rate (1% vectors), at varying parallel levels. -Results will show index building time, recall, and maximum QPS.""" + description: str = """This case tests the search performance of a vector database with a large dataset + (OpenAI 500K vectors, 1536 dimensions) under a low filtering rate (1% vectors), + at varying parallel levels. Results will show index building time, recall, and maximum QPS.""" load_timeout: float | int = config.LOAD_TIMEOUT_1536D_500K optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_1536D_500K @@ -253,8 +259,9 @@ class Performance1536D5M1P(PerformanceCase): filter_rate: float | int | None = 0.01 dataset: DatasetManager = Dataset.OPENAI.manager(5_000_000) name: str = "Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 1%)" - description: str = """This case tests the search performance of a vector database with a large dataset (OpenAI 5M vectors, 1536 dimensions) under a low filtering rate (1% vectors), at varying parallel levels. -Results will show index building time, recall, and maximum QPS.""" + description: str = """This case tests the search performance of a vector database with a large dataset + (OpenAI 5M vectors, 1536 dimensions) under a low filtering rate (1% vectors), + at varying parallel levels. Results will show index building time, recall, and maximum QPS.""" load_timeout: float | int = config.LOAD_TIMEOUT_1536D_5M optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_1536D_5M @@ -264,8 +271,9 @@ class Performance1536D500K99P(PerformanceCase): filter_rate: float | int | None = 0.99 dataset: DatasetManager = Dataset.OPENAI.manager(500_000) name: str = "Filtering Search Performance Test (500K Dataset, 1536 Dim, Filter 99%)" - description: str = """This case tests the search performance of a vector database with a medium dataset (OpenAI 500K vectors, 1536 dimensions) under a high filtering rate (99% vectors), at varying parallel levels. -Results will show index building time, recall, and maximum QPS.""" + description: str = """This case tests the search performance of a vector database with a medium dataset + (OpenAI 500K vectors, 1536 dimensions) under a high filtering rate (99% vectors), + at varying parallel levels. Results will show index building time, recall, and maximum QPS.""" load_timeout: float | int = config.LOAD_TIMEOUT_1536D_500K optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_1536D_500K @@ -275,8 +283,9 @@ class Performance1536D5M99P(PerformanceCase): filter_rate: float | int | None = 0.99 dataset: DatasetManager = Dataset.OPENAI.manager(5_000_000) name: str = "Filtering Search Performance Test (5M Dataset, 1536 Dim, Filter 99%)" - description: str = """This case tests the search performance of a vector database with a medium dataset (OpenAI 5M vectors, 1536 dimensions) under a high filtering rate (99% vectors), at varying parallel levels. -Results will show index building time, recall, and maximum QPS.""" + description: str = """This case tests the search performance of a vector database with a medium dataset + (OpenAI 5M vectors, 1536 dimensions) under a high filtering rate (99% vectors), + at varying parallel levels. Results will show index building time, recall, and maximum QPS.""" load_timeout: float | int = config.LOAD_TIMEOUT_1536D_5M optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_1536D_5M @@ -286,8 +295,9 @@ class Performance1536D50K(PerformanceCase): filter_rate: float | int | None = None dataset: DatasetManager = Dataset.OPENAI.manager(50_000) name: str = "Search Performance Test (50K Dataset, 1536 Dim)" - description: str = """This case tests the search performance of a vector database with a medium 50K dataset (OpenAI 50K vectors, 1536 dimensions), at varying parallel levels. -Results will show index building time, recall, and maximum QPS.""" + description: str = """This case tests the search performance of a vector database with a medium 50K dataset + (OpenAI 50K vectors, 1536 dimensions), at varying parallel levels. Results will show index building time, + recall, and maximum QPS.""" load_timeout: float | int = 3600 optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_DEFAULT @@ -312,11 +322,11 @@ class PerformanceCustomDataset(PerformanceCase): def __init__( self, - name, - description, - load_timeout, - optimize_timeout, - dataset_config, + name: str, + description: str, + load_timeout: float, + optimize_timeout: float, + dataset_config: dict, **kwargs, ): dataset_config = CustomDatasetConfig(**dataset_config) diff --git a/vectordb_bench/backend/clients/__init__.py b/vectordb_bench/backend/clients/__init__.py index e1b66a81d..773cd4948 100644 --- a/vectordb_bench/backend/clients/__init__.py +++ b/vectordb_bench/backend/clients/__init__.py @@ -1,12 +1,12 @@ from enum import Enum -from typing import Type + from .api import ( - VectorDB, - DBConfig, DBCaseConfig, + DBConfig, EmptyDBCaseConfig, IndexType, MetricType, + VectorDB, ) @@ -41,200 +41,255 @@ class DB(Enum): Test = "test" AliyunOpenSearch = "AliyunOpenSearch" - @property - def init_cls(self) -> Type[VectorDB]: + def init_cls(self) -> type[VectorDB]: # noqa: PLR0911, PLR0912 """Import while in use""" if self == DB.Milvus: from .milvus.milvus import Milvus + return Milvus if self == DB.ZillizCloud: from .zilliz_cloud.zilliz_cloud import ZillizCloud + return ZillizCloud if self == DB.Pinecone: from .pinecone.pinecone import Pinecone + return Pinecone if self == DB.ElasticCloud: from .elastic_cloud.elastic_cloud import ElasticCloud + return ElasticCloud if self == DB.QdrantCloud: from .qdrant_cloud.qdrant_cloud import QdrantCloud + return QdrantCloud if self == DB.WeaviateCloud: from .weaviate_cloud.weaviate_cloud import WeaviateCloud + return WeaviateCloud if self == DB.PgVector: from .pgvector.pgvector import PgVector + return PgVector if self == DB.PgVectoRS: from .pgvecto_rs.pgvecto_rs import PgVectoRS + return PgVectoRS - + if self == DB.PgVectorScale: from .pgvectorscale.pgvectorscale import PgVectorScale + return PgVectorScale if self == DB.PgDiskANN: from .pgdiskann.pgdiskann import PgDiskANN + return PgDiskANN if self == DB.Redis: from .redis.redis import Redis + return Redis - + if self == DB.MemoryDB: from .memorydb.memorydb import MemoryDB + return MemoryDB if self == DB.Chroma: from .chroma.chroma import ChromaClient + return ChromaClient if self == DB.AWSOpenSearch: from .aws_opensearch.aws_opensearch import AWSOpenSearch + return AWSOpenSearch - + if self == DB.AlloyDB: from .alloydb.alloydb import AlloyDB + return AlloyDB if self == DB.AliyunElasticsearch: from .aliyun_elasticsearch.aliyun_elasticsearch import AliyunElasticsearch + return AliyunElasticsearch if self == DB.AliyunOpenSearch: from .aliyun_opensearch.aliyun_opensearch import AliyunOpenSearch + return AliyunOpenSearch + msg = f"Unknown DB: {self.name}" + raise ValueError(msg) + @property - def config_cls(self) -> Type[DBConfig]: + def config_cls(self) -> type[DBConfig]: # noqa: PLR0911, PLR0912 """Import while in use""" if self == DB.Milvus: from .milvus.config import MilvusConfig + return MilvusConfig if self == DB.ZillizCloud: from .zilliz_cloud.config import ZillizCloudConfig + return ZillizCloudConfig if self == DB.Pinecone: from .pinecone.config import PineconeConfig + return PineconeConfig if self == DB.ElasticCloud: from .elastic_cloud.config import ElasticCloudConfig + return ElasticCloudConfig if self == DB.QdrantCloud: from .qdrant_cloud.config import QdrantConfig + return QdrantConfig if self == DB.WeaviateCloud: from .weaviate_cloud.config import WeaviateConfig + return WeaviateConfig if self == DB.PgVector: from .pgvector.config import PgVectorConfig + return PgVectorConfig if self == DB.PgVectoRS: from .pgvecto_rs.config import PgVectoRSConfig + return PgVectoRSConfig if self == DB.PgVectorScale: from .pgvectorscale.config import PgVectorScaleConfig + return PgVectorScaleConfig if self == DB.PgDiskANN: from .pgdiskann.config import PgDiskANNConfig + return PgDiskANNConfig if self == DB.Redis: from .redis.config import RedisConfig + return RedisConfig - + if self == DB.MemoryDB: from .memorydb.config import MemoryDBConfig + return MemoryDBConfig if self == DB.Chroma: from .chroma.config import ChromaConfig + return ChromaConfig if self == DB.AWSOpenSearch: from .aws_opensearch.config import AWSOpenSearchConfig + return AWSOpenSearchConfig - + if self == DB.AlloyDB: from .alloydb.config import AlloyDBConfig + return AlloyDBConfig if self == DB.AliyunElasticsearch: from .aliyun_elasticsearch.config import AliyunElasticsearchConfig + return AliyunElasticsearchConfig if self == DB.AliyunOpenSearch: from .aliyun_opensearch.config import AliyunOpenSearchConfig + return AliyunOpenSearchConfig - def case_config_cls(self, index_type: IndexType | None = None) -> Type[DBCaseConfig]: + msg = f"Unknown DB: {self.name}" + raise ValueError(msg) + + def case_config_cls( # noqa: PLR0911 + self, + index_type: IndexType | None = None, + ) -> type[DBCaseConfig]: if self == DB.Milvus: from .milvus.config import _milvus_case_config + return _milvus_case_config.get(index_type) if self == DB.ZillizCloud: from .zilliz_cloud.config import AutoIndexConfig + return AutoIndexConfig if self == DB.ElasticCloud: from .elastic_cloud.config import ElasticCloudIndexConfig + return ElasticCloudIndexConfig if self == DB.QdrantCloud: from .qdrant_cloud.config import QdrantIndexConfig + return QdrantIndexConfig if self == DB.WeaviateCloud: from .weaviate_cloud.config import WeaviateIndexConfig + return WeaviateIndexConfig if self == DB.PgVector: from .pgvector.config import _pgvector_case_config + return _pgvector_case_config.get(index_type) if self == DB.PgVectoRS: from .pgvecto_rs.config import _pgvecto_rs_case_config + return _pgvecto_rs_case_config.get(index_type) if self == DB.AWSOpenSearch: from .aws_opensearch.config import AWSOpenSearchIndexConfig + return AWSOpenSearchIndexConfig if self == DB.PgVectorScale: from .pgvectorscale.config import _pgvectorscale_case_config + return _pgvectorscale_case_config.get(index_type) if self == DB.PgDiskANN: from .pgdiskann.config import _pgdiskann_case_config + return _pgdiskann_case_config.get(index_type) - + if self == DB.AlloyDB: from .alloydb.config import _alloydb_case_config + return _alloydb_case_config.get(index_type) if self == DB.AliyunElasticsearch: from .elastic_cloud.config import ElasticCloudIndexConfig + return ElasticCloudIndexConfig if self == DB.AliyunOpenSearch: from .aliyun_opensearch.config import AliyunOpenSearchIndexConfig + return AliyunOpenSearchIndexConfig # DB.Pinecone, DB.Chroma, DB.Redis @@ -242,5 +297,11 @@ def case_config_cls(self, index_type: IndexType | None = None) -> Type[DBCaseCon __all__ = [ - "DB", "VectorDB", "DBConfig", "DBCaseConfig", "IndexType", "MetricType", "EmptyDBCaseConfig", + "DB", + "DBCaseConfig", + "DBConfig", + "EmptyDBCaseConfig", + "IndexType", + "MetricType", + "VectorDB", ] diff --git a/vectordb_bench/backend/clients/aliyun_elasticsearch/aliyun_elasticsearch.py b/vectordb_bench/backend/clients/aliyun_elasticsearch/aliyun_elasticsearch.py index 41253ca1e..92ec905e5 100644 --- a/vectordb_bench/backend/clients/aliyun_elasticsearch/aliyun_elasticsearch.py +++ b/vectordb_bench/backend/clients/aliyun_elasticsearch/aliyun_elasticsearch.py @@ -1,5 +1,5 @@ -from ..elastic_cloud.elastic_cloud import ElasticCloud from ..elastic_cloud.config import ElasticCloudIndexConfig +from ..elastic_cloud.elastic_cloud import ElasticCloud class AliyunElasticsearch(ElasticCloud): @@ -24,4 +24,3 @@ def __init__( drop_old=drop_old, **kwargs, ) - diff --git a/vectordb_bench/backend/clients/aliyun_elasticsearch/config.py b/vectordb_bench/backend/clients/aliyun_elasticsearch/config.py index a2de4dc75..a8f5100bf 100644 --- a/vectordb_bench/backend/clients/aliyun_elasticsearch/config.py +++ b/vectordb_bench/backend/clients/aliyun_elasticsearch/config.py @@ -1,7 +1,6 @@ -from enum import Enum -from pydantic import SecretStr, BaseModel +from pydantic import BaseModel, SecretStr -from ..api import DBConfig, DBCaseConfig, MetricType, IndexType +from ..api import DBConfig class AliyunElasticsearchConfig(DBConfig, BaseModel): @@ -14,6 +13,6 @@ class AliyunElasticsearchConfig(DBConfig, BaseModel): def to_dict(self) -> dict: return { - "hosts": [{'scheme': self.scheme, 'host': self.host, 'port': self.port}], + "hosts": [{"scheme": self.scheme, "host": self.host, "port": self.port}], "basic_auth": (self.user, self.password.get_secret_value()), } diff --git a/vectordb_bench/backend/clients/aliyun_opensearch/aliyun_opensearch.py b/vectordb_bench/backend/clients/aliyun_opensearch/aliyun_opensearch.py index 5d4dcbfa6..00227cfff 100644 --- a/vectordb_bench/backend/clients/aliyun_opensearch/aliyun_opensearch.py +++ b/vectordb_bench/backend/clients/aliyun_opensearch/aliyun_opensearch.py @@ -1,32 +1,32 @@ import json import logging -from contextlib import contextmanager import time +from contextlib import contextmanager +from alibabacloud_ha3engine_vector import client, models from alibabacloud_ha3engine_vector.models import QueryRequest - -from ..api import VectorDB, MetricType -from .config import AliyunOpenSearchIndexConfig - -from alibabacloud_searchengine20211025.client import Client as searchengineClient from alibabacloud_searchengine20211025 import models as searchengine_models +from alibabacloud_searchengine20211025.client import Client as searchengineClient from alibabacloud_tea_openapi import models as open_api_models -from alibabacloud_ha3engine_vector import models, client + +from ..api import MetricType, VectorDB +from .config import AliyunOpenSearchIndexConfig log = logging.getLogger(__name__) ALIYUN_OPENSEARCH_MAX_SIZE_PER_BATCH = 2 * 1024 * 1024 # 2MB ALIYUN_OPENSEARCH_MAX_NUM_PER_BATCH = 100 + class AliyunOpenSearch(VectorDB): def __init__( - self, - dim: int, - db_config: dict, - db_case_config: AliyunOpenSearchIndexConfig, - collection_name: str = "VectorDBBenchCollection", - drop_old: bool = False, - **kwargs, + self, + dim: int, + db_config: dict, + db_case_config: AliyunOpenSearchIndexConfig, + collection_name: str = "VectorDBBenchCollection", + drop_old: bool = False, + **kwargs, ): self.control_client = None self.dim = dim @@ -41,14 +41,17 @@ def __init__( self._index_name = "vector_idx" self.batch_size = int( - min(ALIYUN_OPENSEARCH_MAX_SIZE_PER_BATCH / (dim * 25), ALIYUN_OPENSEARCH_MAX_NUM_PER_BATCH) + min( + ALIYUN_OPENSEARCH_MAX_SIZE_PER_BATCH / (dim * 25), + ALIYUN_OPENSEARCH_MAX_NUM_PER_BATCH, + ), ) log.info(f"Aliyun_OpenSearch client config: {self.db_config}") control_config = open_api_models.Config( access_key_id=self.db_config["ak"], access_key_secret=self.db_config["sk"], - endpoint=self.db_config["control_host"] + endpoint=self.db_config["control_host"], ) self.control_client = searchengineClient(control_config) @@ -67,7 +70,7 @@ def _create_index(self, client: searchengineClient): create_table_request.field_schema = { self._primary_field: "INT64", self._vector_field: "MULTI_FLOAT", - self._scalar_field: "INT64" + self._scalar_field: "INT64", } vector_index = searchengine_models.ModifyTableRequestVectorIndex() vector_index.index_name = self._index_name @@ -77,8 +80,25 @@ def _create_index(self, client: searchengineClient): vector_index.vector_index_type = "HNSW" advance_params = searchengine_models.ModifyTableRequestVectorIndexAdvanceParams() - advance_params.build_index_params = "{\"proxima.hnsw.builder.max_neighbor_count\":" + str(self.case_config.M) + ",\"proxima.hnsw.builder.efconstruction\":" + str(self.case_config.efConstruction) + ",\"proxima.hnsw.builder.enable_adsampling\":true,\"proxima.hnsw.builder.slack_pruning_factor\":1.1,\"proxima.hnsw.builder.thread_count\":16}" - advance_params.search_index_params = "{\"proxima.hnsw.searcher.ef\":400,\"proxima.hnsw.searcher.dynamic_termination.prob_threshold\":0.7}" + str_max_neighbor_count = f'"proxima.hnsw.builder.max_neighbor_count":{self.case_config.M}' + str_efc = f'"proxima.hnsw.builder.efconstruction":{self.case_config.ef_construction}' + str_enable_adsampling = '"proxima.hnsw.builder.enable_adsampling":true' + str_slack_pruning_factor = '"proxima.hnsw.builder.slack_pruning_factor":1.1' + str_thread_count = '"proxima.hnsw.builder.thread_count":16' + + params = ",".join( + [ + str_max_neighbor_count, + str_efc, + str_enable_adsampling, + str_slack_pruning_factor, + str_thread_count, + ], + ) + advance_params.build_index_params = params + advance_params.search_index_params = ( + '{"proxima.hnsw.searcher.ef":400,"proxima.hnsw.searcher.dynamic_termination.prob_threshold":0.7}' + ) vector_index.advance_params = advance_params create_table_request.vector_index = [vector_index] @@ -88,7 +108,7 @@ def _create_index(self, client: searchengineClient): except Exception as error: log.info(error.message) log.info(error.data.get("Recommend")) - log.info(f"Failed to create index: error: {str(error)}") + log.info(f"Failed to create index: error: {error!s}") raise error from None # check if index create success @@ -102,22 +122,22 @@ def _active_index(self, client: searchengineClient) -> None: log.info(f"begin to {retry_times} times get table") retry_times += 1 response = client.get_table(self.instance_id, self.collection_name) - if response.body.result.status == 'IN_USE': + if response.body.result.status == "IN_USE": log.info(f"{self.collection_name} table begin to use.") return def _index_exists(self, client: searchengineClient) -> bool: try: client.get_table(self.instance_id, self.collection_name) - return True - except Exception as error: - log.info(f'get table from searchengine error') - log.info(error.message) + except Exception as err: + log.warning(f"get table from searchengine error, err={err}") return False + else: + return True # check if index build success, Insert the embeddings to the vector database after index build success def _index_build_success(self, client: searchengineClient) -> None: - log.info(f"begin to check if table build success.") + log.info("begin to check if table build success.") time.sleep(50) retry_times = 0 @@ -139,9 +159,9 @@ def _index_build_success(self, client: searchengineClient) -> None: cur_fsm = fsm break if cur_fsm is None: - print("no build index fsm") + log.warning("no build index fsm") return - if "success" == cur_fsm["status"]: + if cur_fsm["status"] == "success": return def _modify_index(self, client: searchengineClient) -> None: @@ -154,7 +174,7 @@ def _modify_index(self, client: searchengineClient) -> None: modify_table_request.field_schema = { self._primary_field: "INT64", self._vector_field: "MULTI_FLOAT", - self._scalar_field: "INT64" + self._scalar_field: "INT64", } vector_index = searchengine_models.ModifyTableRequestVectorIndex() vector_index.index_name = self._index_name @@ -163,19 +183,41 @@ def _modify_index(self, client: searchengineClient) -> None: vector_index.vector_field = self._vector_field vector_index.vector_index_type = "HNSW" advance_params = searchengine_models.ModifyTableRequestVectorIndexAdvanceParams() - advance_params.build_index_params = "{\"proxima.hnsw.builder.max_neighbor_count\":" + str(self.case_config.M) + ",\"proxima.hnsw.builder.efconstruction\":" + str(self.case_config.efConstruction) + ",\"proxima.hnsw.builder.enable_adsampling\":true,\"proxima.hnsw.builder.slack_pruning_factor\":1.1,\"proxima.hnsw.builder.thread_count\":16}" - advance_params.search_index_params = "{\"proxima.hnsw.searcher.ef\":400,\"proxima.hnsw.searcher.dynamic_termination.prob_threshold\":0.7}" + + str_max_neighbor_count = f'"proxima.hnsw.builder.max_neighbor_count":{self.case_config.M}' + str_efc = f'"proxima.hnsw.builder.efconstruction":{self.case_config.ef_construction}' + str_enable_adsampling = '"proxima.hnsw.builder.enable_adsampling":true' + str_slack_pruning_factor = '"proxima.hnsw.builder.slack_pruning_factor":1.1' + str_thread_count = '"proxima.hnsw.builder.thread_count":16' + + params = ",".join( + [ + str_max_neighbor_count, + str_efc, + str_enable_adsampling, + str_slack_pruning_factor, + str_thread_count, + ], + ) + advance_params.build_index_params = params + advance_params.search_index_params = ( + '{"proxima.hnsw.searcher.ef":400,"proxima.hnsw.searcher.dynamic_termination.prob_threshold":0.7}' + ) vector_index.advance_params = advance_params modify_table_request.vector_index = [vector_index] try: - response = client.modify_table(self.instance_id, self.collection_name, modify_table_request) + response = client.modify_table( + self.instance_id, + self.collection_name, + modify_table_request, + ) log.info(f"modify table success: {response.body}") except Exception as error: log.info(error.message) log.info(error.data.get("Recommend")) - log.info(f"Failed to modify index: error: {str(error)}") + log.info(f"Failed to modify index: error: {error!s}") raise error from None # check if modify index & delete data fsm success @@ -185,15 +227,14 @@ def _modify_index(self, client: searchengineClient) -> None: def _get_total_count(self): try: response = self.client.stats(self.collection_name) + except Exception as e: + log.warning(f"Error querying index: {e}") + else: body = json.loads(response.body) log.info(f"stats info: {response.body}") if "result" in body and "totalDocCount" in body.get("result"): return body.get("result").get("totalDocCount") - else: - return 0 - except Exception as e: - print(f"Error querying index: {e}") return 0 @contextmanager @@ -203,21 +244,20 @@ def init(self) -> None: endpoint=self.db_config["host"], protocol="http", access_user_name=self.db_config["user"], - access_pass_word=self.db_config["password"] + access_pass_word=self.db_config["password"], ) self.client = client.Client(config) yield - # self.client.transport.close() self.client = None del self.client def insert_embeddings( - self, - embeddings: list[list[float]], - metadata: list[int], - **kwargs, + self, + embeddings: list[list[float]], + metadata: list[int], + **kwargs, ) -> tuple[int, Exception]: """Insert the embeddings to the opensearch.""" assert self.client is not None, "should self.init() first" @@ -226,25 +266,24 @@ def insert_embeddings( try: for batch_start_offset in range(0, len(embeddings), self.batch_size): - batch_end_offset = min( - batch_start_offset + self.batch_size, len(embeddings) - ) + batch_end_offset = min(batch_start_offset + self.batch_size, len(embeddings)) documents = [] for i in range(batch_start_offset, batch_end_offset): - documentFields = { + document_fields = { self._primary_field: metadata[i], self._vector_field: embeddings[i], self._scalar_field: metadata[i], - "ops_build_channel": "inc" - } - document = { - "fields": documentFields, - "cmd": "add" + "ops_build_channel": "inc", } + document = {"fields": document_fields, "cmd": "add"} documents.append(document) - pushDocumentsRequest = models.PushDocumentsRequest({}, documents) - self.client.push_documents(self.collection_name, self._primary_field, pushDocumentsRequest) + push_doc_req = models.PushDocumentsRequest({}, documents) + self.client.push_documents( + self.collection_name, + self._primary_field, + push_doc_req, + ) insert_count += batch_end_offset - batch_start_offset except Exception as e: log.info(f"Failed to insert data: {e}") @@ -252,33 +291,36 @@ def insert_embeddings( return (insert_count, None) def search_embedding( - self, - query: list[float], - k: int = 100, - filters: dict | None = None, + self, + query: list[float], + k: int = 100, + filters: dict | None = None, ) -> list[int]: assert self.client is not None, "should self.init() first" - search_params = "{\"proxima.hnsw.searcher.ef\":"+ str(self.case_config.ef_search) +"}" + search_params = '{"proxima.hnsw.searcher.ef":' + str(self.case_config.ef_search) + "}" os_filter = f"{self._scalar_field} {filters.get('metadata')}" if filters else "" try: - request = QueryRequest(table_name=self.collection_name, - vector=query, - top_k=k, - search_params=search_params, filter=os_filter) + request = QueryRequest( + table_name=self.collection_name, + vector=query, + top_k=k, + search_params=search_params, + filter=os_filter, + ) result = self.client.query(request) except Exception as e: log.info(f"Error querying index: {e}") - raise e - res = json.loads(result.body) - id_res = [one_res["id"] for one_res in res["result"]] - return id_res + raise e from e + else: + res = json.loads(result.body) + return [one_res["id"] for one_res in res["result"]] def need_normalize_cosine(self) -> bool: """Wheather this database need to normalize dataset to support COSINE""" if self.case_config.metric_type == MetricType.COSINE: - log.info(f"cosine dataset need normalize.") + log.info("cosine dataset need normalize.") return True return False @@ -296,9 +338,8 @@ def optimize_with_size(self, data_size: int): total_count = self._get_total_count() # check if the data is inserted if total_count == data_size: - log.info(f"optimize table finish.") + log.info("optimize table finish.") return def ready_to_load(self): """ready_to_load will be called before load in load cases.""" - pass diff --git a/vectordb_bench/backend/clients/aliyun_opensearch/config.py b/vectordb_bench/backend/clients/aliyun_opensearch/config.py index 7b2b9ad13..e215b7d68 100644 --- a/vectordb_bench/backend/clients/aliyun_opensearch/config.py +++ b/vectordb_bench/backend/clients/aliyun_opensearch/config.py @@ -1,8 +1,8 @@ import logging -from enum import Enum -from pydantic import SecretStr, BaseModel -from ..api import DBConfig, DBCaseConfig, MetricType, IndexType +from pydantic import BaseModel, SecretStr + +from ..api import DBCaseConfig, DBConfig, MetricType log = logging.getLogger(__name__) @@ -26,18 +26,17 @@ def to_dict(self) -> dict: "control_host": self.control_host, } + class AliyunOpenSearchIndexConfig(BaseModel, DBCaseConfig): metric_type: MetricType = MetricType.L2 - efConstruction: int = 500 + ef_construction: int = 500 M: int = 100 ef_search: int = 40 def distance_type(self) -> str: if self.metric_type == MetricType.L2: return "SquaredEuclidean" - elif self.metric_type == MetricType.IP: - return "InnerProduct" - elif self.metric_type == MetricType.COSINE: + if self.metric_type in (MetricType.IP, MetricType.COSINE): return "InnerProduct" return "SquaredEuclidean" diff --git a/vectordb_bench/backend/clients/alloydb/alloydb.py b/vectordb_bench/backend/clients/alloydb/alloydb.py index 5b275b30f..c81f77675 100644 --- a/vectordb_bench/backend/clients/alloydb/alloydb.py +++ b/vectordb_bench/backend/clients/alloydb/alloydb.py @@ -1,9 +1,9 @@ """Wrapper around the alloydb vector database over VectorDB""" import logging -import pprint +from collections.abc import Generator, Sequence from contextlib import contextmanager -from typing import Any, Generator, Optional, Tuple, Sequence +from typing import Any import numpy as np import psycopg @@ -11,7 +11,7 @@ from psycopg import Connection, Cursor, sql from ..api import VectorDB -from .config import AlloyDBConfigDict, AlloyDBIndexConfig, AlloyDBScaNNConfig +from .config import AlloyDBConfigDict, AlloyDBIndexConfig log = logging.getLogger(__name__) @@ -56,13 +56,14 @@ def __init__( ( self.case_config.create_index_before_load, self.case_config.create_index_after_load, - ) + ), ): - err = f"{self.name} config must create an index using create_index_before_load or create_index_after_load" - log.error(err) - raise RuntimeError( - f"{err}\n{pprint.pformat(self.db_config)}\n{pprint.pformat(self.case_config)}" + msg = ( + f"{self.name} config must create an index using create_index_before_load or create_index_after_load" + "\n{pprint.pformat(self.db_config)}\n{pprint.pformat(self.case_config)}" ) + log.warning(msg) + raise RuntimeError(msg) if drop_old: self._drop_index() @@ -77,7 +78,7 @@ def __init__( self.conn = None @staticmethod - def _create_connection(**kwargs) -> Tuple[Connection, Cursor]: + def _create_connection(**kwargs) -> tuple[Connection, Cursor]: conn = psycopg.connect(**kwargs) register_vector(conn) conn.autocommit = False @@ -86,21 +87,20 @@ def _create_connection(**kwargs) -> Tuple[Connection, Cursor]: assert conn is not None, "Connection is not initialized" assert cursor is not None, "Cursor is not initialized" return conn, cursor - - def _generate_search_query(self, filtered: bool=False) -> sql.Composed: - search_query = sql.Composed( + + def _generate_search_query(self, filtered: bool = False) -> sql.Composed: + return sql.Composed( [ sql.SQL( - "SELECT id FROM public.{table_name} {where_clause} ORDER BY embedding " + "SELECT id FROM public.{table_name} {where_clause} ORDER BY embedding ", ).format( table_name=sql.Identifier(self.table_name), where_clause=sql.SQL("WHERE id >= %s") if filtered else sql.SQL(""), ), sql.SQL(self.case_config.search_param()["metric_fun_op"]), sql.SQL(" %s::vector LIMIT %s::int"), - ] + ], ) - return search_query @contextmanager def init(self) -> Generator[None, None, None]: @@ -119,8 +119,8 @@ def init(self) -> Generator[None, None, None]: if len(session_options) > 0: for setting in session_options: command = sql.SQL("SET {setting_name} " + "= {val};").format( - setting_name=sql.Identifier(setting['parameter']['setting_name']), - val=sql.Identifier(str(setting['parameter']['val'])), + setting_name=sql.Identifier(setting["parameter"]["setting_name"]), + val=sql.Identifier(str(setting["parameter"]["val"])), ) log.debug(command.as_string(self.cursor)) self.cursor.execute(command) @@ -144,8 +144,8 @@ def _drop_table(self): self.cursor.execute( sql.SQL("DROP TABLE IF EXISTS public.{table_name}").format( - table_name=sql.Identifier(self.table_name) - ) + table_name=sql.Identifier(self.table_name), + ), ) self.conn.commit() @@ -167,7 +167,7 @@ def _drop_index(self): log.info(f"{self.name} client drop index : {self._index_name}") drop_index_sql = sql.SQL("DROP INDEX IF EXISTS {index_name}").format( - index_name=sql.Identifier(self._index_name) + index_name=sql.Identifier(self._index_name), ) log.debug(drop_index_sql.as_string(self.cursor)) self.cursor.execute(drop_index_sql) @@ -181,78 +181,64 @@ def _set_parallel_index_build_param(self): if index_param["enable_pca"] is not None: self.cursor.execute( - sql.SQL("SET scann.enable_pca TO {};").format( - index_param["enable_pca"] - ) + sql.SQL("SET scann.enable_pca TO {};").format(index_param["enable_pca"]), ) self.cursor.execute( sql.SQL("ALTER USER {} SET scann.enable_pca TO {};").format( sql.Identifier(self.db_config["user"]), index_param["enable_pca"], - ) + ), ) self.conn.commit() if index_param["maintenance_work_mem"] is not None: self.cursor.execute( sql.SQL("SET maintenance_work_mem TO {};").format( - index_param["maintenance_work_mem"] - ) + index_param["maintenance_work_mem"], + ), ) self.cursor.execute( sql.SQL("ALTER USER {} SET maintenance_work_mem TO {};").format( sql.Identifier(self.db_config["user"]), index_param["maintenance_work_mem"], - ) + ), ) self.conn.commit() if index_param["max_parallel_workers"] is not None: self.cursor.execute( sql.SQL("SET max_parallel_maintenance_workers TO '{}';").format( - index_param["max_parallel_workers"] - ) + index_param["max_parallel_workers"], + ), ) self.cursor.execute( - sql.SQL( - "ALTER USER {} SET max_parallel_maintenance_workers TO '{}';" - ).format( + sql.SQL("ALTER USER {} SET max_parallel_maintenance_workers TO '{}';").format( sql.Identifier(self.db_config["user"]), index_param["max_parallel_workers"], - ) + ), ) self.cursor.execute( sql.SQL("SET max_parallel_workers TO '{}';").format( - index_param["max_parallel_workers"] - ) + index_param["max_parallel_workers"], + ), ) self.cursor.execute( - sql.SQL( - "ALTER USER {} SET max_parallel_workers TO '{}';" - ).format( + sql.SQL("ALTER USER {} SET max_parallel_workers TO '{}';").format( sql.Identifier(self.db_config["user"]), index_param["max_parallel_workers"], - ) + ), ) self.cursor.execute( - sql.SQL( - "ALTER TABLE {} SET (parallel_workers = {});" - ).format( + sql.SQL("ALTER TABLE {} SET (parallel_workers = {});").format( sql.Identifier(self.table_name), index_param["max_parallel_workers"], - ) + ), ) self.conn.commit() - results = self.cursor.execute( - sql.SQL("SHOW max_parallel_maintenance_workers;") - ).fetchall() - results.extend( - self.cursor.execute(sql.SQL("SHOW max_parallel_workers;")).fetchall() - ) - results.extend( - self.cursor.execute(sql.SQL("SHOW maintenance_work_mem;")).fetchall() - ) + results = self.cursor.execute(sql.SQL("SHOW max_parallel_maintenance_workers;")).fetchall() + results.extend(self.cursor.execute(sql.SQL("SHOW max_parallel_workers;")).fetchall()) + results.extend(self.cursor.execute(sql.SQL("SHOW maintenance_work_mem;")).fetchall()) log.info(f"{self.name} parallel index creation parameters: {results}") def _create_index(self): @@ -264,23 +250,20 @@ def _create_index(self): self._set_parallel_index_build_param() options = [] for option in index_param["index_creation_with_options"]: - if option['val'] is not None: + if option["val"] is not None: options.append( sql.SQL("{option_name} = {val}").format( - option_name=sql.Identifier(option['option_name']), - val=sql.Identifier(str(option['val'])), - ) + option_name=sql.Identifier(option["option_name"]), + val=sql.Identifier(str(option["val"])), + ), ) - if any(options): - with_clause = sql.SQL("WITH ({});").format(sql.SQL(", ").join(options)) - else: - with_clause = sql.Composed(()) + with_clause = sql.SQL("WITH ({});").format(sql.SQL(", ").join(options)) if any(options) else sql.Composed(()) index_create_sql = sql.SQL( """ - CREATE INDEX IF NOT EXISTS {index_name} ON public.{table_name} + CREATE INDEX IF NOT EXISTS {index_name} ON public.{table_name} USING {index_type} (embedding {embedding_metric}) - """ + """, ).format( index_name=sql.Identifier(self._index_name), table_name=sql.Identifier(self.table_name), @@ -288,9 +271,7 @@ def _create_index(self): embedding_metric=sql.Identifier(index_param["metric"]), ) - index_create_sql_with_with_clause = ( - index_create_sql + with_clause - ).join(" ") + index_create_sql_with_with_clause = (index_create_sql + with_clause).join(" ") log.debug(index_create_sql_with_with_clause.as_string(self.cursor)) self.cursor.execute(index_create_sql_with_with_clause) self.conn.commit() @@ -305,14 +286,12 @@ def _create_table(self, dim: int): # create table self.cursor.execute( sql.SQL( - "CREATE TABLE IF NOT EXISTS public.{table_name} (id BIGINT PRIMARY KEY, embedding vector({dim}));" - ).format(table_name=sql.Identifier(self.table_name), dim=dim) + "CREATE TABLE IF NOT EXISTS public.{table_name} (id BIGINT PRIMARY KEY, embedding vector({dim}));", + ).format(table_name=sql.Identifier(self.table_name), dim=dim), ) self.conn.commit() except Exception as e: - log.warning( - f"Failed to create alloydb table: {self.table_name} error: {e}" - ) + log.warning(f"Failed to create alloydb table: {self.table_name} error: {e}") raise e from None def insert_embeddings( @@ -320,7 +299,7 @@ def insert_embeddings( embeddings: list[list[float]], metadata: list[int], **kwargs: Any, - ) -> Tuple[int, Optional[Exception]]: + ) -> tuple[int, Exception | None]: assert self.conn is not None, "Connection is not initialized" assert self.cursor is not None, "Cursor is not initialized" @@ -330,8 +309,8 @@ def insert_embeddings( with self.cursor.copy( sql.SQL("COPY public.{table_name} FROM STDIN (FORMAT BINARY)").format( - table_name=sql.Identifier(self.table_name) - ) + table_name=sql.Identifier(self.table_name), + ), ) as copy: copy.set_types(["bigint", "vector"]) for i, row in enumerate(metadata_arr): @@ -343,9 +322,7 @@ def insert_embeddings( return len(metadata), None except Exception as e: - log.warning( - f"Failed to insert data into alloydb table ({self.table_name}), error: {e}" - ) + log.warning(f"Failed to insert data into alloydb table ({self.table_name}), error: {e}") return 0, e def search_embedding( @@ -362,11 +339,12 @@ def search_embedding( if filters: gt = filters.get("id") result = self.cursor.execute( - self._filtered_search, (gt, q, k), prepare=True, binary=True + self._filtered_search, + (gt, q, k), + prepare=True, + binary=True, ) else: - result = self.cursor.execute( - self._unfiltered_search, (q, k), prepare=True, binary=True - ) + result = self.cursor.execute(self._unfiltered_search, (q, k), prepare=True, binary=True) return [int(i[0]) for i in result.fetchall()] diff --git a/vectordb_bench/backend/clients/alloydb/cli.py b/vectordb_bench/backend/clients/alloydb/cli.py index 54d9b9fa3..e207c9c21 100644 --- a/vectordb_bench/backend/clients/alloydb/cli.py +++ b/vectordb_bench/backend/clients/alloydb/cli.py @@ -1,10 +1,10 @@ -from typing import Annotated, Optional, TypedDict, Unpack +import os +from typing import Annotated, Unpack import click -import os from pydantic import SecretStr -from vectordb_bench.backend.clients.api import MetricType +from vectordb_bench.backend.clients import DB from ....cli.cli import ( CommonTypedDict, @@ -13,31 +13,28 @@ get_custom_case_config, run, ) -from vectordb_bench.backend.clients import DB class AlloyDBTypedDict(CommonTypedDict): user_name: Annotated[ - str, click.option("--user-name", type=str, help="Db username", required=True) + str, + click.option("--user-name", type=str, help="Db username", required=True), ] password: Annotated[ str, - click.option("--password", - type=str, - help="Postgres database password", - default=lambda: os.environ.get("POSTGRES_PASSWORD", ""), - show_default="$POSTGRES_PASSWORD", - ), + click.option( + "--password", + type=str, + help="Postgres database password", + default=lambda: os.environ.get("POSTGRES_PASSWORD", ""), + show_default="$POSTGRES_PASSWORD", + ), ] - host: Annotated[ - str, click.option("--host", type=str, help="Db host", required=True) - ] - db_name: Annotated[ - str, click.option("--db-name", type=str, help="Db name", required=True) - ] + host: Annotated[str, click.option("--host", type=str, help="Db host", required=True)] + db_name: Annotated[str, click.option("--db-name", type=str, help="Db name", required=True)] maintenance_work_mem: Annotated[ - Optional[str], + str | None, click.option( "--maintenance-work-mem", type=str, @@ -49,7 +46,7 @@ class AlloyDBTypedDict(CommonTypedDict): ), ] max_parallel_workers: Annotated[ - Optional[int], + int | None, click.option( "--max-parallel-workers", type=int, @@ -58,32 +55,51 @@ class AlloyDBTypedDict(CommonTypedDict): ), ] - class AlloyDBScaNNTypedDict(AlloyDBTypedDict): num_leaves: Annotated[ int, - click.option("--num-leaves", type=int, help="Number of leaves", required=True) + click.option("--num-leaves", type=int, help="Number of leaves", required=True), ] num_leaves_to_search: Annotated[ int, - click.option("--num-leaves-to-search", type=int, help="Number of leaves to search", required=True) + click.option( + "--num-leaves-to-search", + type=int, + help="Number of leaves to search", + required=True, + ), ] pre_reordering_num_neighbors: Annotated[ int, - click.option("--pre-reordering-num-neighbors", type=int, help="Pre-reordering number of neighbors", default=200) + click.option( + "--pre-reordering-num-neighbors", + type=int, + help="Pre-reordering number of neighbors", + default=200, + ), ] max_top_neighbors_buffer_size: Annotated[ int, - click.option("--max-top-neighbors-buffer-size", type=int, help="Maximum top neighbors buffer size", default=20_000) + click.option( + "--max-top-neighbors-buffer-size", + type=int, + help="Maximum top neighbors buffer size", + default=20_000, + ), ] num_search_threads: Annotated[ int, - click.option("--num-search-threads", type=int, help="Number of search threads", default=2) + click.option("--num-search-threads", type=int, help="Number of search threads", default=2), ] max_num_prefetch_datasets: Annotated[ int, - click.option("--max-num-prefetch-datasets", type=int, help="Maximum number of prefetch datasets", default=100) + click.option( + "--max-num-prefetch-datasets", + type=int, + help="Maximum number of prefetch datasets", + default=100, + ), ] quantizer: Annotated[ str, @@ -91,16 +107,17 @@ class AlloyDBScaNNTypedDict(AlloyDBTypedDict): "--quantizer", type=click.Choice(["SQ8", "FLAT"]), help="Quantizer type", - default="SQ8" - ) + default="SQ8", + ), ] enable_pca: Annotated[ - bool, click.option( + bool, + click.option( "--enable-pca", type=click.Choice(["on", "off"]), help="Enable PCA", - default="on" - ) + default="on", + ), ] max_num_levels: Annotated[ int, @@ -108,8 +125,8 @@ class AlloyDBScaNNTypedDict(AlloyDBTypedDict): "--max-num-levels", type=click.Choice(["1", "2"]), help="Maximum number of levels", - default=1 - ) + default=1, + ), ] @@ -144,4 +161,4 @@ def AlloyDBScaNN( maintenance_work_mem=parameters["maintenance_work_mem"], ), **parameters, - ) \ No newline at end of file + ) diff --git a/vectordb_bench/backend/clients/alloydb/config.py b/vectordb_bench/backend/clients/alloydb/config.py index 1d5dde519..d6e54e487 100644 --- a/vectordb_bench/backend/clients/alloydb/config.py +++ b/vectordb_bench/backend/clients/alloydb/config.py @@ -1,7 +1,9 @@ from abc import abstractmethod -from typing import Any, Mapping, Optional, Sequence, TypedDict +from collections.abc import Mapping, Sequence +from typing import Any, LiteralString, TypedDict + from pydantic import BaseModel, SecretStr -from typing_extensions import LiteralString + from ..api import DBCaseConfig, DBConfig, IndexType, MetricType POSTGRE_URL_PLACEHOLDER = "postgresql://%s:%s@%s/%s" @@ -9,7 +11,7 @@ class AlloyDBConfigDict(TypedDict): """These keys will be directly used as kwargs in psycopg connection string, - so the names must match exactly psycopg API""" + so the names must match exactly psycopg API""" user: str password: str @@ -41,8 +43,8 @@ class AlloyDBIndexParam(TypedDict): metric: str index_type: str index_creation_with_options: Sequence[dict[str, Any]] - maintenance_work_mem: Optional[str] - max_parallel_workers: Optional[int] + maintenance_work_mem: str | None + max_parallel_workers: int | None class AlloyDBSearchParam(TypedDict): @@ -61,31 +63,30 @@ class AlloyDBIndexConfig(BaseModel, DBCaseConfig): def parse_metric(self) -> str: if self.metric_type == MetricType.L2: return "l2" - elif self.metric_type == MetricType.DP: + if self.metric_type == MetricType.DP: return "dot_product" return "cosine" def parse_metric_fun_op(self) -> LiteralString: if self.metric_type == MetricType.L2: return "<->" - elif self.metric_type == MetricType.IP: + if self.metric_type == MetricType.IP: return "<#>" return "<=>" @abstractmethod - def index_param(self) -> AlloyDBIndexParam: - ... + def index_param(self) -> AlloyDBIndexParam: ... @abstractmethod - def search_param(self) -> AlloyDBSearchParam: - ... + def search_param(self) -> AlloyDBSearchParam: ... @abstractmethod - def session_param(self) -> AlloyDBSessionCommands: - ... + def session_param(self) -> AlloyDBSessionCommands: ... @staticmethod - def _optionally_build_with_options(with_options: Mapping[str, Any]) -> Sequence[dict[str, Any]]: + def _optionally_build_with_options( + with_options: Mapping[str, Any], + ) -> Sequence[dict[str, Any]]: """Walk through mappings, creating a List of {key1 = value} pairs. That will be used to build a where clause""" options = [] for option_name, value in with_options.items(): @@ -94,24 +95,25 @@ def _optionally_build_with_options(with_options: Mapping[str, Any]) -> Sequence[ { "option_name": option_name, "val": str(value), - } + }, ) return options @staticmethod def _optionally_build_set_options( - set_mapping: Mapping[str, Any] + set_mapping: Mapping[str, Any], ) -> Sequence[dict[str, Any]]: """Walk through options, creating 'SET 'key1 = "value1";' list""" session_options = [] for setting_name, value in set_mapping.items(): if value: session_options.append( - {"parameter": { + { + "parameter": { "setting_name": setting_name, "val": str(value), }, - } + }, ) return session_options @@ -124,22 +126,22 @@ class AlloyDBScaNNConfig(AlloyDBIndexConfig): max_num_levels: int | None num_leaves_to_search: int | None max_top_neighbors_buffer_size: int | None - pre_reordering_num_neighbors: int | None - num_search_threads: int | None + pre_reordering_num_neighbors: int | None + num_search_threads: int | None max_num_prefetch_datasets: int | None - maintenance_work_mem: Optional[str] = None - max_parallel_workers: Optional[int] = None + maintenance_work_mem: str | None = None + max_parallel_workers: int | None = None def index_param(self) -> AlloyDBIndexParam: index_parameters = { - "num_leaves": self.num_leaves, "max_num_levels": self.max_num_levels, "quantizer": self.quantizer, + "num_leaves": self.num_leaves, + "max_num_levels": self.max_num_levels, + "quantizer": self.quantizer, } return { "metric": self.parse_metric(), "index_type": self.index.value, - "index_creation_with_options": self._optionally_build_with_options( - index_parameters - ), + "index_creation_with_options": self._optionally_build_with_options(index_parameters), "maintenance_work_mem": self.maintenance_work_mem, "max_parallel_workers": self.max_parallel_workers, "enable_pca": self.enable_pca, @@ -158,11 +160,9 @@ def session_param(self) -> AlloyDBSessionCommands: "scann.num_search_threads": self.num_search_threads, "scann.max_num_prefetch_datasets": self.max_num_prefetch_datasets, } - return { - "session_options": self._optionally_build_set_options(session_parameters) - } + return {"session_options": self._optionally_build_set_options(session_parameters)} _alloydb_case_config = { - IndexType.SCANN: AlloyDBScaNNConfig, + IndexType.SCANN: AlloyDBScaNNConfig, } diff --git a/vectordb_bench/backend/clients/api.py b/vectordb_bench/backend/clients/api.py index fe2e554f3..aa93abc12 100644 --- a/vectordb_bench/backend/clients/api.py +++ b/vectordb_bench/backend/clients/api.py @@ -1,9 +1,8 @@ from abc import ABC, abstractmethod -from enum import Enum -from typing import Any, Type from contextlib import contextmanager +from enum import Enum -from pydantic import BaseModel, validator, SecretStr +from pydantic import BaseModel, SecretStr, validator class MetricType(str, Enum): @@ -65,13 +64,10 @@ def to_dict(self) -> dict: raise NotImplementedError @validator("*") - def not_empty_field(cls, v, field): - if ( - field.name in cls.common_short_configs() - or field.name in cls.common_long_configs() - ): + def not_empty_field(cls, v: any, field: any): + if field.name in cls.common_short_configs() or field.name in cls.common_long_configs(): return v - if not v and isinstance(v, (str, SecretStr)): + if not v and isinstance(v, str | SecretStr): raise ValueError("Empty string!") return v diff --git a/vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py b/vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py index 60e519d7a..487ec67cc 100644 --- a/vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py +++ b/vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py @@ -1,14 +1,18 @@ import logging -from contextlib import contextmanager import time -from typing import Iterable, Type -from ..api import VectorDB, DBCaseConfig, DBConfig, IndexType -from .config import AWSOpenSearchConfig, AWSOpenSearchIndexConfig, AWSOS_Engine +from collections.abc import Iterable +from contextlib import contextmanager + from opensearchpy import OpenSearch -from opensearchpy.helpers import bulk + +from ..api import IndexType, VectorDB +from .config import AWSOpenSearchConfig, AWSOpenSearchIndexConfig, AWSOS_Engine log = logging.getLogger(__name__) +WAITING_FOR_REFRESH_SEC = 30 +WAITING_FOR_FORCE_MERGE_SEC = 30 + class AWSOpenSearch(VectorDB): def __init__( @@ -27,9 +31,7 @@ def __init__( self.case_config = db_case_config self.index_name = index_name self.id_col_name = id_col_name - self.category_col_names = [ - f"scalar-{categoryCount}" for categoryCount in [2, 5, 10, 100, 1000] - ] + self.category_col_names = [f"scalar-{categoryCount}" for categoryCount in [2, 5, 10, 100, 1000]] self.vector_col_name = vector_col_name log.info(f"AWS_OpenSearch client config: {self.db_config}") @@ -46,38 +48,32 @@ def config_cls(cls) -> AWSOpenSearchConfig: return AWSOpenSearchConfig @classmethod - def case_config_cls( - cls, index_type: IndexType | None = None - ) -> AWSOpenSearchIndexConfig: + def case_config_cls(cls, index_type: IndexType | None = None) -> AWSOpenSearchIndexConfig: return AWSOpenSearchIndexConfig def _create_index(self, client: OpenSearch): settings = { "index": { "knn": True, - # "number_of_shards": 5, - # "refresh_interval": "600s", - } + }, } mappings = { "properties": { - **{ - categoryCol: {"type": "keyword"} - for categoryCol in self.category_col_names - }, + **{categoryCol: {"type": "keyword"} for categoryCol in self.category_col_names}, self.vector_col_name: { "type": "knn_vector", "dimension": self.dim, "method": self.case_config.index_param(), }, - } + }, } try: client.indices.create( - index=self.index_name, body=dict(settings=settings, mappings=mappings) + index=self.index_name, + body={"settings": settings, "mappings": mappings}, ) except Exception as e: - log.warning(f"Failed to create index: {self.index_name} error: {str(e)}") + log.warning(f"Failed to create index: {self.index_name} error: {e!s}") raise e from None @contextmanager @@ -86,7 +82,6 @@ def init(self) -> None: self.client = OpenSearch(**self.db_config) yield - # self.client.transport.close() self.client = None del self.client @@ -101,16 +96,20 @@ def insert_embeddings( insert_data = [] for i in range(len(embeddings)): - insert_data.append({"index": {"_index": self.index_name, self.id_col_name: metadata[i]}}) + insert_data.append( + {"index": {"_index": self.index_name, self.id_col_name: metadata[i]}}, + ) insert_data.append({self.vector_col_name: embeddings[i]}) try: resp = self.client.bulk(insert_data) log.info(f"AWS_OpenSearch adding documents: {len(resp['items'])}") resp = self.client.indices.stats(self.index_name) - log.info(f"Total document count in index: {resp['_all']['primaries']['indexing']['index_total']}") + log.info( + f"Total document count in index: {resp['_all']['primaries']['indexing']['index_total']}", + ) return (len(embeddings), None) except Exception as e: - log.warning(f"Failed to insert data: {self.index_name} error: {str(e)}") + log.warning(f"Failed to insert data: {self.index_name} error: {e!s}") time.sleep(10) return self.insert_embeddings(embeddings, metadata) @@ -135,20 +134,23 @@ def search_embedding( body = { "size": k, "query": {"knn": {self.vector_col_name: {"vector": query, "k": k}}}, - **({"filter": {"range": {self.id_col_name: {"gt": filters["id"]}}}} if filters else {}) + **({"filter": {"range": {self.id_col_name: {"gt": filters["id"]}}}} if filters else {}), } try: - resp = self.client.search(index=self.index_name, body=body,size=k,_source=False,docvalue_fields=[self.id_col_name],stored_fields="_none_",) + resp = self.client.search( + index=self.index_name, + body=body, + size=k, + _source=False, + docvalue_fields=[self.id_col_name], + stored_fields="_none_", + ) log.info(f'Search took: {resp["took"]}') log.info(f'Search shards: {resp["_shards"]}') log.info(f'Search hits total: {resp["hits"]["total"]}') - result = [int(h["fields"][self.id_col_name][0]) for h in resp["hits"]["hits"]] - #result = [int(d["_id"]) for d in resp["hits"]["hits"]] - # log.info(f'success! length={len(res)}') - - return result + return [int(h["fields"][self.id_col_name][0]) for h in resp["hits"]["hits"]] except Exception as e: - log.warning(f"Failed to search: {self.index_name} error: {str(e)}") + log.warning(f"Failed to search: {self.index_name} error: {e!s}") raise e from None def optimize(self): @@ -163,37 +165,35 @@ def optimize(self): def _refresh_index(self): log.debug(f"Starting refresh for index {self.index_name}") - SECONDS_WAITING_FOR_REFRESH_API_CALL_SEC = 30 while True: try: - log.info(f"Starting the Refresh Index..") + log.info("Starting the Refresh Index..") self.client.indices.refresh(index=self.index_name) break except Exception as e: log.info( - f"Refresh errored out. Sleeping for {SECONDS_WAITING_FOR_REFRESH_API_CALL_SEC} sec and then Retrying : {e}") - time.sleep(SECONDS_WAITING_FOR_REFRESH_API_CALL_SEC) + f"Refresh errored out. Sleeping for {WAITING_FOR_REFRESH_SEC} sec and then Retrying : {e}", + ) + time.sleep(WAITING_FOR_REFRESH_SEC) continue log.debug(f"Completed refresh for index {self.index_name}") def _do_force_merge(self): log.debug(f"Starting force merge for index {self.index_name}") - force_merge_endpoint = f'/{self.index_name}/_forcemerge?max_num_segments=1&wait_for_completion=false' - force_merge_task_id = self.client.transport.perform_request('POST', force_merge_endpoint)['task'] - SECONDS_WAITING_FOR_FORCE_MERGE_API_CALL_SEC = 30 + force_merge_endpoint = f"/{self.index_name}/_forcemerge?max_num_segments=1&wait_for_completion=false" + force_merge_task_id = self.client.transport.perform_request("POST", force_merge_endpoint)["task"] while True: - time.sleep(SECONDS_WAITING_FOR_FORCE_MERGE_API_CALL_SEC) + time.sleep(WAITING_FOR_FORCE_MERGE_SEC) task_status = self.client.tasks.get(task_id=force_merge_task_id) - if task_status['completed']: + if task_status["completed"]: break log.debug(f"Completed force merge for index {self.index_name}") def _load_graphs_to_memory(self): if self.case_config.engine != AWSOS_Engine.lucene: log.info("Calling warmup API to load graphs into memory") - warmup_endpoint = f'/_plugins/_knn/warmup/{self.index_name}' - self.client.transport.perform_request('GET', warmup_endpoint) + warmup_endpoint = f"/_plugins/_knn/warmup/{self.index_name}" + self.client.transport.perform_request("GET", warmup_endpoint) def ready_to_load(self): """ready_to_load will be called before load in load cases.""" - pass diff --git a/vectordb_bench/backend/clients/aws_opensearch/cli.py b/vectordb_bench/backend/clients/aws_opensearch/cli.py index 5cb4ebbe1..bb0c2450d 100644 --- a/vectordb_bench/backend/clients/aws_opensearch/cli.py +++ b/vectordb_bench/backend/clients/aws_opensearch/cli.py @@ -14,22 +14,20 @@ class AWSOpenSearchTypedDict(TypedDict): - host: Annotated[ - str, click.option("--host", type=str, help="Db host", required=True) - ] + host: Annotated[str, click.option("--host", type=str, help="Db host", required=True)] port: Annotated[int, click.option("--port", type=int, default=443, help="Db Port")] user: Annotated[str, click.option("--user", type=str, default="admin", help="Db User")] password: Annotated[str, click.option("--password", type=str, help="Db password")] -class AWSOpenSearchHNSWTypedDict(CommonTypedDict, AWSOpenSearchTypedDict, HNSWFlavor2): - ... +class AWSOpenSearchHNSWTypedDict(CommonTypedDict, AWSOpenSearchTypedDict, HNSWFlavor2): ... @cli.command() @click_parameter_decorators_from_typed_dict(AWSOpenSearchHNSWTypedDict) def AWSOpenSearch(**parameters: Unpack[AWSOpenSearchHNSWTypedDict]): from .config import AWSOpenSearchConfig, AWSOpenSearchIndexConfig + run( db=DB.AWSOpenSearch, db_config=AWSOpenSearchConfig( @@ -38,7 +36,6 @@ def AWSOpenSearch(**parameters: Unpack[AWSOpenSearchHNSWTypedDict]): user=parameters["user"], password=SecretStr(parameters["password"]), ), - db_case_config=AWSOpenSearchIndexConfig( - ), + db_case_config=AWSOpenSearchIndexConfig(), **parameters, ) diff --git a/vectordb_bench/backend/clients/aws_opensearch/config.py b/vectordb_bench/backend/clients/aws_opensearch/config.py index 15cd4ead8..e9ccc7277 100644 --- a/vectordb_bench/backend/clients/aws_opensearch/config.py +++ b/vectordb_bench/backend/clients/aws_opensearch/config.py @@ -1,10 +1,13 @@ import logging from enum import Enum -from pydantic import SecretStr, BaseModel -from ..api import DBConfig, DBCaseConfig, MetricType, IndexType +from pydantic import BaseModel, SecretStr + +from ..api import DBCaseConfig, DBConfig, MetricType log = logging.getLogger(__name__) + + class AWSOpenSearchConfig(DBConfig, BaseModel): host: str = "" port: int = 443 @@ -13,7 +16,7 @@ class AWSOpenSearchConfig(DBConfig, BaseModel): def to_dict(self) -> dict: return { - "hosts": [{'host': self.host, 'port': self.port}], + "hosts": [{"host": self.host, "port": self.port}], "http_auth": (self.user, self.password.get_secret_value()), "use_ssl": True, "http_compress": True, @@ -40,25 +43,26 @@ class AWSOpenSearchIndexConfig(BaseModel, DBCaseConfig): def parse_metric(self) -> str: if self.metric_type == MetricType.IP: return "innerproduct" - elif self.metric_type == MetricType.COSINE: + if self.metric_type == MetricType.COSINE: if self.engine == AWSOS_Engine.faiss: - log.info(f"Using metric type as innerproduct because faiss doesn't support cosine as metric type for Opensearch") + log.info( + "Using innerproduct because faiss doesn't support cosine as metric type for Opensearch", + ) return "innerproduct" return "cosinesimil" return "l2" def index_param(self) -> dict: - params = { + return { "name": "hnsw", "space_type": self.parse_metric(), "engine": self.engine.value, "parameters": { "ef_construction": self.efConstruction, "m": self.M, - "ef_search": self.efSearch - } + "ef_search": self.efSearch, + }, } - return params def search_param(self) -> dict: return {} diff --git a/vectordb_bench/backend/clients/aws_opensearch/run.py b/vectordb_bench/backend/clients/aws_opensearch/run.py index d2698d139..68aa200a5 100644 --- a/vectordb_bench/backend/clients/aws_opensearch/run.py +++ b/vectordb_bench/backend/clients/aws_opensearch/run.py @@ -1,12 +1,16 @@ -import time, random +import logging +import random +import time + from opensearchpy import OpenSearch -from opensearch_dsl import Search, Document, Text, Keyword -_HOST = 'xxxxxx.us-west-2.es.amazonaws.com' +log = logging.getLogger(__name__) + +_HOST = "xxxxxx.us-west-2.es.amazonaws.com" _PORT = 443 -_AUTH = ('admin', 'xxxxxx') # For testing only. Don't store credentials in code. +_AUTH = ("admin", "xxxxxx") # For testing only. Don't store credentials in code. -_INDEX_NAME = 'my-dsl-index' +_INDEX_NAME = "my-dsl-index" _BATCH = 100 _ROWS = 100 _DIM = 128 @@ -14,25 +18,24 @@ def create_client(): - client = OpenSearch( - hosts=[{'host': _HOST, 'port': _PORT}], - http_compress=True, # enables gzip compression for request bodies + return OpenSearch( + hosts=[{"host": _HOST, "port": _PORT}], + http_compress=True, # enables gzip compression for request bodies http_auth=_AUTH, use_ssl=True, verify_certs=True, ssl_assert_hostname=False, ssl_show_warn=False, ) - return client -def create_index(client, index_name): +def create_index(client: OpenSearch, index_name: str): settings = { "index": { "knn": True, "number_of_shards": 1, "refresh_interval": "5s", - } + }, } mappings = { "properties": { @@ -46,41 +49,46 @@ def create_index(client, index_name): "parameters": { "ef_construction": 256, "m": 16, - } - } - } - } + }, + }, + }, + }, } - response = client.indices.create(index=index_name, body=dict(settings=settings, mappings=mappings)) - print('\nCreating index:') - print(response) + response = client.indices.create( + index=index_name, + body={"settings": settings, "mappings": mappings}, + ) + log.info("\nCreating index:") + log.info(response) -def delete_index(client, index_name): +def delete_index(client: OpenSearch, index_name: str): response = client.indices.delete(index=index_name) - print('\nDeleting index:') - print(response) + log.info("\nDeleting index:") + log.info(response) -def bulk_insert(client, index_name): +def bulk_insert(client: OpenSearch, index_name: str): # Perform bulk operations - ids = [i for i in range(_ROWS)] + ids = list(range(_ROWS)) vec = [[random.random() for _ in range(_DIM)] for _ in range(_ROWS)] docs = [] for i in range(0, _ROWS, _BATCH): docs.clear() - for j in range(0, _BATCH): - docs.append({"index": {"_index": index_name, "_id": ids[i+j]}}) - docs.append({"embedding": vec[i+j]}) + for j in range(_BATCH): + docs.append({"index": {"_index": index_name, "_id": ids[i + j]}}) + docs.append({"embedding": vec[i + j]}) response = client.bulk(docs) - print('\nAdding documents:', len(response['items']), response['errors']) + log.info(f"Adding documents: {len(response['items'])}, {response['errors']}") response = client.indices.stats(index_name) - print('\nTotal document count in index:', response['_all']['primaries']['indexing']['index_total']) + log.info( + f'Total document count in index: { response["_all"]["primaries"]["indexing"]["index_total"] }', + ) -def search(client, index_name): +def search(client: OpenSearch, index_name: str): # Search for the document. search_body = { "size": _TOPK, @@ -89,53 +97,55 @@ def search(client, index_name): "embedding": { "vector": [random.random() for _ in range(_DIM)], "k": _TOPK, - } - } - } + }, + }, + }, } while True: response = client.search(index=index_name, body=search_body) - print(f'\nSearch took: {response["took"]}') - print(f'\nSearch shards: {response["_shards"]}') - print(f'\nSearch hits total: {response["hits"]["total"]}') + log.info(f'\nSearch took: {response["took"]}') + log.info(f'\nSearch shards: {response["_shards"]}') + log.info(f'\nSearch hits total: {response["hits"]["total"]}') result = response["hits"]["hits"] if len(result) != 0: - print('\nSearch results:') + log.info("\nSearch results:") for hit in response["hits"]["hits"]: - print(hit["_id"], hit["_score"]) + log.info(hit["_id"], hit["_score"]) break - else: - print('\nSearch not ready, sleep 1s') - time.sleep(1) - -def optimize_index(client, index_name): - print(f"Starting force merge for index {index_name}") - force_merge_endpoint = f'/{index_name}/_forcemerge?max_num_segments=1&wait_for_completion=false' - force_merge_task_id = client.transport.perform_request('POST', force_merge_endpoint)['task'] - SECONDS_WAITING_FOR_FORCE_MERGE_API_CALL_SEC = 30 + log.info("\nSearch not ready, sleep 1s") + time.sleep(1) + + +SECONDS_WAITING_FOR_FORCE_MERGE_API_CALL_SEC = 30 +WAITINT_FOR_REFRESH_SEC = 30 + + +def optimize_index(client: OpenSearch, index_name: str): + log.info(f"Starting force merge for index {index_name}") + force_merge_endpoint = f"/{index_name}/_forcemerge?max_num_segments=1&wait_for_completion=false" + force_merge_task_id = client.transport.perform_request("POST", force_merge_endpoint)["task"] while True: time.sleep(SECONDS_WAITING_FOR_FORCE_MERGE_API_CALL_SEC) task_status = client.tasks.get(task_id=force_merge_task_id) - if task_status['completed']: + if task_status["completed"]: break - print(f"Completed force merge for index {index_name}") + log.info(f"Completed force merge for index {index_name}") -def refresh_index(client, index_name): - print(f"Starting refresh for index {index_name}") - SECONDS_WAITING_FOR_REFRESH_API_CALL_SEC = 30 +def refresh_index(client: OpenSearch, index_name: str): + log.info(f"Starting refresh for index {index_name}") while True: try: - print(f"Starting the Refresh Index..") + log.info("Starting the Refresh Index..") client.indices.refresh(index=index_name) break except Exception as e: - print( - f"Refresh errored out. Sleeping for {SECONDS_WAITING_FOR_REFRESH_API_CALL_SEC} sec and then Retrying : {e}") - time.sleep(SECONDS_WAITING_FOR_REFRESH_API_CALL_SEC) + log.info( + f"Refresh errored out. Sleeping for {WAITINT_FOR_REFRESH_SEC} sec and then Retrying : {e}", + ) + time.sleep(WAITINT_FOR_REFRESH_SEC) continue - print(f"Completed refresh for index {index_name}") - + log.info(f"Completed refresh for index {index_name}") def main(): @@ -148,9 +158,9 @@ def main(): search(client, _INDEX_NAME) delete_index(client, _INDEX_NAME) except Exception as e: - print(e) + log.info(e) delete_index(client, _INDEX_NAME) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/vectordb_bench/backend/clients/chroma/chroma.py b/vectordb_bench/backend/clients/chroma/chroma.py index 235cb595b..a148fa141 100644 --- a/vectordb_bench/backend/clients/chroma/chroma.py +++ b/vectordb_bench/backend/clients/chroma/chroma.py @@ -1,55 +1,55 @@ -import chromadb -import logging +import logging from contextlib import contextmanager from typing import Any -from ..api import VectorDB, DBCaseConfig + +import chromadb + +from ..api import DBCaseConfig, VectorDB log = logging.getLogger(__name__) + + class ChromaClient(VectorDB): - """Chroma client for VectorDB. + """Chroma client for VectorDB. To set up Chroma in docker, see https://docs.trychroma.com/usage-guide or the instructions in tests/test_chroma.py To change to running in process, modify the HttpClient() in __init__() and init(). - """ + """ def __init__( - self, - dim: int, - db_config: dict, - db_case_config: DBCaseConfig, - drop_old: bool = False, - - **kwargs - ): - + self, + dim: int, + db_config: dict, + db_case_config: DBCaseConfig, + drop_old: bool = False, + **kwargs, + ): self.db_config = db_config self.case_config = db_case_config - self.collection_name = 'example2' + self.collection_name = "example2" - client = chromadb.HttpClient(host=self.db_config["host"], - port=self.db_config["port"]) + client = chromadb.HttpClient(host=self.db_config["host"], port=self.db_config["port"]) assert client.heartbeat() is not None if drop_old: try: - client.reset() # Reset the database - except: + client.reset() # Reset the database + except Exception: drop_old = False log.info(f"Chroma client drop_old collection: {self.collection_name}") @contextmanager def init(self) -> None: - """ create and destory connections to database. + """create and destory connections to database. Examples: >>> with self.init(): >>> self.insert_embeddings() """ - #create connection - self.client = chromadb.HttpClient(host=self.db_config["host"], - port=self.db_config["port"]) - - self.collection = self.client.get_or_create_collection('example2') + # create connection + self.client = chromadb.HttpClient(host=self.db_config["host"], port=self.db_config["port"]) + + self.collection = self.client.get_or_create_collection("example2") yield self.client = None self.collection = None @@ -79,12 +79,12 @@ def insert_embeddings( Returns: (int, Exception): number of embeddings inserted and exception if any """ - ids=[str(i) for i in metadata] - metadata = [{"id": int(i)} for i in metadata] + ids = [str(i) for i in metadata] + metadata = [{"id": int(i)} for i in metadata] if len(embeddings) > 0: self.collection.add(embeddings=embeddings, ids=ids, metadatas=metadata) return len(embeddings), None - + def search_embedding( self, query: list[float], @@ -100,17 +100,19 @@ def search_embedding( kwargs: other arguments Returns: - Dict {ids: list[list[int]], - embedding: list[list[float]] + Dict {ids: list[list[int]], + embedding: list[list[float]] distance: list[list[float]]} """ if filters: # assumes benchmark test filters of format: {'metadata': '>=10000', 'id': 10000} id_value = filters.get("id") - results = self.collection.query(query_embeddings=query, n_results=k, - where={"id": {"$gt": id_value}}) - #return list of id's in results - return [int(i) for i in results.get('ids')[0]] + results = self.collection.query( + query_embeddings=query, + n_results=k, + where={"id": {"$gt": id_value}}, + ) + # return list of id's in results + return [int(i) for i in results.get("ids")[0]] results = self.collection.query(query_embeddings=query, n_results=k) - return [int(i) for i in results.get('ids')[0]] - + return [int(i) for i in results.get("ids")[0]] diff --git a/vectordb_bench/backend/clients/chroma/config.py b/vectordb_bench/backend/clients/chroma/config.py index 85c59c973..af34cf513 100644 --- a/vectordb_bench/backend/clients/chroma/config.py +++ b/vectordb_bench/backend/clients/chroma/config.py @@ -1,14 +1,16 @@ from pydantic import SecretStr + from ..api import DBConfig + class ChromaConfig(DBConfig): password: SecretStr host: SecretStr - port: int + port: int def to_dict(self) -> dict: return { "host": self.host.get_secret_value(), "port": self.port, "password": self.password.get_secret_value(), - } \ No newline at end of file + } diff --git a/vectordb_bench/backend/clients/elastic_cloud/config.py b/vectordb_bench/backend/clients/elastic_cloud/config.py index 204a4fc9d..d35ee68ec 100644 --- a/vectordb_bench/backend/clients/elastic_cloud/config.py +++ b/vectordb_bench/backend/clients/elastic_cloud/config.py @@ -1,7 +1,8 @@ from enum import Enum -from pydantic import SecretStr, BaseModel -from ..api import DBConfig, DBCaseConfig, MetricType, IndexType +from pydantic import BaseModel, SecretStr + +from ..api import DBCaseConfig, DBConfig, IndexType, MetricType class ElasticCloudConfig(DBConfig, BaseModel): @@ -32,12 +33,12 @@ class ElasticCloudIndexConfig(BaseModel, DBCaseConfig): def parse_metric(self) -> str: if self.metric_type == MetricType.L2: return "l2_norm" - elif self.metric_type == MetricType.IP: + if self.metric_type == MetricType.IP: return "dot_product" return "cosine" def index_param(self) -> dict: - params = { + return { "type": "dense_vector", "index": True, "element_type": self.element_type.value, @@ -48,7 +49,6 @@ def index_param(self) -> dict: "ef_construction": self.efConstruction, }, } - return params def search_param(self) -> dict: return { diff --git a/vectordb_bench/backend/clients/elastic_cloud/elastic_cloud.py b/vectordb_bench/backend/clients/elastic_cloud/elastic_cloud.py index 64f27e490..a3183bcb7 100644 --- a/vectordb_bench/backend/clients/elastic_cloud/elastic_cloud.py +++ b/vectordb_bench/backend/clients/elastic_cloud/elastic_cloud.py @@ -1,17 +1,22 @@ import logging import time +from collections.abc import Iterable from contextlib import contextmanager -from typing import Iterable -from ..api import VectorDB -from .config import ElasticCloudIndexConfig + from elasticsearch.helpers import bulk +from ..api import VectorDB +from .config import ElasticCloudIndexConfig for logger in ("elasticsearch", "elastic_transport"): logging.getLogger(logger).setLevel(logging.WARNING) log = logging.getLogger(__name__) + +SECONDS_WAITING_FOR_FORCE_MERGE_API_CALL_SEC = 30 + + class ElasticCloud(VectorDB): def __init__( self, @@ -46,14 +51,14 @@ def __init__( def init(self) -> None: """connect to elasticsearch""" from elasticsearch import Elasticsearch + self.client = Elasticsearch(**self.db_config, request_timeout=180) yield - # self.client.transport.close() self.client = None - del(self.client) + del self.client - def _create_indice(self, client) -> None: + def _create_indice(self, client: any) -> None: mappings = { "_source": {"excludes": [self.vector_col_name]}, "properties": { @@ -62,13 +67,13 @@ def _create_indice(self, client) -> None: "dims": self.dim, **self.case_config.index_param(), }, - } + }, } try: client.indices.create(index=self.indice, mappings=mappings) except Exception as e: - log.warning(f"Failed to create indice: {self.indice} error: {str(e)}") + log.warning(f"Failed to create indice: {self.indice} error: {e!s}") raise e from None def insert_embeddings( @@ -94,7 +99,7 @@ def insert_embeddings( bulk_insert_res = bulk(self.client, insert_data) return (bulk_insert_res[0], None) except Exception as e: - log.warning(f"Failed to insert data: {self.indice} error: {str(e)}") + log.warning(f"Failed to insert data: {self.indice} error: {e!s}") return (0, e) def search_embedding( @@ -114,16 +119,12 @@ def search_embedding( list[tuple[int, float]]: list of k most similar embeddings in (id, score) tuple to the query embedding. """ assert self.client is not None, "should self.init() first" - # is_existed_res = self.client.indices.exists(index=self.indice) - # assert is_existed_res.raw == True, "should self.init() first" knn = { "field": self.vector_col_name, "k": k, "num_candidates": self.case_config.num_candidates, - "filter": [{"range": {self.id_col_name: {"gt": filters["id"]}}}] - if filters - else [], + "filter": [{"range": {self.id_col_name: {"gt": filters["id"]}}}] if filters else [], "query_vector": query, } size = k @@ -137,26 +138,26 @@ def search_embedding( stored_fields="_none_", filter_path=[f"hits.hits.fields.{self.id_col_name}"], ) - res = [h["fields"][self.id_col_name][0] for h in res["hits"]["hits"]] - - return res + return [h["fields"][self.id_col_name][0] for h in res["hits"]["hits"]] except Exception as e: - log.warning(f"Failed to search: {self.indice} error: {str(e)}") + log.warning(f"Failed to search: {self.indice} error: {e!s}") raise e from None def optimize(self): """optimize will be called between insertion and search in performance cases.""" assert self.client is not None, "should self.init() first" self.client.indices.refresh(index=self.indice) - force_merge_task_id = self.client.indices.forcemerge(index=self.indice, max_num_segments=1, wait_for_completion=False)['task'] + force_merge_task_id = self.client.indices.forcemerge( + index=self.indice, + max_num_segments=1, + wait_for_completion=False, + )["task"] log.info(f"Elasticsearch force merge task id: {force_merge_task_id}") - SECONDS_WAITING_FOR_FORCE_MERGE_API_CALL_SEC = 30 while True: time.sleep(SECONDS_WAITING_FOR_FORCE_MERGE_API_CALL_SEC) task_status = self.client.tasks.get(task_id=force_merge_task_id) - if task_status['completed']: + if task_status["completed"]: return def ready_to_load(self): """ready_to_load will be called before load in load cases.""" - pass diff --git a/vectordb_bench/backend/clients/memorydb/cli.py b/vectordb_bench/backend/clients/memorydb/cli.py index 50b5f89ba..ae00bfd17 100644 --- a/vectordb_bench/backend/clients/memorydb/cli.py +++ b/vectordb_bench/backend/clients/memorydb/cli.py @@ -14,9 +14,7 @@ class MemoryDBTypedDict(TypedDict): - host: Annotated[ - str, click.option("--host", type=str, help="Db host", required=True) - ] + host: Annotated[str, click.option("--host", type=str, help="Db host", required=True)] password: Annotated[str, click.option("--password", type=str, help="Db password")] port: Annotated[int, click.option("--port", type=int, default=6379, help="Db Port")] ssl: Annotated[ @@ -44,7 +42,10 @@ class MemoryDBTypedDict(TypedDict): is_flag=True, show_default=True, default=False, - help="Cluster Mode Disabled (CMD), use this flag when testing locally on a single node instance. In production, MemoryDB only supports cluster mode (CME)", + help=( + "Cluster Mode Disabled (CMD), use this flag when testing locally on a single node instance.", + " In production, MemoryDB only supports cluster mode (CME)", + ), ), ] insert_batch_size: Annotated[ @@ -58,8 +59,7 @@ class MemoryDBTypedDict(TypedDict): ] -class MemoryDBHNSWTypedDict(CommonTypedDict, MemoryDBTypedDict, HNSWFlavor2): - ... +class MemoryDBHNSWTypedDict(CommonTypedDict, MemoryDBTypedDict, HNSWFlavor2): ... @cli.command() @@ -82,7 +82,7 @@ def MemoryDB(**parameters: Unpack[MemoryDBHNSWTypedDict]): M=parameters["m"], ef_construction=parameters["ef_construction"], ef_runtime=parameters["ef_runtime"], - insert_batch_size=parameters["insert_batch_size"] + insert_batch_size=parameters["insert_batch_size"], ), **parameters, - ) \ No newline at end of file + ) diff --git a/vectordb_bench/backend/clients/memorydb/config.py b/vectordb_bench/backend/clients/memorydb/config.py index 1284d3449..2c40ff546 100644 --- a/vectordb_bench/backend/clients/memorydb/config.py +++ b/vectordb_bench/backend/clients/memorydb/config.py @@ -29,7 +29,7 @@ class MemoryDBIndexConfig(BaseModel, DBCaseConfig): def parse_metric(self) -> str: if self.metric_type == MetricType.L2: return "l2" - elif self.metric_type == MetricType.IP: + if self.metric_type == MetricType.IP: return "ip" return "cosine" @@ -51,4 +51,4 @@ def index_param(self) -> dict: def search_param(self) -> dict: return { "ef_runtime": self.ef_runtime, - } \ No newline at end of file + } diff --git a/vectordb_bench/backend/clients/memorydb/memorydb.py b/vectordb_bench/backend/clients/memorydb/memorydb.py index c5f80eb2a..d05e30be1 100644 --- a/vectordb_bench/backend/clients/memorydb/memorydb.py +++ b/vectordb_bench/backend/clients/memorydb/memorydb.py @@ -1,30 +1,33 @@ -import logging, time +import logging +import time +from collections.abc import Generator from contextlib import contextmanager -from typing import Any, Generator, Optional, Tuple, Type -from ..api import VectorDB, DBCaseConfig, IndexType -from .config import MemoryDBIndexConfig +from typing import Any + +import numpy as np import redis from redis import Redis from redis.cluster import RedisCluster -from redis.commands.search.field import TagField, VectorField, NumericField -from redis.commands.search.indexDefinition import IndexDefinition, IndexType +from redis.commands.search.field import NumericField, TagField, VectorField +from redis.commands.search.indexDefinition import IndexDefinition from redis.commands.search.query import Query -import numpy as np +from ..api import IndexType, VectorDB +from .config import MemoryDBIndexConfig log = logging.getLogger(__name__) -INDEX_NAME = "index" # Vector Index Name +INDEX_NAME = "index" # Vector Index Name + class MemoryDB(VectorDB): def __init__( - self, - dim: int, - db_config: dict, - db_case_config: MemoryDBIndexConfig, - drop_old: bool = False, - **kwargs - ): - + self, + dim: int, + db_config: dict, + db_case_config: MemoryDBIndexConfig, + drop_old: bool = False, + **kwargs, + ): self.db_config = db_config self.case_config = db_case_config self.collection_name = INDEX_NAME @@ -44,10 +47,10 @@ def __init__( info = conn.ft(INDEX_NAME).info() log.info(f"Index info: {info}") except redis.exceptions.ResponseError as e: - log.error(e) + log.warning(e) drop_old = False log.info(f"MemoryDB client drop_old collection: {self.collection_name}") - + log.info("Executing FLUSHALL") conn.flushall() @@ -59,7 +62,7 @@ def __init__( self.wait_until(self.wait_for_empty_db, 3, "", rc) log.debug(f"Flushall done in the host: {host}") rc.close() - + self.make_index(dim, conn) conn.close() conn = None @@ -69,7 +72,7 @@ def make_index(self, vector_dimensions: int, conn: redis.Redis): # check to see if index exists conn.ft(INDEX_NAME).info() except Exception as e: - log.warn(f"Error getting info for index '{INDEX_NAME}': {e}") + log.warning(f"Error getting info for index '{INDEX_NAME}': {e}") index_param = self.case_config.index_param() search_param = self.case_config.search_param() vector_parameters = { # Vector Index Type: FLAT or HNSW @@ -85,17 +88,19 @@ def make_index(self, vector_dimensions: int, conn: redis.Redis): vector_parameters["EF_RUNTIME"] = search_param["ef_runtime"] schema = ( - TagField("id"), - NumericField("metadata"), - VectorField("vector", # Vector Field Name - "HNSW", vector_parameters + TagField("id"), + NumericField("metadata"), + VectorField( + "vector", # Vector Field Name + "HNSW", + vector_parameters, ), ) definition = IndexDefinition(index_type=IndexType.HASH) rs = conn.ft(INDEX_NAME) rs.create_index(schema, definition=definition) - + def get_client(self, **kwargs): """ Gets either cluster connection or normal connection based on `cmd` flag. @@ -143,7 +148,7 @@ def get_client(self, **kwargs): @contextmanager def init(self) -> Generator[None, None, None]: - """ create and destory connections to database. + """create and destory connections to database. Examples: >>> with self.init(): @@ -170,7 +175,7 @@ def insert_embeddings( embeddings: list[list[float]], metadata: list[int], **kwargs: Any, - ) -> Tuple[int, Optional[Exception]]: + ) -> tuple[int, Exception | None]: """Insert embeddings into the database. Should call self.init() first. """ @@ -178,12 +183,15 @@ def insert_embeddings( try: with self.conn.pipeline(transaction=False) as pipe: for i, embedding in enumerate(embeddings): - embedding = np.array(embedding).astype(np.float32) - pipe.hset(metadata[i], mapping = { - "id": str(metadata[i]), - "metadata": metadata[i], - "vector": embedding.tobytes(), - }) + ndarr_emb = np.array(embedding).astype(np.float32) + pipe.hset( + metadata[i], + mapping={ + "id": str(metadata[i]), + "metadata": metadata[i], + "vector": ndarr_emb.tobytes(), + }, + ) # Execute the pipe so we don't keep too much in memory at once if (i + 1) % self.insert_batch_size == 0: pipe.execute() @@ -192,9 +200,9 @@ def insert_embeddings( result_len = i + 1 except Exception as e: return 0, e - + return result_len, None - + def _post_insert(self): """Wait for indexing to finish""" client = self.get_client(primary=True) @@ -208,21 +216,17 @@ def _post_insert(self): self.wait_until(*args) log.debug(f"Background indexing completed in the host: {host_name}") rc.close() - - def wait_until( - self, condition, interval=5, message="Operation took too long", *args - ): + + def wait_until(self, condition: any, interval: int = 5, message: str = "Operation took too long", *args): while not condition(*args): time.sleep(interval) - + def wait_for_no_activity(self, client: redis.RedisCluster | redis.Redis): - return ( - client.info("search")["search_background_indexing_status"] == "NO_ACTIVITY" - ) - + return client.info("search")["search_background_indexing_status"] == "NO_ACTIVITY" + def wait_for_empty_db(self, client: redis.RedisCluster | redis.Redis): return client.execute_command("DBSIZE") == 0 - + def search_embedding( self, query: list[float], @@ -230,13 +234,13 @@ def search_embedding( filters: dict | None = None, timeout: int | None = None, **kwargs: Any, - ) -> (list[int]): + ) -> list[int]: assert self.conn is not None - + query_vector = np.array(query).astype(np.float32).tobytes() query_obj = Query(f"*=>[KNN {k} @vector $vec]").return_fields("id").paging(0, k) query_params = {"vec": query_vector} - + if filters: # benchmark test filters of format: {'metadata': '>=10000', 'id': 10000} # gets exact match for id, and range for metadata if they exist in filters @@ -244,11 +248,19 @@ def search_embedding( # Removing '>=' from the id_value: '>=10000' metadata_value = filters.get("metadata")[2:] if id_value and metadata_value: - query_obj = Query(f"(@metadata:[{metadata_value} +inf] @id:{ {id_value} })=>[KNN {k} @vector $vec]").return_fields("id").paging(0, k) + query_obj = ( + Query( + f"(@metadata:[{metadata_value} +inf] @id:{ {id_value} })=>[KNN {k} @vector $vec]", + ) + .return_fields("id") + .paging(0, k) + ) elif id_value: - #gets exact match for id + # gets exact match for id query_obj = Query(f"@id:{ {id_value} }=>[KNN {k} @vector $vec]").return_fields("id").paging(0, k) - else: #metadata only case, greater than or equal to metadata value - query_obj = Query(f"@metadata:[{metadata_value} +inf]=>[KNN {k} @vector $vec]").return_fields("id").paging(0, k) + else: # metadata only case, greater than or equal to metadata value + query_obj = ( + Query(f"@metadata:[{metadata_value} +inf]=>[KNN {k} @vector $vec]").return_fields("id").paging(0, k) + ) res = self.conn.ft(INDEX_NAME).search(query_obj, query_params) - return [int(doc["id"]) for doc in res.docs] \ No newline at end of file + return [int(doc["id"]) for doc in res.docs] diff --git a/vectordb_bench/backend/clients/milvus/cli.py b/vectordb_bench/backend/clients/milvus/cli.py index 885995def..51ea82eff 100644 --- a/vectordb_bench/backend/clients/milvus/cli.py +++ b/vectordb_bench/backend/clients/milvus/cli.py @@ -1,8 +1,9 @@ -from typing import Annotated, TypedDict, Unpack, Optional +from typing import Annotated, TypedDict, Unpack import click from pydantic import SecretStr +from vectordb_bench.backend.clients import DB from vectordb_bench.cli.cli import ( CommonTypedDict, HNSWFlavor3, @@ -10,33 +11,33 @@ cli, click_parameter_decorators_from_typed_dict, run, - ) -from vectordb_bench.backend.clients import DB DBTYPE = DB.Milvus class MilvusTypedDict(TypedDict): uri: Annotated[ - str, click.option("--uri", type=str, help="uri connection string", required=True) + str, + click.option("--uri", type=str, help="uri connection string", required=True), ] user_name: Annotated[ - Optional[str], click.option("--user-name", type=str, help="Db username", required=False) + str | None, + click.option("--user-name", type=str, help="Db username", required=False), ] password: Annotated[ - Optional[str], click.option("--password", type=str, help="Db password", required=False) + str | None, + click.option("--password", type=str, help="Db password", required=False), ] -class MilvusAutoIndexTypedDict(CommonTypedDict, MilvusTypedDict): - ... +class MilvusAutoIndexTypedDict(CommonTypedDict, MilvusTypedDict): ... @cli.command() @click_parameter_decorators_from_typed_dict(MilvusAutoIndexTypedDict) def MilvusAutoIndex(**parameters: Unpack[MilvusAutoIndexTypedDict]): - from .config import MilvusConfig, AutoIndexConfig + from .config import AutoIndexConfig, MilvusConfig run( db=DBTYPE, @@ -54,7 +55,7 @@ def MilvusAutoIndex(**parameters: Unpack[MilvusAutoIndexTypedDict]): @cli.command() @click_parameter_decorators_from_typed_dict(MilvusAutoIndexTypedDict) def MilvusFlat(**parameters: Unpack[MilvusAutoIndexTypedDict]): - from .config import MilvusConfig, FLATConfig + from .config import FLATConfig, MilvusConfig run( db=DBTYPE, @@ -69,14 +70,13 @@ def MilvusFlat(**parameters: Unpack[MilvusAutoIndexTypedDict]): ) -class MilvusHNSWTypedDict(CommonTypedDict, MilvusTypedDict, HNSWFlavor3): - ... +class MilvusHNSWTypedDict(CommonTypedDict, MilvusTypedDict, HNSWFlavor3): ... @cli.command() @click_parameter_decorators_from_typed_dict(MilvusHNSWTypedDict) def MilvusHNSW(**parameters: Unpack[MilvusHNSWTypedDict]): - from .config import MilvusConfig, HNSWConfig + from .config import HNSWConfig, MilvusConfig run( db=DBTYPE, @@ -95,14 +95,13 @@ def MilvusHNSW(**parameters: Unpack[MilvusHNSWTypedDict]): ) -class MilvusIVFFlatTypedDict(CommonTypedDict, MilvusTypedDict, IVFFlatTypedDictN): - ... +class MilvusIVFFlatTypedDict(CommonTypedDict, MilvusTypedDict, IVFFlatTypedDictN): ... @cli.command() @click_parameter_decorators_from_typed_dict(MilvusIVFFlatTypedDict) def MilvusIVFFlat(**parameters: Unpack[MilvusIVFFlatTypedDict]): - from .config import MilvusConfig, IVFFlatConfig + from .config import IVFFlatConfig, MilvusConfig run( db=DBTYPE, @@ -123,7 +122,7 @@ def MilvusIVFFlat(**parameters: Unpack[MilvusIVFFlatTypedDict]): @cli.command() @click_parameter_decorators_from_typed_dict(MilvusIVFFlatTypedDict) def MilvusIVFSQ8(**parameters: Unpack[MilvusIVFFlatTypedDict]): - from .config import MilvusConfig, IVFSQ8Config + from .config import IVFSQ8Config, MilvusConfig run( db=DBTYPE, @@ -142,17 +141,13 @@ def MilvusIVFSQ8(**parameters: Unpack[MilvusIVFFlatTypedDict]): class MilvusDISKANNTypedDict(CommonTypedDict, MilvusTypedDict): - search_list: Annotated[ - str, click.option("--search-list", - type=int, - required=True) - ] + search_list: Annotated[str, click.option("--search-list", type=int, required=True)] @cli.command() @click_parameter_decorators_from_typed_dict(MilvusDISKANNTypedDict) def MilvusDISKANN(**parameters: Unpack[MilvusDISKANNTypedDict]): - from .config import MilvusConfig, DISKANNConfig + from .config import DISKANNConfig, MilvusConfig run( db=DBTYPE, @@ -171,21 +166,16 @@ def MilvusDISKANN(**parameters: Unpack[MilvusDISKANNTypedDict]): class MilvusGPUIVFTypedDict(CommonTypedDict, MilvusTypedDict, MilvusIVFFlatTypedDict): cache_dataset_on_device: Annotated[ - str, click.option("--cache-dataset-on-device", - type=str, - required=True) - ] - refine_ratio: Annotated[ - str, click.option("--refine-ratio", - type=float, - required=True) + str, + click.option("--cache-dataset-on-device", type=str, required=True), ] + refine_ratio: Annotated[str, click.option("--refine-ratio", type=float, required=True)] @cli.command() @click_parameter_decorators_from_typed_dict(MilvusGPUIVFTypedDict) def MilvusGPUIVFFlat(**parameters: Unpack[MilvusGPUIVFTypedDict]): - from .config import MilvusConfig, GPUIVFFlatConfig + from .config import GPUIVFFlatConfig, MilvusConfig run( db=DBTYPE, @@ -205,23 +195,20 @@ def MilvusGPUIVFFlat(**parameters: Unpack[MilvusGPUIVFTypedDict]): ) -class MilvusGPUIVFPQTypedDict(CommonTypedDict, MilvusTypedDict, MilvusIVFFlatTypedDict, MilvusGPUIVFTypedDict): - m: Annotated[ - str, click.option("--m", - type=int, help="hnsw m", - required=True) - ] - nbits: Annotated[ - str, click.option("--nbits", - type=int, - required=True) - ] +class MilvusGPUIVFPQTypedDict( + CommonTypedDict, + MilvusTypedDict, + MilvusIVFFlatTypedDict, + MilvusGPUIVFTypedDict, +): + m: Annotated[str, click.option("--m", type=int, help="hnsw m", required=True)] + nbits: Annotated[str, click.option("--nbits", type=int, required=True)] @cli.command() @click_parameter_decorators_from_typed_dict(MilvusGPUIVFPQTypedDict) def MilvusGPUIVFPQ(**parameters: Unpack[MilvusGPUIVFPQTypedDict]): - from .config import MilvusConfig, GPUIVFPQConfig + from .config import GPUIVFPQConfig, MilvusConfig run( db=DBTYPE, @@ -245,51 +232,22 @@ def MilvusGPUIVFPQ(**parameters: Unpack[MilvusGPUIVFPQTypedDict]): class MilvusGPUCAGRATypedDict(CommonTypedDict, MilvusTypedDict, MilvusGPUIVFTypedDict): intermediate_graph_degree: Annotated[ - str, click.option("--intermediate-graph-degree", - type=int, - required=True) - ] - graph_degree: Annotated[ - str, click.option("--graph-degree", - type=int, - required=True) - ] - build_algo: Annotated[ - str, click.option("--build_algo", - type=str, - required=True) - ] - team_size: Annotated[ - str, click.option("--team-size", - type=int, - required=True) - ] - search_width: Annotated[ - str, click.option("--search-width", - type=int, - required=True) - ] - itopk_size: Annotated[ - str, click.option("--itopk-size", - type=int, - required=True) - ] - min_iterations: Annotated[ - str, click.option("--min-iterations", - type=int, - required=True) - ] - max_iterations: Annotated[ - str, click.option("--max-iterations", - type=int, - required=True) + str, + click.option("--intermediate-graph-degree", type=int, required=True), ] + graph_degree: Annotated[str, click.option("--graph-degree", type=int, required=True)] + build_algo: Annotated[str, click.option("--build_algo", type=str, required=True)] + team_size: Annotated[str, click.option("--team-size", type=int, required=True)] + search_width: Annotated[str, click.option("--search-width", type=int, required=True)] + itopk_size: Annotated[str, click.option("--itopk-size", type=int, required=True)] + min_iterations: Annotated[str, click.option("--min-iterations", type=int, required=True)] + max_iterations: Annotated[str, click.option("--max-iterations", type=int, required=True)] @cli.command() @click_parameter_decorators_from_typed_dict(MilvusGPUCAGRATypedDict) def MilvusGPUCAGRA(**parameters: Unpack[MilvusGPUCAGRATypedDict]): - from .config import MilvusConfig, GPUCAGRAConfig + from .config import GPUCAGRAConfig, MilvusConfig run( db=DBTYPE, diff --git a/vectordb_bench/backend/clients/milvus/config.py b/vectordb_bench/backend/clients/milvus/config.py index 059ef0461..7d0df803a 100644 --- a/vectordb_bench/backend/clients/milvus/config.py +++ b/vectordb_bench/backend/clients/milvus/config.py @@ -1,5 +1,6 @@ from pydantic import BaseModel, SecretStr, validator -from ..api import DBConfig, DBCaseConfig, MetricType, IndexType + +from ..api import DBCaseConfig, DBConfig, IndexType, MetricType class MilvusConfig(DBConfig): @@ -15,10 +16,14 @@ def to_dict(self) -> dict: } @validator("*") - def not_empty_field(cls, v, field): - if field.name in cls.common_short_configs() or field.name in cls.common_long_configs() or field.name in ["user", "password"]: + def not_empty_field(cls, v: any, field: any): + if ( + field.name in cls.common_short_configs() + or field.name in cls.common_long_configs() + or field.name in ["user", "password"] + ): return v - if isinstance(v, (str, SecretStr)) and len(v) == 0: + if isinstance(v, str | SecretStr) and len(v) == 0: raise ValueError("Empty string!") return v @@ -28,10 +33,14 @@ class MilvusIndexConfig(BaseModel): index: IndexType metric_type: MetricType | None = None - + @property def is_gpu_index(self) -> bool: - return self.index in [IndexType.GPU_CAGRA, IndexType.GPU_IVF_FLAT, IndexType.GPU_IVF_PQ] + return self.index in [ + IndexType.GPU_CAGRA, + IndexType.GPU_IVF_FLAT, + IndexType.GPU_IVF_PQ, + ] def parse_metric(self) -> str: if not self.metric_type: @@ -113,7 +122,8 @@ def search_param(self) -> dict: "metric_type": self.parse_metric(), "params": {"nprobe": self.nprobe}, } - + + class IVFSQ8Config(MilvusIndexConfig, DBCaseConfig): nlist: int nprobe: int | None = None @@ -210,7 +220,7 @@ class GPUCAGRAConfig(MilvusIndexConfig, DBCaseConfig): search_width: int = 4 min_iterations: int = 0 max_iterations: int = 0 - build_algo: str = "IVF_PQ" # IVF_PQ; NN_DESCENT; + build_algo: str = "IVF_PQ" # IVF_PQ; NN_DESCENT; cache_dataset_on_device: str refine_ratio: float | None = None index: IndexType = IndexType.GPU_CAGRA diff --git a/vectordb_bench/backend/clients/milvus/milvus.py b/vectordb_bench/backend/clients/milvus/milvus.py index 251dee8ad..45fe7269b 100644 --- a/vectordb_bench/backend/clients/milvus/milvus.py +++ b/vectordb_bench/backend/clients/milvus/milvus.py @@ -2,19 +2,18 @@ import logging import time +from collections.abc import Iterable from contextlib import contextmanager -from typing import Iterable -from pymilvus import Collection, utility -from pymilvus import CollectionSchema, DataType, FieldSchema, MilvusException +from pymilvus import Collection, CollectionSchema, DataType, FieldSchema, MilvusException, utility from ..api import VectorDB from .config import MilvusIndexConfig - log = logging.getLogger(__name__) -MILVUS_LOAD_REQS_SIZE = 1.5 * 1024 *1024 +MILVUS_LOAD_REQS_SIZE = 1.5 * 1024 * 1024 + class Milvus(VectorDB): def __init__( @@ -32,7 +31,7 @@ def __init__( self.db_config = db_config self.case_config = db_case_config self.collection_name = collection_name - self.batch_size = int(MILVUS_LOAD_REQS_SIZE / (dim *4)) + self.batch_size = int(MILVUS_LOAD_REQS_SIZE / (dim * 4)) self._primary_field = "pk" self._scalar_field = "id" @@ -40,6 +39,7 @@ def __init__( self._index_name = "vector_idx" from pymilvus import connections + connections.connect(**self.db_config, timeout=30) if drop_old and utility.has_collection(self.collection_name): log.info(f"{self.name} client drop_old collection: {self.collection_name}") @@ -49,7 +49,7 @@ def __init__( fields = [ FieldSchema(self._primary_field, DataType.INT64, is_primary=True), FieldSchema(self._scalar_field, DataType.INT64), - FieldSchema(self._vector_field, DataType.FLOAT_VECTOR, dim=dim) + FieldSchema(self._vector_field, DataType.FLOAT_VECTOR, dim=dim), ] log.info(f"{self.name} create collection: {self.collection_name}") @@ -79,6 +79,7 @@ def init(self) -> None: >>> self.search_embedding() """ from pymilvus import connections + self.col: Collection | None = None connections.connect(**self.db_config, timeout=60) @@ -108,6 +109,7 @@ def _post_insert(self): ) utility.wait_for_index_building_complete(self.collection_name) + def wait_index(): while True: progress = utility.index_building_progress(self.collection_name) @@ -120,18 +122,17 @@ def wait_index(): # Skip compaction if use GPU indexType if self.case_config.is_gpu_index: log.debug("skip compaction for gpu index type.") - else : + else: try: self.col.compact() self.col.wait_for_compaction_completed() except Exception as e: log.warning(f"{self.name} compact error: {e}") - if hasattr(e, 'code'): - if e.code().name == 'PERMISSION_DENIED': + if hasattr(e, "code"): + if e.code().name == "PERMISSION_DENIED": log.warning("Skip compact due to permission denied.") - pass else: - raise e + raise e from e wait_index() except Exception as e: log.warning(f"{self.name} optimize error: {e}") @@ -156,7 +157,6 @@ def _pre_load(self, coll: Collection): log.warning(f"{self.name} pre load error: {e}") raise e from None - def optimize(self): assert self.col, "Please call self.init() before" self._optimize() @@ -164,7 +164,7 @@ def optimize(self): def need_normalize_cosine(self) -> bool: """Wheather this database need to normalize dataset to support COSINE""" if self.case_config.is_gpu_index: - log.info(f"current gpu_index only supports IP / L2, cosine dataset need normalize.") + log.info("current gpu_index only supports IP / L2, cosine dataset need normalize.") return True return False @@ -184,9 +184,9 @@ def insert_embeddings( for batch_start_offset in range(0, len(embeddings), self.batch_size): batch_end_offset = min(batch_start_offset + self.batch_size, len(embeddings)) insert_data = [ - metadata[batch_start_offset : batch_end_offset], - metadata[batch_start_offset : batch_end_offset], - embeddings[batch_start_offset : batch_end_offset], + metadata[batch_start_offset:batch_end_offset], + metadata[batch_start_offset:batch_end_offset], + embeddings[batch_start_offset:batch_end_offset], ] res = self.col.insert(insert_data) insert_count += len(res.primary_keys) @@ -217,5 +217,4 @@ def search_embedding( ) # Organize results. - ret = [result.id for result in res[0]] - return ret + return [result.id for result in res[0]] diff --git a/vectordb_bench/backend/clients/pgdiskann/cli.py b/vectordb_bench/backend/clients/pgdiskann/cli.py index 18a9ecbd5..19f47988f 100644 --- a/vectordb_bench/backend/clients/pgdiskann/cli.py +++ b/vectordb_bench/backend/clients/pgdiskann/cli.py @@ -1,57 +1,63 @@ -import click import os +from typing import Annotated, Unpack + +import click from pydantic import SecretStr +from vectordb_bench.backend.clients import DB + from ....cli.cli import ( CommonTypedDict, cli, click_parameter_decorators_from_typed_dict, run, ) -from typing import Annotated, Optional, Unpack -from vectordb_bench.backend.clients import DB class PgDiskAnnTypedDict(CommonTypedDict): user_name: Annotated[ - str, click.option("--user-name", type=str, help="Db username", required=True) + str, + click.option("--user-name", type=str, help="Db username", required=True), ] password: Annotated[ str, - click.option("--password", - type=str, - help="Postgres database password", - default=lambda: os.environ.get("POSTGRES_PASSWORD", ""), - show_default="$POSTGRES_PASSWORD", - ), + click.option( + "--password", + type=str, + help="Postgres database password", + default=lambda: os.environ.get("POSTGRES_PASSWORD", ""), + show_default="$POSTGRES_PASSWORD", + ), ] - host: Annotated[ - str, click.option("--host", type=str, help="Db host", required=True) - ] - db_name: Annotated[ - str, click.option("--db-name", type=str, help="Db name", required=True) - ] + host: Annotated[str, click.option("--host", type=str, help="Db host", required=True)] + db_name: Annotated[str, click.option("--db-name", type=str, help="Db name", required=True)] max_neighbors: Annotated[ int, click.option( - "--max-neighbors", type=int, help="PgDiskAnn max neighbors", + "--max-neighbors", + type=int, + help="PgDiskAnn max neighbors", ), ] l_value_ib: Annotated[ int, click.option( - "--l-value-ib", type=int, help="PgDiskAnn l_value_ib", + "--l-value-ib", + type=int, + help="PgDiskAnn l_value_ib", ), ] l_value_is: Annotated[ float, click.option( - "--l-value-is", type=float, help="PgDiskAnn l_value_is", + "--l-value-is", + type=float, + help="PgDiskAnn l_value_is", ), ] maintenance_work_mem: Annotated[ - Optional[str], + str | None, click.option( "--maintenance-work-mem", type=str, @@ -63,7 +69,7 @@ class PgDiskAnnTypedDict(CommonTypedDict): ), ] max_parallel_workers: Annotated[ - Optional[int], + int | None, click.option( "--max-parallel-workers", type=int, @@ -72,6 +78,7 @@ class PgDiskAnnTypedDict(CommonTypedDict): ), ] + @cli.command() @click_parameter_decorators_from_typed_dict(PgDiskAnnTypedDict) def PgDiskAnn( @@ -96,4 +103,4 @@ def PgDiskAnn( maintenance_work_mem=parameters["maintenance_work_mem"], ), **parameters, - ) \ No newline at end of file + ) diff --git a/vectordb_bench/backend/clients/pgdiskann/config.py b/vectordb_bench/backend/clients/pgdiskann/config.py index 970720afa..ed478acc2 100644 --- a/vectordb_bench/backend/clients/pgdiskann/config.py +++ b/vectordb_bench/backend/clients/pgdiskann/config.py @@ -1,7 +1,9 @@ from abc import abstractmethod -from typing import Any, Mapping, Optional, Sequence, TypedDict +from collections.abc import Mapping, Sequence +from typing import Any, LiteralString, TypedDict + from pydantic import BaseModel, SecretStr -from typing_extensions import LiteralString + from ..api import DBCaseConfig, DBConfig, IndexType, MetricType POSTGRE_URL_PLACEHOLDER = "postgresql://%s:%s@%s/%s" @@ -9,7 +11,7 @@ class PgDiskANNConfigDict(TypedDict): """These keys will be directly used as kwargs in psycopg connection string, - so the names must match exactly psycopg API""" + so the names must match exactly psycopg API""" user: str password: str @@ -41,44 +43,43 @@ class PgDiskANNIndexConfig(BaseModel, DBCaseConfig): metric_type: MetricType | None = None create_index_before_load: bool = False create_index_after_load: bool = True - maintenance_work_mem: Optional[str] - max_parallel_workers: Optional[int] + maintenance_work_mem: str | None + max_parallel_workers: int | None def parse_metric(self) -> str: if self.metric_type == MetricType.L2: return "vector_l2_ops" - elif self.metric_type == MetricType.IP: + if self.metric_type == MetricType.IP: return "vector_ip_ops" return "vector_cosine_ops" def parse_metric_fun_op(self) -> LiteralString: if self.metric_type == MetricType.L2: return "<->" - elif self.metric_type == MetricType.IP: + if self.metric_type == MetricType.IP: return "<#>" return "<=>" def parse_metric_fun_str(self) -> str: if self.metric_type == MetricType.L2: return "l2_distance" - elif self.metric_type == MetricType.IP: + if self.metric_type == MetricType.IP: return "max_inner_product" return "cosine_distance" - + @abstractmethod - def index_param(self) -> dict: - ... + def index_param(self) -> dict: ... @abstractmethod - def search_param(self) -> dict: - ... + def search_param(self) -> dict: ... @abstractmethod - def session_param(self) -> dict: - ... + def session_param(self) -> dict: ... @staticmethod - def _optionally_build_with_options(with_options: Mapping[str, Any]) -> Sequence[dict[str, Any]]: + def _optionally_build_with_options( + with_options: Mapping[str, Any], + ) -> Sequence[dict[str, Any]]: """Walk through mappings, creating a List of {key1 = value} pairs. That will be used to build a where clause""" options = [] for option_name, value in with_options.items(): @@ -87,35 +88,36 @@ def _optionally_build_with_options(with_options: Mapping[str, Any]) -> Sequence[ { "option_name": option_name, "val": str(value), - } + }, ) return options @staticmethod def _optionally_build_set_options( - set_mapping: Mapping[str, Any] + set_mapping: Mapping[str, Any], ) -> Sequence[dict[str, Any]]: """Walk through options, creating 'SET 'key1 = "value1";' list""" session_options = [] for setting_name, value in set_mapping.items(): if value: session_options.append( - {"parameter": { + { + "parameter": { "setting_name": setting_name, "val": str(value), }, - } + }, ) return session_options - + class PgDiskANNImplConfig(PgDiskANNIndexConfig): index: IndexType = IndexType.DISKANN max_neighbors: int | None l_value_ib: int | None l_value_is: float | None - maintenance_work_mem: Optional[str] = None - max_parallel_workers: Optional[int] = None + maintenance_work_mem: str | None = None + max_parallel_workers: int | None = None def index_param(self) -> dict: return { @@ -128,18 +130,19 @@ def index_param(self) -> dict: "maintenance_work_mem": self.maintenance_work_mem, "max_parallel_workers": self.max_parallel_workers, } - + def search_param(self) -> dict: return { "metric": self.parse_metric(), "metric_fun_op": self.parse_metric_fun_op(), } - + def session_param(self) -> dict: return { "diskann.l_value_is": self.l_value_is, } - + + _pgdiskann_case_config = { IndexType.DISKANN: PgDiskANNImplConfig, } diff --git a/vectordb_bench/backend/clients/pgdiskann/pgdiskann.py b/vectordb_bench/backend/clients/pgdiskann/pgdiskann.py index c363490f7..c21972902 100644 --- a/vectordb_bench/backend/clients/pgdiskann/pgdiskann.py +++ b/vectordb_bench/backend/clients/pgdiskann/pgdiskann.py @@ -1,9 +1,9 @@ """Wrapper around the pg_diskann vector database over VectorDB""" import logging -import pprint +from collections.abc import Generator from contextlib import contextmanager -from typing import Any, Generator, Optional, Tuple +from typing import Any import numpy as np import psycopg @@ -44,20 +44,21 @@ def __init__( self._primary_field = "id" self._vector_field = "embedding" - self.conn, self.cursor = self._create_connection(**self.db_config) + self.conn, self.cursor = self._create_connection(**self.db_config) log.info(f"{self.name} config values: {self.db_config}\n{self.case_config}") if not any( ( self.case_config.create_index_before_load, self.case_config.create_index_after_load, - ) + ), ): - err = f"{self.name} config must create an index using create_index_before_load or create_index_after_load" - log.error(err) - raise RuntimeError( - f"{err}\n{pprint.pformat(self.db_config)}\n{pprint.pformat(self.case_config)}" + msg = ( + f"{self.name} config must create an index using create_index_before_load or create_index_after_load" + f"{self.name} config values: {self.db_config}\n{self.case_config}" ) + log.error(msg) + raise RuntimeError(msg) if drop_old: self._drop_index() @@ -72,7 +73,7 @@ def __init__( self.conn = None @staticmethod - def _create_connection(**kwargs) -> Tuple[Connection, Cursor]: + def _create_connection(**kwargs) -> tuple[Connection, Cursor]: conn = psycopg.connect(**kwargs) conn.cursor().execute("CREATE EXTENSION IF NOT EXISTS pg_diskann CASCADE") conn.commit() @@ -101,25 +102,25 @@ def init(self) -> Generator[None, None, None]: log.debug(command.as_string(self.cursor)) self.cursor.execute(command) self.conn.commit() - + self._filtered_search = sql.Composed( [ sql.SQL( - "SELECT id FROM public.{table_name} WHERE id >= %s ORDER BY embedding " - ).format(table_name=sql.Identifier(self.table_name)), + "SELECT id FROM public.{table_name} WHERE id >= %s ORDER BY embedding ", + ).format(table_name=sql.Identifier(self.table_name)), sql.SQL(self.case_config.search_param()["metric_fun_op"]), sql.SQL(" %s::vector LIMIT %s::int"), - ] + ], ) self._unfiltered_search = sql.Composed( [ sql.SQL("SELECT id FROM public.{} ORDER BY embedding ").format( - sql.Identifier(self.table_name) + sql.Identifier(self.table_name), ), sql.SQL(self.case_config.search_param()["metric_fun_op"]), sql.SQL(" %s::vector LIMIT %s::int"), - ] + ], ) try: @@ -137,8 +138,8 @@ def _drop_table(self): self.cursor.execute( sql.SQL("DROP TABLE IF EXISTS public.{table_name}").format( - table_name=sql.Identifier(self.table_name) - ) + table_name=sql.Identifier(self.table_name), + ), ) self.conn.commit() @@ -160,7 +161,7 @@ def _drop_index(self): log.info(f"{self.name} client drop index : {self._index_name}") drop_index_sql = sql.SQL("DROP INDEX IF EXISTS {index_name}").format( - index_name=sql.Identifier(self._index_name) + index_name=sql.Identifier(self._index_name), ) log.debug(drop_index_sql.as_string(self.cursor)) self.cursor.execute(drop_index_sql) @@ -175,64 +176,53 @@ def _set_parallel_index_build_param(self): if index_param["maintenance_work_mem"] is not None: self.cursor.execute( sql.SQL("SET maintenance_work_mem TO {};").format( - index_param["maintenance_work_mem"] - ) + index_param["maintenance_work_mem"], + ), ) self.cursor.execute( sql.SQL("ALTER USER {} SET maintenance_work_mem TO {};").format( sql.Identifier(self.db_config["user"]), index_param["maintenance_work_mem"], - ) + ), ) self.conn.commit() if index_param["max_parallel_workers"] is not None: self.cursor.execute( sql.SQL("SET max_parallel_maintenance_workers TO '{}';").format( - index_param["max_parallel_workers"] - ) + index_param["max_parallel_workers"], + ), ) self.cursor.execute( - sql.SQL( - "ALTER USER {} SET max_parallel_maintenance_workers TO '{}';" - ).format( + sql.SQL("ALTER USER {} SET max_parallel_maintenance_workers TO '{}';").format( sql.Identifier(self.db_config["user"]), index_param["max_parallel_workers"], - ) + ), ) self.cursor.execute( sql.SQL("SET max_parallel_workers TO '{}';").format( - index_param["max_parallel_workers"] - ) + index_param["max_parallel_workers"], + ), ) self.cursor.execute( - sql.SQL( - "ALTER USER {} SET max_parallel_workers TO '{}';" - ).format( + sql.SQL("ALTER USER {} SET max_parallel_workers TO '{}';").format( sql.Identifier(self.db_config["user"]), index_param["max_parallel_workers"], - ) + ), ) self.cursor.execute( - sql.SQL( - "ALTER TABLE {} SET (parallel_workers = {});" - ).format( + sql.SQL("ALTER TABLE {} SET (parallel_workers = {});").format( sql.Identifier(self.table_name), index_param["max_parallel_workers"], - ) + ), ) self.conn.commit() - results = self.cursor.execute( - sql.SQL("SHOW max_parallel_maintenance_workers;") - ).fetchall() - results.extend( - self.cursor.execute(sql.SQL("SHOW max_parallel_workers;")).fetchall() - ) - results.extend( - self.cursor.execute(sql.SQL("SHOW maintenance_work_mem;")).fetchall() - ) + results = self.cursor.execute(sql.SQL("SHOW max_parallel_maintenance_workers;")).fetchall() + results.extend(self.cursor.execute(sql.SQL("SHOW max_parallel_workers;")).fetchall()) + results.extend(self.cursor.execute(sql.SQL("SHOW maintenance_work_mem;")).fetchall()) log.info(f"{self.name} parallel index creation parameters: {results}") + def _create_index(self): assert self.conn is not None, "Connection is not initialized" assert self.cursor is not None, "Cursor is not initialized" @@ -248,28 +238,23 @@ def _create_index(self): sql.SQL("{option_name} = {val}").format( option_name=sql.Identifier(option_name), val=sql.Identifier(str(option_val)), - ) + ), ) - - if any(options): - with_clause = sql.SQL("WITH ({});").format(sql.SQL(", ").join(options)) - else: - with_clause = sql.Composed(()) + + with_clause = sql.SQL("WITH ({});").format(sql.SQL(", ").join(options)) if any(options) else sql.Composed(()) index_create_sql = sql.SQL( """ - CREATE INDEX IF NOT EXISTS {index_name} ON public.{table_name} + CREATE INDEX IF NOT EXISTS {index_name} ON public.{table_name} USING {index_type} (embedding {embedding_metric}) - """ + """, ).format( index_name=sql.Identifier(self._index_name), table_name=sql.Identifier(self.table_name), index_type=sql.Identifier(index_param["index_type"].lower()), embedding_metric=sql.Identifier(index_param["metric"]), ) - index_create_sql_with_with_clause = ( - index_create_sql + with_clause - ).join(" ") + index_create_sql_with_with_clause = (index_create_sql + with_clause).join(" ") log.debug(index_create_sql_with_with_clause.as_string(self.cursor)) self.cursor.execute(index_create_sql_with_with_clause) self.conn.commit() @@ -283,14 +268,12 @@ def _create_table(self, dim: int): self.cursor.execute( sql.SQL( - "CREATE TABLE IF NOT EXISTS public.{table_name} (id BIGINT PRIMARY KEY, embedding vector({dim}));" - ).format(table_name=sql.Identifier(self.table_name), dim=dim) + "CREATE TABLE IF NOT EXISTS public.{table_name} (id BIGINT PRIMARY KEY, embedding vector({dim}));", + ).format(table_name=sql.Identifier(self.table_name), dim=dim), ) self.conn.commit() except Exception as e: - log.warning( - f"Failed to create pgdiskann table: {self.table_name} error: {e}" - ) + log.warning(f"Failed to create pgdiskann table: {self.table_name} error: {e}") raise e from None def insert_embeddings( @@ -298,7 +281,7 @@ def insert_embeddings( embeddings: list[list[float]], metadata: list[int], **kwargs: Any, - ) -> Tuple[int, Optional[Exception]]: + ) -> tuple[int, Exception | None]: assert self.conn is not None, "Connection is not initialized" assert self.cursor is not None, "Cursor is not initialized" @@ -308,8 +291,8 @@ def insert_embeddings( with self.cursor.copy( sql.SQL("COPY public.{table_name} FROM STDIN (FORMAT BINARY)").format( - table_name=sql.Identifier(self.table_name) - ) + table_name=sql.Identifier(self.table_name), + ), ) as copy: copy.set_types(["bigint", "vector"]) for i, row in enumerate(metadata_arr): @@ -321,9 +304,7 @@ def insert_embeddings( return len(metadata), None except Exception as e: - log.warning( - f"Failed to insert data into table ({self.table_name}), error: {e}" - ) + log.warning(f"Failed to insert data into table ({self.table_name}), error: {e}") return 0, e def search_embedding( @@ -340,11 +321,12 @@ def search_embedding( if filters: gt = filters.get("id") result = self.cursor.execute( - self._filtered_search, (gt, q, k), prepare=True, binary=True - ) + self._filtered_search, + (gt, q, k), + prepare=True, + binary=True, + ) else: - result = self.cursor.execute( - self._unfiltered_search, (q, k), prepare=True, binary=True - ) + result = self.cursor.execute(self._unfiltered_search, (q, k), prepare=True, binary=True) return [int(i[0]) for i in result.fetchall()] diff --git a/vectordb_bench/backend/clients/pgvecto_rs/cli.py b/vectordb_bench/backend/clients/pgvecto_rs/cli.py index 10dbff556..24d2cf800 100644 --- a/vectordb_bench/backend/clients/pgvecto_rs/cli.py +++ b/vectordb_bench/backend/clients/pgvecto_rs/cli.py @@ -1,9 +1,11 @@ -from typing import Annotated, Optional, Unpack +import os +from typing import Annotated, Unpack import click -import os from pydantic import SecretStr +from vectordb_bench.backend.clients import DB + from ....cli.cli import ( CommonTypedDict, HNSWFlavor1, @@ -12,12 +14,12 @@ click_parameter_decorators_from_typed_dict, run, ) -from vectordb_bench.backend.clients import DB class PgVectoRSTypedDict(CommonTypedDict): user_name: Annotated[ - str, click.option("--user-name", type=str, help="Db username", required=True) + str, + click.option("--user-name", type=str, help="Db username", required=True), ] password: Annotated[ str, @@ -30,14 +32,10 @@ class PgVectoRSTypedDict(CommonTypedDict): ), ] - host: Annotated[ - str, click.option("--host", type=str, help="Db host", required=True) - ] - db_name: Annotated[ - str, click.option("--db-name", type=str, help="Db name", required=True) - ] + host: Annotated[str, click.option("--host", type=str, help="Db host", required=True)] + db_name: Annotated[str, click.option("--db-name", type=str, help="Db name", required=True)] max_parallel_workers: Annotated[ - Optional[int], + int | None, click.option( "--max-parallel-workers", type=int, diff --git a/vectordb_bench/backend/clients/pgvecto_rs/config.py b/vectordb_bench/backend/clients/pgvecto_rs/config.py index c671a236c..fbb7c5d81 100644 --- a/vectordb_bench/backend/clients/pgvecto_rs/config.py +++ b/vectordb_bench/backend/clients/pgvecto_rs/config.py @@ -1,11 +1,11 @@ from abc import abstractmethod from typing import TypedDict +from pgvecto_rs.types import Flat, Hnsw, IndexOption, Ivf, Quantization +from pgvecto_rs.types.index import QuantizationRatio, QuantizationType from pydantic import BaseModel, SecretStr -from pgvecto_rs.types import IndexOption, Ivf, Hnsw, Flat, Quantization -from pgvecto_rs.types.index import QuantizationType, QuantizationRatio -from ..api import DBConfig, DBCaseConfig, IndexType, MetricType +from ..api import DBCaseConfig, DBConfig, IndexType, MetricType POSTGRE_URL_PLACEHOLDER = "postgresql://%s:%s@%s/%s" @@ -52,14 +52,14 @@ class PgVectoRSIndexConfig(BaseModel, DBCaseConfig): def parse_metric(self) -> str: if self.metric_type == MetricType.L2: return "vector_l2_ops" - elif self.metric_type == MetricType.IP: + if self.metric_type == MetricType.IP: return "vector_dot_ops" return "vector_cos_ops" def parse_metric_fun_op(self) -> str: if self.metric_type == MetricType.L2: return "<->" - elif self.metric_type == MetricType.IP: + if self.metric_type == MetricType.IP: return "<#>" return "<=>" @@ -85,9 +85,7 @@ def index_param(self) -> dict[str, str]: if self.quantization_type is None: quantization = None else: - quantization = Quantization( - typ=self.quantization_type, ratio=self.quantization_ratio - ) + quantization = Quantization(typ=self.quantization_type, ratio=self.quantization_ratio) option = IndexOption( index=Hnsw( @@ -115,9 +113,7 @@ def index_param(self) -> dict[str, str]: if self.quantization_type is None: quantization = None else: - quantization = Quantization( - typ=self.quantization_type, ratio=self.quantization_ratio - ) + quantization = Quantization(typ=self.quantization_type, ratio=self.quantization_ratio) option = IndexOption( index=Ivf(nlist=self.lists, quantization=quantization), @@ -139,9 +135,7 @@ def index_param(self) -> dict[str, str]: if self.quantization_type is None: quantization = None else: - quantization = Quantization( - typ=self.quantization_type, ratio=self.quantization_ratio - ) + quantization = Quantization(typ=self.quantization_type, ratio=self.quantization_ratio) option = IndexOption( index=Flat( diff --git a/vectordb_bench/backend/clients/pgvecto_rs/pgvecto_rs.py b/vectordb_bench/backend/clients/pgvecto_rs/pgvecto_rs.py index bc042cc57..fc4f17807 100644 --- a/vectordb_bench/backend/clients/pgvecto_rs/pgvecto_rs.py +++ b/vectordb_bench/backend/clients/pgvecto_rs/pgvecto_rs.py @@ -1,14 +1,14 @@ """Wrapper around the Pgvecto.rs vector database over VectorDB""" import logging -import pprint +from collections.abc import Generator from contextlib import contextmanager -from typing import Any, Generator, Optional, Tuple +from typing import Any import numpy as np import psycopg -from psycopg import Connection, Cursor, sql from pgvecto_rs.psycopg import register_vector +from psycopg import Connection, Cursor, sql from ..api import VectorDB from .config import PgVectoRSConfig, PgVectoRSIndexConfig @@ -33,7 +33,6 @@ def __init__( drop_old: bool = False, **kwargs, ): - self.name = "PgVectorRS" self.db_config = db_config self.case_config = db_case_config @@ -52,13 +51,14 @@ def __init__( ( self.case_config.create_index_before_load, self.case_config.create_index_after_load, - ) + ), ): - err = f"{self.name} config must create an index using create_index_before_load or create_index_after_load" - log.error(err) - raise RuntimeError( - f"{err}\n{pprint.pformat(self.db_config)}\n{pprint.pformat(self.case_config)}" + msg = ( + f"{self.name} config must create an index using create_index_before_load or create_index_after_load" + f"{self.name} config values: {self.db_config}\n{self.case_config}" ) + log.error(msg) + raise RuntimeError(msg) if drop_old: log.info(f"Pgvecto.rs client drop table : {self.table_name}") @@ -74,7 +74,7 @@ def __init__( self.conn = None @staticmethod - def _create_connection(**kwargs) -> Tuple[Connection, Cursor]: + def _create_connection(**kwargs) -> tuple[Connection, Cursor]: conn = psycopg.connect(**kwargs) # create vector extension @@ -116,21 +116,21 @@ def init(self) -> Generator[None, None, None]: self._filtered_search = sql.Composed( [ sql.SQL( - "SELECT id FROM public.{table_name} WHERE id >= %s ORDER BY embedding " + "SELECT id FROM public.{table_name} WHERE id >= %s ORDER BY embedding ", ).format(table_name=sql.Identifier(self.table_name)), sql.SQL(self.case_config.search_param()["metric_fun_op"]), sql.SQL(" %s::vector LIMIT %s::int"), - ] + ], ) self._unfiltered_search = sql.Composed( [ - sql.SQL( - "SELECT id FROM public.{table_name} ORDER BY embedding " - ).format(table_name=sql.Identifier(self.table_name)), + sql.SQL("SELECT id FROM public.{table_name} ORDER BY embedding ").format( + table_name=sql.Identifier(self.table_name), + ), sql.SQL(self.case_config.search_param()["metric_fun_op"]), sql.SQL(" %s::vector LIMIT %s::int"), - ] + ], ) try: @@ -148,8 +148,8 @@ def _drop_table(self): self.cursor.execute( sql.SQL("DROP TABLE IF EXISTS public.{table_name}").format( - table_name=sql.Identifier(self.table_name) - ) + table_name=sql.Identifier(self.table_name), + ), ) self.conn.commit() @@ -171,7 +171,7 @@ def _drop_index(self): log.info(f"{self.name} client drop index : {self._index_name}") drop_index_sql = sql.SQL("DROP INDEX IF EXISTS {index_name}").format( - index_name=sql.Identifier(self._index_name) + index_name=sql.Identifier(self._index_name), ) log.debug(drop_index_sql.as_string(self.cursor)) self.cursor.execute(drop_index_sql) @@ -186,9 +186,9 @@ def _create_index(self): index_create_sql = sql.SQL( """ - CREATE INDEX IF NOT EXISTS {index_name} ON public.{table_name} + CREATE INDEX IF NOT EXISTS {index_name} ON public.{table_name} USING vectors (embedding {embedding_metric}) WITH (options = {index_options}) - """ + """, ).format( index_name=sql.Identifier(self._index_name), table_name=sql.Identifier(self.table_name), @@ -202,7 +202,7 @@ def _create_index(self): except Exception as e: log.warning( f"Failed to create pgvecto.rs index {self._index_name} \ - at table {self.table_name} error: {e}" + at table {self.table_name} error: {e}", ) raise e from None @@ -214,7 +214,7 @@ def _create_table(self, dim: int): """ CREATE TABLE IF NOT EXISTS public.{table_name} (id BIGINT PRIMARY KEY, embedding vector({dim})) - """ + """, ).format( table_name=sql.Identifier(self.table_name), dim=dim, @@ -224,9 +224,7 @@ def _create_table(self, dim: int): self.cursor.execute(table_create_sql) self.conn.commit() except Exception as e: - log.warning( - f"Failed to create pgvecto.rs table: {self.table_name} error: {e}" - ) + log.warning(f"Failed to create pgvecto.rs table: {self.table_name} error: {e}") raise e from None def insert_embeddings( @@ -234,7 +232,7 @@ def insert_embeddings( embeddings: list[list[float]], metadata: list[int], **kwargs: Any, - ) -> Tuple[int, Optional[Exception]]: + ) -> tuple[int, Exception | None]: assert self.conn is not None, "Connection is not initialized" assert self.cursor is not None, "Cursor is not initialized" @@ -247,8 +245,8 @@ def insert_embeddings( with self.cursor.copy( sql.SQL("COPY public.{table_name} FROM STDIN (FORMAT BINARY)").format( - table_name=sql.Identifier(self.table_name) - ) + table_name=sql.Identifier(self.table_name), + ), ) as copy: copy.set_types(["bigint", "vector"]) for i, row in enumerate(metadata_arr): @@ -261,7 +259,7 @@ def insert_embeddings( return len(metadata), None except Exception as e: log.warning( - f"Failed to insert data into pgvecto.rs table ({self.table_name}), error: {e}" + f"Failed to insert data into pgvecto.rs table ({self.table_name}), error: {e}", ) return 0, e @@ -281,12 +279,13 @@ def search_embedding( log.debug(self._filtered_search.as_string(self.cursor)) gt = filters.get("id") result = self.cursor.execute( - self._filtered_search, (gt, q, k), prepare=True, binary=True + self._filtered_search, + (gt, q, k), + prepare=True, + binary=True, ) else: log.debug(self._unfiltered_search.as_string(self.cursor)) - result = self.cursor.execute( - self._unfiltered_search, (q, k), prepare=True, binary=True - ) + result = self.cursor.execute(self._unfiltered_search, (q, k), prepare=True, binary=True) return [int(i[0]) for i in result.fetchall()] diff --git a/vectordb_bench/backend/clients/pgvector/cli.py b/vectordb_bench/backend/clients/pgvector/cli.py index ef8914be0..55a462055 100644 --- a/vectordb_bench/backend/clients/pgvector/cli.py +++ b/vectordb_bench/backend/clients/pgvector/cli.py @@ -1,9 +1,10 @@ -from typing import Annotated, Optional, TypedDict, Unpack +import os +from typing import Annotated, Unpack import click -import os from pydantic import SecretStr +from vectordb_bench.backend.clients import DB from vectordb_bench.backend.clients.api import MetricType from ....cli.cli import ( @@ -15,49 +16,48 @@ get_custom_case_config, run, ) -from vectordb_bench.backend.clients import DB -def set_default_quantized_fetch_limit(ctx, param, value): +# ruff: noqa +def set_default_quantized_fetch_limit(ctx: any, param: any, value: any): if ctx.params.get("reranking") and value is None: # ef_search is the default value for quantized_fetch_limit as it's bound by ef_search. # 100 is default value for quantized_fetch_limit for IVFFlat. - default_value = ctx.params["ef_search"] if ctx.command.name == "pgvectorhnsw" else 100 - return default_value + return ctx.params["ef_search"] if ctx.command.name == "pgvectorhnsw" else 100 return value + class PgVectorTypedDict(CommonTypedDict): user_name: Annotated[ - str, click.option("--user-name", type=str, help="Db username", required=True) + str, + click.option("--user-name", type=str, help="Db username", required=True), ] password: Annotated[ str, - click.option("--password", - type=str, - help="Postgres database password", - default=lambda: os.environ.get("POSTGRES_PASSWORD", ""), - show_default="$POSTGRES_PASSWORD", - ), + click.option( + "--password", + type=str, + help="Postgres database password", + default=lambda: os.environ.get("POSTGRES_PASSWORD", ""), + show_default="$POSTGRES_PASSWORD", + ), ] - host: Annotated[ - str, click.option("--host", type=str, help="Db host", required=True) - ] - port: Annotated[ + host: Annotated[str, click.option("--host", type=str, help="Db host", required=True)] + port: Annotated[ int, - click.option("--port", - type=int, - help="Postgres database port", - default=5432, - show_default=True, - required=False - ), - ] - db_name: Annotated[ - str, click.option("--db-name", type=str, help="Db name", required=True) + click.option( + "--port", + type=int, + help="Postgres database port", + default=5432, + show_default=True, + required=False, + ), ] + db_name: Annotated[str, click.option("--db-name", type=str, help="Db name", required=True)] maintenance_work_mem: Annotated[ - Optional[str], + str | None, click.option( "--maintenance-work-mem", type=str, @@ -69,7 +69,7 @@ class PgVectorTypedDict(CommonTypedDict): ), ] max_parallel_workers: Annotated[ - Optional[int], + int | None, click.option( "--max-parallel-workers", type=int, @@ -78,7 +78,7 @@ class PgVectorTypedDict(CommonTypedDict): ), ] quantization_type: Annotated[ - Optional[str], + str | None, click.option( "--quantization-type", type=click.Choice(["none", "bit", "halfvec"]), @@ -87,7 +87,7 @@ class PgVectorTypedDict(CommonTypedDict): ), ] reranking: Annotated[ - Optional[bool], + bool | None, click.option( "--reranking/--skip-reranking", type=bool, @@ -96,11 +96,11 @@ class PgVectorTypedDict(CommonTypedDict): ), ] reranking_metric: Annotated[ - Optional[str], + str | None, click.option( "--reranking-metric", type=click.Choice( - [metric.value for metric in MetricType if metric.value not in ["HAMMING", "JACCARD"]] + [metric.value for metric in MetricType if metric.value not in ["HAMMING", "JACCARD"]], ), help="Distance metric for reranking", default="COSINE", @@ -108,7 +108,7 @@ class PgVectorTypedDict(CommonTypedDict): ), ] quantized_fetch_limit: Annotated[ - Optional[int], + int | None, click.option( "--quantized-fetch-limit", type=int, @@ -116,13 +116,11 @@ class PgVectorTypedDict(CommonTypedDict): -- bound by ef_search", required=False, callback=set_default_quantized_fetch_limit, - ) + ), ] - -class PgVectorIVFFlatTypedDict(PgVectorTypedDict, IVFFlatTypedDict): - ... +class PgVectorIVFFlatTypedDict(PgVectorTypedDict, IVFFlatTypedDict): ... @cli.command() @@ -156,8 +154,7 @@ def PgVectorIVFFlat( ) -class PgVectorHNSWTypedDict(PgVectorTypedDict, HNSWFlavor1): - ... +class PgVectorHNSWTypedDict(PgVectorTypedDict, HNSWFlavor1): ... @cli.command() diff --git a/vectordb_bench/backend/clients/pgvector/config.py b/vectordb_bench/backend/clients/pgvector/config.py index 16d547445..c386d75ef 100644 --- a/vectordb_bench/backend/clients/pgvector/config.py +++ b/vectordb_bench/backend/clients/pgvector/config.py @@ -1,7 +1,9 @@ from abc import abstractmethod -from typing import Any, Mapping, Optional, Sequence, TypedDict +from collections.abc import Mapping, Sequence +from typing import Any, LiteralString, TypedDict + from pydantic import BaseModel, SecretStr -from typing_extensions import LiteralString + from ..api import DBCaseConfig, DBConfig, IndexType, MetricType POSTGRE_URL_PLACEHOLDER = "postgresql://%s:%s@%s/%s" @@ -9,7 +11,7 @@ class PgVectorConfigDict(TypedDict): """These keys will be directly used as kwargs in psycopg connection string, - so the names must match exactly psycopg API""" + so the names must match exactly psycopg API""" user: str password: str @@ -41,8 +43,8 @@ class PgVectorIndexParam(TypedDict): metric: str index_type: str index_creation_with_options: Sequence[dict[str, Any]] - maintenance_work_mem: Optional[str] - max_parallel_workers: Optional[int] + maintenance_work_mem: str | None + max_parallel_workers: int | None class PgVectorSearchParam(TypedDict): @@ -59,61 +61,60 @@ class PgVectorIndexConfig(BaseModel, DBCaseConfig): create_index_after_load: bool = True def parse_metric(self) -> str: - if self.quantization_type == "halfvec": - if self.metric_type == MetricType.L2: - return "halfvec_l2_ops" - elif self.metric_type == MetricType.IP: - return "halfvec_ip_ops" - return "halfvec_cosine_ops" - elif self.quantization_type == "bit": - if self.metric_type == MetricType.JACCARD: - return "bit_jaccard_ops" - return "bit_hamming_ops" - else: - if self.metric_type == MetricType.L2: - return "vector_l2_ops" - elif self.metric_type == MetricType.IP: - return "vector_ip_ops" - return "vector_cosine_ops" + d = { + "halfvec": { + MetricType.L2: "halfvec_l2_ops", + MetricType.IP: "halfvec_ip_ops", + MetricType.COSINE: "halfvec_cosine_ops", + }, + "bit": { + MetricType.JACCARD: "bit_jaccard_ops", + MetricType.HAMMING: "bit_hamming_ops", + }, + "_fallback": { + MetricType.L2: "vector_l2_ops", + MetricType.IP: "vector_ip_ops", + MetricType.COSINE: "vector_cosine_ops", + }, + } + + if d.get(self.quantization_type) is None: + return d.get("_fallback").get(self.metric_type) + return d.get(self.quantization_type).get(self.metric_type) def parse_metric_fun_op(self) -> LiteralString: if self.quantization_type == "bit": if self.metric_type == MetricType.JACCARD: return "<%>" return "<~>" - else: - if self.metric_type == MetricType.L2: - return "<->" - elif self.metric_type == MetricType.IP: - return "<#>" - return "<=>" + if self.metric_type == MetricType.L2: + return "<->" + if self.metric_type == MetricType.IP: + return "<#>" + return "<=>" def parse_metric_fun_str(self) -> str: if self.metric_type == MetricType.L2: return "l2_distance" - elif self.metric_type == MetricType.IP: + if self.metric_type == MetricType.IP: return "max_inner_product" return "cosine_distance" - + def parse_reranking_metric_fun_op(self) -> LiteralString: if self.reranking_metric == MetricType.L2: return "<->" - elif self.reranking_metric == MetricType.IP: + if self.reranking_metric == MetricType.IP: return "<#>" return "<=>" - @abstractmethod - def index_param(self) -> PgVectorIndexParam: - ... + def index_param(self) -> PgVectorIndexParam: ... @abstractmethod - def search_param(self) -> PgVectorSearchParam: - ... + def search_param(self) -> PgVectorSearchParam: ... @abstractmethod - def session_param(self) -> PgVectorSessionCommands: - ... + def session_param(self) -> PgVectorSessionCommands: ... @staticmethod def _optionally_build_with_options(with_options: Mapping[str, Any]) -> Sequence[dict[str, Any]]: @@ -125,24 +126,23 @@ def _optionally_build_with_options(with_options: Mapping[str, Any]) -> Sequence[ { "option_name": option_name, "val": str(value), - } + }, ) return options @staticmethod - def _optionally_build_set_options( - set_mapping: Mapping[str, Any] - ) -> Sequence[dict[str, Any]]: + def _optionally_build_set_options(set_mapping: Mapping[str, Any]) -> Sequence[dict[str, Any]]: """Walk through options, creating 'SET 'key1 = "value1";' list""" session_options = [] for setting_name, value in set_mapping.items(): if value: session_options.append( - {"parameter": { + { + "parameter": { "setting_name": setting_name, "val": str(value), }, - } + }, ) return session_options @@ -165,12 +165,12 @@ class PgVectorIVFFlatConfig(PgVectorIndexConfig): lists: int | None probes: int | None index: IndexType = IndexType.ES_IVFFlat - maintenance_work_mem: Optional[str] = None - max_parallel_workers: Optional[int] = None - quantization_type: Optional[str] = None - reranking: Optional[bool] = None - quantized_fetch_limit: Optional[int] = None - reranking_metric: Optional[str] = None + maintenance_work_mem: str | None = None + max_parallel_workers: int | None = None + quantization_type: str | None = None + reranking: bool | None = None + quantized_fetch_limit: int | None = None + reranking_metric: str | None = None def index_param(self) -> PgVectorIndexParam: index_parameters = {"lists": self.lists} @@ -179,9 +179,7 @@ def index_param(self) -> PgVectorIndexParam: return { "metric": self.parse_metric(), "index_type": self.index.value, - "index_creation_with_options": self._optionally_build_with_options( - index_parameters - ), + "index_creation_with_options": self._optionally_build_with_options(index_parameters), "maintenance_work_mem": self.maintenance_work_mem, "max_parallel_workers": self.max_parallel_workers, "quantization_type": self.quantization_type, @@ -197,9 +195,7 @@ def search_param(self) -> PgVectorSearchParam: def session_param(self) -> PgVectorSessionCommands: session_parameters = {"ivfflat.probes": self.probes} - return { - "session_options": self._optionally_build_set_options(session_parameters) - } + return {"session_options": self._optionally_build_set_options(session_parameters)} class PgVectorHNSWConfig(PgVectorIndexConfig): @@ -210,17 +206,15 @@ class PgVectorHNSWConfig(PgVectorIndexConfig): """ m: int | None # DETAIL: Valid values are between "2" and "100". - ef_construction: ( - int | None - ) # ef_construction must be greater than or equal to 2 * m + ef_construction: int | None # ef_construction must be greater than or equal to 2 * m ef_search: int | None index: IndexType = IndexType.ES_HNSW - maintenance_work_mem: Optional[str] = None - max_parallel_workers: Optional[int] = None - quantization_type: Optional[str] = None - reranking: Optional[bool] = None - quantized_fetch_limit: Optional[int] = None - reranking_metric: Optional[str] = None + maintenance_work_mem: str | None = None + max_parallel_workers: int | None = None + quantization_type: str | None = None + reranking: bool | None = None + quantized_fetch_limit: int | None = None + reranking_metric: str | None = None def index_param(self) -> PgVectorIndexParam: index_parameters = {"m": self.m, "ef_construction": self.ef_construction} @@ -229,9 +223,7 @@ def index_param(self) -> PgVectorIndexParam: return { "metric": self.parse_metric(), "index_type": self.index.value, - "index_creation_with_options": self._optionally_build_with_options( - index_parameters - ), + "index_creation_with_options": self._optionally_build_with_options(index_parameters), "maintenance_work_mem": self.maintenance_work_mem, "max_parallel_workers": self.max_parallel_workers, "quantization_type": self.quantization_type, @@ -247,13 +239,11 @@ def search_param(self) -> PgVectorSearchParam: def session_param(self) -> PgVectorSessionCommands: session_parameters = {"hnsw.ef_search": self.ef_search} - return { - "session_options": self._optionally_build_set_options(session_parameters) - } + return {"session_options": self._optionally_build_set_options(session_parameters)} _pgvector_case_config = { - IndexType.HNSW: PgVectorHNSWConfig, - IndexType.ES_HNSW: PgVectorHNSWConfig, - IndexType.IVFFlat: PgVectorIVFFlatConfig, + IndexType.HNSW: PgVectorHNSWConfig, + IndexType.ES_HNSW: PgVectorHNSWConfig, + IndexType.IVFFlat: PgVectorIVFFlatConfig, } diff --git a/vectordb_bench/backend/clients/pgvector/pgvector.py b/vectordb_bench/backend/clients/pgvector/pgvector.py index 069b89381..62a7971bb 100644 --- a/vectordb_bench/backend/clients/pgvector/pgvector.py +++ b/vectordb_bench/backend/clients/pgvector/pgvector.py @@ -1,9 +1,9 @@ """Wrapper around the Pgvector vector database over VectorDB""" import logging -import pprint +from collections.abc import Generator, Sequence from contextlib import contextmanager -from typing import Any, Generator, Optional, Tuple, Sequence +from typing import Any import numpy as np import psycopg @@ -11,7 +11,7 @@ from psycopg import Connection, Cursor, sql from ..api import VectorDB -from .config import PgVectorConfigDict, PgVectorIndexConfig, PgVectorHNSWConfig +from .config import PgVectorConfigDict, PgVectorIndexConfig log = logging.getLogger(__name__) @@ -56,13 +56,14 @@ def __init__( ( self.case_config.create_index_before_load, self.case_config.create_index_after_load, - ) + ), ): - err = f"{self.name} config must create an index using create_index_before_load or create_index_after_load" - log.error(err) - raise RuntimeError( - f"{err}\n{pprint.pformat(self.db_config)}\n{pprint.pformat(self.case_config)}" + msg = ( + f"{self.name} config must create an index using create_index_before_load or create_index_after_load" + f"{self.name} config values: {self.db_config}\n{self.case_config}" ) + log.error(msg) + raise RuntimeError(msg) if drop_old: self._drop_index() @@ -77,7 +78,7 @@ def __init__( self.conn = None @staticmethod - def _create_connection(**kwargs) -> Tuple[Connection, Cursor]: + def _create_connection(**kwargs) -> tuple[Connection, Cursor]: conn = psycopg.connect(**kwargs) register_vector(conn) conn.autocommit = False @@ -87,8 +88,8 @@ def _create_connection(**kwargs) -> Tuple[Connection, Cursor]: assert cursor is not None, "Cursor is not initialized" return conn, cursor - - def _generate_search_query(self, filtered: bool=False) -> sql.Composed: + + def _generate_search_query(self, filtered: bool = False) -> sql.Composed: index_param = self.case_config.index_param() reranking = self.case_config.search_param()["reranking"] column_name = ( @@ -103,23 +104,25 @@ def _generate_search_query(self, filtered: bool=False) -> sql.Composed: ) # The following sections assume that the quantization_type value matches the quantization function name - if index_param["quantization_type"] != None: + if index_param["quantization_type"] is not None: if index_param["quantization_type"] == "bit" and reranking: # Embeddings needs to be passed to binary_quantize function if quantization_type is bit search_query = sql.Composed( [ sql.SQL( """ - SELECT i.id + SELECT i.id FROM ( - SELECT id, embedding {reranking_metric_fun_op} %s::vector AS distance + SELECT id, embedding {reranking_metric_fun_op} %s::vector AS distance FROM public.{table_name} {where_clause} ORDER BY {column_name}::{quantization_type}({dim}) - """ + """, ).format( table_name=sql.Identifier(self.table_name), column_name=column_name, - reranking_metric_fun_op=sql.SQL(self.case_config.search_param()["reranking_metric_fun_op"]), + reranking_metric_fun_op=sql.SQL( + self.case_config.search_param()["reranking_metric_fun_op"], + ), quantization_type=sql.SQL(index_param["quantization_type"]), dim=sql.Literal(self.dim), where_clause=sql.SQL("WHERE id >= %s") if filtered else sql.SQL(""), @@ -127,25 +130,28 @@ def _generate_search_query(self, filtered: bool=False) -> sql.Composed: sql.SQL(self.case_config.search_param()["metric_fun_op"]), sql.SQL( """ - {search_vector} + {search_vector} LIMIT {quantized_fetch_limit} ) i - ORDER BY i.distance + ORDER BY i.distance LIMIT %s::int - """ + """, ).format( search_vector=search_vector, quantized_fetch_limit=sql.Literal( - self.case_config.search_param()["quantized_fetch_limit"] + self.case_config.search_param()["quantized_fetch_limit"], ), ), - ] + ], ) else: search_query = sql.Composed( [ sql.SQL( - "SELECT id FROM public.{table_name} {where_clause} ORDER BY {column_name}::{quantization_type}({dim}) " + """ + SELECT id FROM public.{table_name} + {where_clause} ORDER BY {column_name}::{quantization_type}({dim}) + """, ).format( table_name=sql.Identifier(self.table_name), column_name=column_name, @@ -154,25 +160,26 @@ def _generate_search_query(self, filtered: bool=False) -> sql.Composed: where_clause=sql.SQL("WHERE id >= %s") if filtered else sql.SQL(""), ), sql.SQL(self.case_config.search_param()["metric_fun_op"]), - sql.SQL(" {search_vector} LIMIT %s::int").format(search_vector=search_vector), - ] + sql.SQL(" {search_vector} LIMIT %s::int").format( + search_vector=search_vector, + ), + ], ) else: search_query = sql.Composed( [ sql.SQL( - "SELECT id FROM public.{table_name} {where_clause} ORDER BY embedding " + "SELECT id FROM public.{table_name} {where_clause} ORDER BY embedding ", ).format( table_name=sql.Identifier(self.table_name), where_clause=sql.SQL("WHERE id >= %s") if filtered else sql.SQL(""), ), sql.SQL(self.case_config.search_param()["metric_fun_op"]), sql.SQL(" %s::vector LIMIT %s::int"), - ] + ], ) - + return search_query - @contextmanager def init(self) -> Generator[None, None, None]: @@ -191,8 +198,8 @@ def init(self) -> Generator[None, None, None]: if len(session_options) > 0: for setting in session_options: command = sql.SQL("SET {setting_name} " + "= {val};").format( - setting_name=sql.Identifier(setting['parameter']['setting_name']), - val=sql.Identifier(str(setting['parameter']['val'])), + setting_name=sql.Identifier(setting["parameter"]["setting_name"]), + val=sql.Identifier(str(setting["parameter"]["val"])), ) log.debug(command.as_string(self.cursor)) self.cursor.execute(command) @@ -216,8 +223,8 @@ def _drop_table(self): self.cursor.execute( sql.SQL("DROP TABLE IF EXISTS public.{table_name}").format( - table_name=sql.Identifier(self.table_name) - ) + table_name=sql.Identifier(self.table_name), + ), ) self.conn.commit() @@ -239,7 +246,7 @@ def _drop_index(self): log.info(f"{self.name} client drop index : {self._index_name}") drop_index_sql = sql.SQL("DROP INDEX IF EXISTS {index_name}").format( - index_name=sql.Identifier(self._index_name) + index_name=sql.Identifier(self._index_name), ) log.debug(drop_index_sql.as_string(self.cursor)) self.cursor.execute(drop_index_sql) @@ -254,63 +261,51 @@ def _set_parallel_index_build_param(self): if index_param["maintenance_work_mem"] is not None: self.cursor.execute( sql.SQL("SET maintenance_work_mem TO {};").format( - index_param["maintenance_work_mem"] - ) + index_param["maintenance_work_mem"], + ), ) self.cursor.execute( sql.SQL("ALTER USER {} SET maintenance_work_mem TO {};").format( sql.Identifier(self.db_config["user"]), index_param["maintenance_work_mem"], - ) + ), ) self.conn.commit() if index_param["max_parallel_workers"] is not None: self.cursor.execute( sql.SQL("SET max_parallel_maintenance_workers TO '{}';").format( - index_param["max_parallel_workers"] - ) + index_param["max_parallel_workers"], + ), ) self.cursor.execute( - sql.SQL( - "ALTER USER {} SET max_parallel_maintenance_workers TO '{}';" - ).format( + sql.SQL("ALTER USER {} SET max_parallel_maintenance_workers TO '{}';").format( sql.Identifier(self.db_config["user"]), index_param["max_parallel_workers"], - ) + ), ) self.cursor.execute( sql.SQL("SET max_parallel_workers TO '{}';").format( - index_param["max_parallel_workers"] - ) + index_param["max_parallel_workers"], + ), ) self.cursor.execute( - sql.SQL( - "ALTER USER {} SET max_parallel_workers TO '{}';" - ).format( + sql.SQL("ALTER USER {} SET max_parallel_workers TO '{}';").format( sql.Identifier(self.db_config["user"]), index_param["max_parallel_workers"], - ) + ), ) self.cursor.execute( - sql.SQL( - "ALTER TABLE {} SET (parallel_workers = {});" - ).format( + sql.SQL("ALTER TABLE {} SET (parallel_workers = {});").format( sql.Identifier(self.table_name), index_param["max_parallel_workers"], - ) + ), ) self.conn.commit() - results = self.cursor.execute( - sql.SQL("SHOW max_parallel_maintenance_workers;") - ).fetchall() - results.extend( - self.cursor.execute(sql.SQL("SHOW max_parallel_workers;")).fetchall() - ) - results.extend( - self.cursor.execute(sql.SQL("SHOW maintenance_work_mem;")).fetchall() - ) + results = self.cursor.execute(sql.SQL("SHOW max_parallel_maintenance_workers;")).fetchall() + results.extend(self.cursor.execute(sql.SQL("SHOW max_parallel_workers;")).fetchall()) + results.extend(self.cursor.execute(sql.SQL("SHOW maintenance_work_mem;")).fetchall()) log.info(f"{self.name} parallel index creation parameters: {results}") def _create_index(self): @@ -322,24 +317,21 @@ def _create_index(self): self._set_parallel_index_build_param() options = [] for option in index_param["index_creation_with_options"]: - if option['val'] is not None: + if option["val"] is not None: options.append( sql.SQL("{option_name} = {val}").format( - option_name=sql.Identifier(option['option_name']), - val=sql.Identifier(str(option['val'])), - ) + option_name=sql.Identifier(option["option_name"]), + val=sql.Identifier(str(option["val"])), + ), ) - if any(options): - with_clause = sql.SQL("WITH ({});").format(sql.SQL(", ").join(options)) - else: - with_clause = sql.Composed(()) + with_clause = sql.SQL("WITH ({});").format(sql.SQL(", ").join(options)) if any(options) else sql.Composed(()) - if index_param["quantization_type"] != None: + if index_param["quantization_type"] is not None: index_create_sql = sql.SQL( """ CREATE INDEX IF NOT EXISTS {index_name} ON public.{table_name} USING {index_type} (({column_name}::{quantization_type}({dim})) {embedding_metric}) - """ + """, ).format( index_name=sql.Identifier(self._index_name), table_name=sql.Identifier(self.table_name), @@ -357,9 +349,9 @@ def _create_index(self): else: index_create_sql = sql.SQL( """ - CREATE INDEX IF NOT EXISTS {index_name} ON public.{table_name} + CREATE INDEX IF NOT EXISTS {index_name} ON public.{table_name} USING {index_type} (embedding {embedding_metric}) - """ + """, ).format( index_name=sql.Identifier(self._index_name), table_name=sql.Identifier(self.table_name), @@ -367,9 +359,7 @@ def _create_index(self): embedding_metric=sql.Identifier(index_param["metric"]), ) - index_create_sql_with_with_clause = ( - index_create_sql + with_clause - ).join(" ") + index_create_sql_with_with_clause = (index_create_sql + with_clause).join(" ") log.debug(index_create_sql_with_with_clause.as_string(self.cursor)) self.cursor.execute(index_create_sql_with_with_clause) self.conn.commit() @@ -384,19 +374,17 @@ def _create_table(self, dim: int): # create table self.cursor.execute( sql.SQL( - "CREATE TABLE IF NOT EXISTS public.{table_name} (id BIGINT PRIMARY KEY, embedding vector({dim}));" - ).format(table_name=sql.Identifier(self.table_name), dim=dim) + "CREATE TABLE IF NOT EXISTS public.{table_name} (id BIGINT PRIMARY KEY, embedding vector({dim}));", + ).format(table_name=sql.Identifier(self.table_name), dim=dim), ) self.cursor.execute( sql.SQL( - "ALTER TABLE public.{table_name} ALTER COLUMN embedding SET STORAGE PLAIN;" - ).format(table_name=sql.Identifier(self.table_name)) + "ALTER TABLE public.{table_name} ALTER COLUMN embedding SET STORAGE PLAIN;", + ).format(table_name=sql.Identifier(self.table_name)), ) self.conn.commit() except Exception as e: - log.warning( - f"Failed to create pgvector table: {self.table_name} error: {e}" - ) + log.warning(f"Failed to create pgvector table: {self.table_name} error: {e}") raise e from None def insert_embeddings( @@ -404,7 +392,7 @@ def insert_embeddings( embeddings: list[list[float]], metadata: list[int], **kwargs: Any, - ) -> Tuple[int, Optional[Exception]]: + ) -> tuple[int, Exception | None]: assert self.conn is not None, "Connection is not initialized" assert self.cursor is not None, "Cursor is not initialized" @@ -414,8 +402,8 @@ def insert_embeddings( with self.cursor.copy( sql.SQL("COPY public.{table_name} FROM STDIN (FORMAT BINARY)").format( - table_name=sql.Identifier(self.table_name) - ) + table_name=sql.Identifier(self.table_name), + ), ) as copy: copy.set_types(["bigint", "vector"]) for i, row in enumerate(metadata_arr): @@ -428,7 +416,7 @@ def insert_embeddings( return len(metadata), None except Exception as e: log.warning( - f"Failed to insert data into pgvector table ({self.table_name}), error: {e}" + f"Failed to insert data into pgvector table ({self.table_name}), error: {e}", ) return 0, e @@ -449,21 +437,32 @@ def search_embedding( gt = filters.get("id") if index_param["quantization_type"] == "bit" and search_param["reranking"]: result = self.cursor.execute( - self._filtered_search, (q, gt, q, k), prepare=True, binary=True + self._filtered_search, + (q, gt, q, k), + prepare=True, + binary=True, ) else: result = self.cursor.execute( - self._filtered_search, (gt, q, k), prepare=True, binary=True + self._filtered_search, + (gt, q, k), + prepare=True, + binary=True, ) - + + elif index_param["quantization_type"] == "bit" and search_param["reranking"]: + result = self.cursor.execute( + self._unfiltered_search, + (q, q, k), + prepare=True, + binary=True, + ) else: - if index_param["quantization_type"] == "bit" and search_param["reranking"]: - result = self.cursor.execute( - self._unfiltered_search, (q, q, k), prepare=True, binary=True - ) - else: - result = self.cursor.execute( - self._unfiltered_search, (q, k), prepare=True, binary=True - ) + result = self.cursor.execute( + self._unfiltered_search, + (q, k), + prepare=True, + binary=True, + ) return [int(i[0]) for i in result.fetchall()] diff --git a/vectordb_bench/backend/clients/pgvectorscale/cli.py b/vectordb_bench/backend/clients/pgvectorscale/cli.py index e5a161c6b..fca9d51d8 100644 --- a/vectordb_bench/backend/clients/pgvectorscale/cli.py +++ b/vectordb_bench/backend/clients/pgvectorscale/cli.py @@ -1,80 +1,94 @@ -import click import os +from typing import Annotated, Unpack + +import click from pydantic import SecretStr +from vectordb_bench.backend.clients import DB + from ....cli.cli import ( CommonTypedDict, cli, click_parameter_decorators_from_typed_dict, run, ) -from typing import Annotated, Unpack -from vectordb_bench.backend.clients import DB class PgVectorScaleTypedDict(CommonTypedDict): user_name: Annotated[ - str, click.option("--user-name", type=str, help="Db username", required=True) + str, + click.option("--user-name", type=str, help="Db username", required=True), ] password: Annotated[ str, - click.option("--password", - type=str, - help="Postgres database password", - default=lambda: os.environ.get("POSTGRES_PASSWORD", ""), - show_default="$POSTGRES_PASSWORD", - ), + click.option( + "--password", + type=str, + help="Postgres database password", + default=lambda: os.environ.get("POSTGRES_PASSWORD", ""), + show_default="$POSTGRES_PASSWORD", + ), ] - host: Annotated[ - str, click.option("--host", type=str, help="Db host", required=True) - ] - db_name: Annotated[ - str, click.option("--db-name", type=str, help="Db name", required=True) - ] + host: Annotated[str, click.option("--host", type=str, help="Db host", required=True)] + db_name: Annotated[str, click.option("--db-name", type=str, help="Db name", required=True)] class PgVectorScaleDiskAnnTypedDict(PgVectorScaleTypedDict): storage_layout: Annotated[ str, click.option( - "--storage-layout", type=str, help="Streaming DiskANN storage layout", + "--storage-layout", + type=str, + help="Streaming DiskANN storage layout", ), ] num_neighbors: Annotated[ int, click.option( - "--num-neighbors", type=int, help="Streaming DiskANN num neighbors", + "--num-neighbors", + type=int, + help="Streaming DiskANN num neighbors", ), ] search_list_size: Annotated[ int, click.option( - "--search-list-size", type=int, help="Streaming DiskANN search list size", + "--search-list-size", + type=int, + help="Streaming DiskANN search list size", ), ] max_alpha: Annotated[ float, click.option( - "--max-alpha", type=float, help="Streaming DiskANN max alpha", + "--max-alpha", + type=float, + help="Streaming DiskANN max alpha", ), ] num_dimensions: Annotated[ int, click.option( - "--num-dimensions", type=int, help="Streaming DiskANN num dimensions", + "--num-dimensions", + type=int, + help="Streaming DiskANN num dimensions", ), ] query_search_list_size: Annotated[ int, click.option( - "--query-search-list-size", type=int, help="Streaming DiskANN query search list size", + "--query-search-list-size", + type=int, + help="Streaming DiskANN query search list size", ), ] query_rescore: Annotated[ int, click.option( - "--query-rescore", type=int, help="Streaming DiskANN query rescore", + "--query-rescore", + type=int, + help="Streaming DiskANN query rescore", ), ] @@ -105,4 +119,4 @@ def PgVectorScaleDiskAnn( query_rescore=parameters["query_rescore"], ), **parameters, - ) \ No newline at end of file + ) diff --git a/vectordb_bench/backend/clients/pgvectorscale/config.py b/vectordb_bench/backend/clients/pgvectorscale/config.py index bd9f6106b..e22c45c8d 100644 --- a/vectordb_bench/backend/clients/pgvectorscale/config.py +++ b/vectordb_bench/backend/clients/pgvectorscale/config.py @@ -1,7 +1,8 @@ from abc import abstractmethod -from typing import TypedDict +from typing import LiteralString, TypedDict + from pydantic import BaseModel, SecretStr -from typing_extensions import LiteralString + from ..api import DBCaseConfig, DBConfig, IndexType, MetricType POSTGRE_URL_PLACEHOLDER = "postgresql://%s:%s@%s/%s" @@ -9,7 +10,7 @@ class PgVectorScaleConfigDict(TypedDict): """These keys will be directly used as kwargs in psycopg connection string, - so the names must match exactly psycopg API""" + so the names must match exactly psycopg API""" user: str password: str @@ -46,7 +47,7 @@ def parse_metric(self) -> str: if self.metric_type == MetricType.COSINE: return "vector_cosine_ops" return "" - + def parse_metric_fun_op(self) -> LiteralString: if self.metric_type == MetricType.COSINE: return "<=>" @@ -56,19 +57,16 @@ def parse_metric_fun_str(self) -> str: if self.metric_type == MetricType.COSINE: return "cosine_distance" return "" - + @abstractmethod - def index_param(self) -> dict: - ... + def index_param(self) -> dict: ... @abstractmethod - def search_param(self) -> dict: - ... + def search_param(self) -> dict: ... @abstractmethod - def session_param(self) -> dict: - ... - + def session_param(self) -> dict: ... + class PgVectorScaleStreamingDiskANNConfig(PgVectorScaleIndexConfig): index: IndexType = IndexType.STREAMING_DISKANN @@ -93,19 +91,20 @@ def index_param(self) -> dict: "num_dimensions": self.num_dimensions, }, } - + def search_param(self) -> dict: return { "metric": self.parse_metric(), "metric_fun_op": self.parse_metric_fun_op(), } - + def session_param(self) -> dict: return { "diskann.query_search_list_size": self.query_search_list_size, "diskann.query_rescore": self.query_rescore, } - + + _pgvectorscale_case_config = { IndexType.STREAMING_DISKANN: PgVectorScaleStreamingDiskANNConfig, } diff --git a/vectordb_bench/backend/clients/pgvectorscale/pgvectorscale.py b/vectordb_bench/backend/clients/pgvectorscale/pgvectorscale.py index d8f26394c..981accc2e 100644 --- a/vectordb_bench/backend/clients/pgvectorscale/pgvectorscale.py +++ b/vectordb_bench/backend/clients/pgvectorscale/pgvectorscale.py @@ -1,9 +1,9 @@ """Wrapper around the Pgvectorscale vector database over VectorDB""" import logging -import pprint +from collections.abc import Generator from contextlib import contextmanager -from typing import Any, Generator, Optional, Tuple +from typing import Any import numpy as np import psycopg @@ -44,20 +44,21 @@ def __init__( self._primary_field = "id" self._vector_field = "embedding" - self.conn, self.cursor = self._create_connection(**self.db_config) + self.conn, self.cursor = self._create_connection(**self.db_config) log.info(f"{self.name} config values: {self.db_config}\n{self.case_config}") if not any( ( self.case_config.create_index_before_load, self.case_config.create_index_after_load, - ) + ), ): - err = f"{self.name} config must create an index using create_index_before_load or create_index_after_load" - log.error(err) - raise RuntimeError( - f"{err}\n{pprint.pformat(self.db_config)}\n{pprint.pformat(self.case_config)}" + msg = ( + f"{self.name} config must create an index using create_index_before_load or create_index_after_load" + f"{self.name} config values: {self.db_config}\n{self.case_config}" ) + log.error(msg) + raise RuntimeError(msg) if drop_old: self._drop_index() @@ -72,7 +73,7 @@ def __init__( self.conn = None @staticmethod - def _create_connection(**kwargs) -> Tuple[Connection, Cursor]: + def _create_connection(**kwargs) -> tuple[Connection, Cursor]: conn = psycopg.connect(**kwargs) conn.cursor().execute("CREATE EXTENSION IF NOT EXISTS vectorscale CASCADE") conn.commit() @@ -101,25 +102,25 @@ def init(self) -> Generator[None, None, None]: log.debug(command.as_string(self.cursor)) self.cursor.execute(command) self.conn.commit() - + self._filtered_search = sql.Composed( [ sql.SQL("SELECT id FROM public.{} WHERE id >= %s ORDER BY embedding ").format( sql.Identifier(self.table_name), ), sql.SQL(self.case_config.search_param()["metric_fun_op"]), - sql.SQL(" %s::vector LIMIT %s::int") - ] + sql.SQL(" %s::vector LIMIT %s::int"), + ], ) - + self._unfiltered_search = sql.Composed( [ sql.SQL("SELECT id FROM public.{} ORDER BY embedding ").format( - sql.Identifier(self.table_name) + sql.Identifier(self.table_name), ), sql.SQL(self.case_config.search_param()["metric_fun_op"]), sql.SQL(" %s::vector LIMIT %s::int"), - ] + ], ) try: @@ -137,8 +138,8 @@ def _drop_table(self): self.cursor.execute( sql.SQL("DROP TABLE IF EXISTS public.{table_name}").format( - table_name=sql.Identifier(self.table_name) - ) + table_name=sql.Identifier(self.table_name), + ), ) self.conn.commit() @@ -160,7 +161,7 @@ def _drop_index(self): log.info(f"{self.name} client drop index : {self._index_name}") drop_index_sql = sql.SQL("DROP INDEX IF EXISTS {index_name}").format( - index_name=sql.Identifier(self._index_name) + index_name=sql.Identifier(self._index_name), ) log.debug(drop_index_sql.as_string(self.cursor)) self.cursor.execute(drop_index_sql) @@ -180,36 +181,31 @@ def _create_index(self): sql.SQL("{option_name} = {val}").format( option_name=sql.Identifier(option_name), val=sql.Identifier(str(option_val)), - ) + ), ) - + num_bits_per_dimension = "2" if self.dim < 900 else "1" options.append( sql.SQL("{option_name} = {val}").format( option_name=sql.Identifier("num_bits_per_dimension"), val=sql.Identifier(num_bits_per_dimension), - ) + ), ) - if any(options): - with_clause = sql.SQL("WITH ({});").format(sql.SQL(", ").join(options)) - else: - with_clause = sql.Composed(()) + with_clause = sql.SQL("WITH ({});").format(sql.SQL(", ").join(options)) if any(options) else sql.Composed(()) index_create_sql = sql.SQL( """ - CREATE INDEX IF NOT EXISTS {index_name} ON public.{table_name} + CREATE INDEX IF NOT EXISTS {index_name} ON public.{table_name} USING {index_type} (embedding {embedding_metric}) - """ + """, ).format( index_name=sql.Identifier(self._index_name), table_name=sql.Identifier(self.table_name), index_type=sql.Identifier(index_param["index_type"].lower()), embedding_metric=sql.Identifier(index_param["metric"]), ) - index_create_sql_with_with_clause = ( - index_create_sql + with_clause - ).join(" ") + index_create_sql_with_with_clause = (index_create_sql + with_clause).join(" ") log.debug(index_create_sql_with_with_clause.as_string(self.cursor)) self.cursor.execute(index_create_sql_with_with_clause) self.conn.commit() @@ -223,14 +219,12 @@ def _create_table(self, dim: int): self.cursor.execute( sql.SQL( - "CREATE TABLE IF NOT EXISTS public.{table_name} (id BIGINT PRIMARY KEY, embedding vector({dim}));" - ).format(table_name=sql.Identifier(self.table_name), dim=dim) + "CREATE TABLE IF NOT EXISTS public.{table_name} (id BIGINT PRIMARY KEY, embedding vector({dim}));", + ).format(table_name=sql.Identifier(self.table_name), dim=dim), ) self.conn.commit() except Exception as e: - log.warning( - f"Failed to create pgvectorscale table: {self.table_name} error: {e}" - ) + log.warning(f"Failed to create pgvectorscale table: {self.table_name} error: {e}") raise e from None def insert_embeddings( @@ -238,7 +232,7 @@ def insert_embeddings( embeddings: list[list[float]], metadata: list[int], **kwargs: Any, - ) -> Tuple[int, Optional[Exception]]: + ) -> tuple[int, Exception | None]: assert self.conn is not None, "Connection is not initialized" assert self.cursor is not None, "Cursor is not initialized" @@ -248,8 +242,8 @@ def insert_embeddings( with self.cursor.copy( sql.SQL("COPY public.{table_name} FROM STDIN (FORMAT BINARY)").format( - table_name=sql.Identifier(self.table_name) - ) + table_name=sql.Identifier(self.table_name), + ), ) as copy: copy.set_types(["bigint", "vector"]) for i, row in enumerate(metadata_arr): @@ -262,7 +256,7 @@ def insert_embeddings( return len(metadata), None except Exception as e: log.warning( - f"Failed to insert data into pgvector table ({self.table_name}), error: {e}" + f"Failed to insert data into pgvector table ({self.table_name}), error: {e}", ) return 0, e @@ -280,11 +274,12 @@ def search_embedding( if filters: gt = filters.get("id") result = self.cursor.execute( - self._filtered_search, (gt, q, k), prepare=True, binary=True + self._filtered_search, + (gt, q, k), + prepare=True, + binary=True, ) else: - result = self.cursor.execute( - self._unfiltered_search, (q, k), prepare=True, binary=True - ) + result = self.cursor.execute(self._unfiltered_search, (q, k), prepare=True, binary=True) return [int(i[0]) for i in result.fetchall()] diff --git a/vectordb_bench/backend/clients/pinecone/config.py b/vectordb_bench/backend/clients/pinecone/config.py index 2bbcbb350..fe1a039ed 100644 --- a/vectordb_bench/backend/clients/pinecone/config.py +++ b/vectordb_bench/backend/clients/pinecone/config.py @@ -1,4 +1,5 @@ from pydantic import SecretStr + from ..api import DBConfig diff --git a/vectordb_bench/backend/clients/pinecone/pinecone.py b/vectordb_bench/backend/clients/pinecone/pinecone.py index c1351f7a9..c59ee8760 100644 --- a/vectordb_bench/backend/clients/pinecone/pinecone.py +++ b/vectordb_bench/backend/clients/pinecone/pinecone.py @@ -2,11 +2,11 @@ import logging from contextlib import contextmanager -from typing import Type + import pinecone -from ..api import VectorDB, DBConfig, DBCaseConfig, EmptyDBCaseConfig, IndexType -from .config import PineconeConfig +from ..api import DBCaseConfig, DBConfig, EmptyDBCaseConfig, IndexType, VectorDB +from .config import PineconeConfig log = logging.getLogger(__name__) @@ -17,7 +17,7 @@ class Pinecone(VectorDB): def __init__( self, - dim, + dim: int, db_config: dict, db_case_config: DBCaseConfig, drop_old: bool = False, @@ -27,7 +27,7 @@ def __init__( self.index_name = db_config.get("index_name", "") self.api_key = db_config.get("api_key", "") self.batch_size = int( - min(PINECONE_MAX_SIZE_PER_BATCH / (dim * 5), PINECONE_MAX_NUM_PER_BATCH) + min(PINECONE_MAX_SIZE_PER_BATCH / (dim * 5), PINECONE_MAX_NUM_PER_BATCH), ) pc = pinecone.Pinecone(api_key=self.api_key) @@ -37,9 +37,8 @@ def __init__( index_stats = index.describe_index_stats() index_dim = index_stats["dimension"] if index_dim != dim: - raise ValueError( - f"Pinecone index {self.index_name} dimension mismatch, expected {index_dim} got {dim}" - ) + msg = f"Pinecone index {self.index_name} dimension mismatch, expected {index_dim} got {dim}" + raise ValueError(msg) for namespace in index_stats["namespaces"]: log.info(f"Pinecone index delete namespace: {namespace}") index.delete(delete_all=True, namespace=namespace) @@ -47,11 +46,11 @@ def __init__( self._metadata_key = "meta" @classmethod - def config_cls(cls) -> Type[DBConfig]: + def config_cls(cls) -> type[DBConfig]: return PineconeConfig @classmethod - def case_config_cls(cls, index_type: IndexType | None = None) -> Type[DBCaseConfig]: + def case_config_cls(cls, index_type: IndexType | None = None) -> type[DBCaseConfig]: return EmptyDBCaseConfig @contextmanager @@ -76,9 +75,7 @@ def insert_embeddings( insert_count = 0 try: for batch_start_offset in range(0, len(embeddings), self.batch_size): - batch_end_offset = min( - batch_start_offset + self.batch_size, len(embeddings) - ) + batch_end_offset = min(batch_start_offset + self.batch_size, len(embeddings)) insert_datas = [] for i in range(batch_start_offset, batch_end_offset): insert_data = ( @@ -100,10 +97,7 @@ def search_embedding( filters: dict | None = None, timeout: int | None = None, ) -> list[int]: - if filters is None: - pinecone_filters = {} - else: - pinecone_filters = {self._metadata_key: {"$gte": filters["id"]}} + pinecone_filters = {} if filters is None else {self._metadata_key: {"$gte": filters["id"]}} try: res = self.index.query( top_k=k, @@ -111,7 +105,6 @@ def search_embedding( filter=pinecone_filters, )["matches"] except Exception as e: - print(f"Error querying index: {e}") - raise e - id_res = [int(one_res["id"]) for one_res in res] - return id_res + log.warning(f"Error querying index: {e}") + raise e from e + return [int(one_res["id"]) for one_res in res] diff --git a/vectordb_bench/backend/clients/qdrant_cloud/config.py b/vectordb_bench/backend/clients/qdrant_cloud/config.py index 5b1dd7f18..c1d6882c0 100644 --- a/vectordb_bench/backend/clients/qdrant_cloud/config.py +++ b/vectordb_bench/backend/clients/qdrant_cloud/config.py @@ -1,7 +1,7 @@ -from pydantic import BaseModel, SecretStr +from pydantic import BaseModel, SecretStr, validator + +from ..api import DBCaseConfig, DBConfig, MetricType -from ..api import DBConfig, DBCaseConfig, MetricType -from pydantic import validator # Allowing `api_key` to be left empty, to ensure compatibility with the open-source Qdrant. class QdrantConfig(DBConfig): @@ -16,17 +16,19 @@ def to_dict(self) -> dict: "api_key": self.api_key.get_secret_value(), "prefer_grpc": True, } - else: - return {"url": self.url.get_secret_value(),} - + return { + "url": self.url.get_secret_value(), + } + @validator("*") - def not_empty_field(cls, v, field): + def not_empty_field(cls, v: any, field: any): if field.name in ["api_key", "db_label"]: return v - if isinstance(v, (str, SecretStr)) and len(v) == 0: + if isinstance(v, str | SecretStr) and len(v) == 0: raise ValueError("Empty string!") return v + class QdrantIndexConfig(BaseModel, DBCaseConfig): metric_type: MetricType | None = None @@ -40,8 +42,7 @@ def parse_metric(self) -> str: return "Cosine" def index_param(self) -> dict: - params = {"distance": self.parse_metric()} - return params + return {"distance": self.parse_metric()} def search_param(self) -> dict: return {} diff --git a/vectordb_bench/backend/clients/qdrant_cloud/qdrant_cloud.py b/vectordb_bench/backend/clients/qdrant_cloud/qdrant_cloud.py index a51632bc6..0861e8938 100644 --- a/vectordb_bench/backend/clients/qdrant_cloud/qdrant_cloud.py +++ b/vectordb_bench/backend/clients/qdrant_cloud/qdrant_cloud.py @@ -4,23 +4,26 @@ import time from contextlib import contextmanager -from ..api import VectorDB, DBCaseConfig +from qdrant_client import QdrantClient from qdrant_client.http.models import ( - CollectionStatus, - VectorParams, - PayloadSchemaType, Batch, - Filter, + CollectionStatus, FieldCondition, + Filter, + PayloadSchemaType, Range, + VectorParams, ) -from qdrant_client import QdrantClient - +from ..api import DBCaseConfig, VectorDB log = logging.getLogger(__name__) +SECONDS_WAITING_FOR_INDEXING_API_CALL = 5 +QDRANT_BATCH_SIZE = 500 + + class QdrantCloud(VectorDB): def __init__( self, @@ -57,16 +60,14 @@ def init(self) -> None: self.qdrant_client = QdrantClient(**self.db_config) yield self.qdrant_client = None - del(self.qdrant_client) + del self.qdrant_client def ready_to_load(self): pass - def optimize(self): assert self.qdrant_client, "Please call self.init() before" # wait for vectors to be fully indexed - SECONDS_WAITING_FOR_INDEXING_API_CALL = 5 try: while True: info = self.qdrant_client.get_collection(self.collection_name) @@ -74,19 +75,26 @@ def optimize(self): if info.status != CollectionStatus.GREEN: continue if info.status == CollectionStatus.GREEN: - log.info(f"Stored vectors: {info.vectors_count}, Indexed vectors: {info.indexed_vectors_count}, Collection status: {info.indexed_vectors_count}") + msg = ( + f"Stored vectors: {info.vectors_count}, Indexed vectors: {info.indexed_vectors_count}, ", + f"Collection status: {info.indexed_vectors_count}", + ) + log.info(msg) return except Exception as e: log.warning(f"QdrantCloud ready to search error: {e}") raise e from None - def _create_collection(self, dim, qdrant_client: int): + def _create_collection(self, dim: int, qdrant_client: QdrantClient): log.info(f"Create collection: {self.collection_name}") try: qdrant_client.create_collection( collection_name=self.collection_name, - vectors_config=VectorParams(size=dim, distance=self.case_config.index_param()["distance"]) + vectors_config=VectorParams( + size=dim, + distance=self.case_config.index_param()["distance"], + ), ) qdrant_client.create_payload_index( @@ -109,13 +117,12 @@ def insert_embeddings( ) -> (int, Exception): """Insert embeddings into Milvus. should call self.init() first""" assert self.qdrant_client is not None - QDRANT_BATCH_SIZE = 500 try: # TODO: counts for offset in range(0, len(embeddings), QDRANT_BATCH_SIZE): - vectors = embeddings[offset: offset + QDRANT_BATCH_SIZE] - ids = metadata[offset: offset + QDRANT_BATCH_SIZE] - payloads=[{self._primary_field: v} for v in ids] + vectors = embeddings[offset : offset + QDRANT_BATCH_SIZE] + ids = metadata[offset : offset + QDRANT_BATCH_SIZE] + payloads = [{self._primary_field: v} for v in ids] _ = self.qdrant_client.upsert( collection_name=self.collection_name, wait=True, @@ -142,21 +149,23 @@ def search_embedding( f = None if filters: f = Filter( - must=[FieldCondition( - key = self._primary_field, - range = Range( - gt=filters.get('id'), + must=[ + FieldCondition( + key=self._primary_field, + range=Range( + gt=filters.get("id"), + ), ), - )] + ], ) - res = self.qdrant_client.search( - collection_name=self.collection_name, - query_vector=query, - limit=k, - query_filter=f, - # with_payload=True, - ), + res = ( + self.qdrant_client.search( + collection_name=self.collection_name, + query_vector=query, + limit=k, + query_filter=f, + ), + ) - ret = [result.id for result in res[0]] - return ret + return [result.id for result in res[0]] diff --git a/vectordb_bench/backend/clients/redis/cli.py b/vectordb_bench/backend/clients/redis/cli.py index eb86b4c00..69277c00f 100644 --- a/vectordb_bench/backend/clients/redis/cli.py +++ b/vectordb_bench/backend/clients/redis/cli.py @@ -3,9 +3,6 @@ import click from pydantic import SecretStr -from .config import RedisHNSWConfig - - from ....cli.cli import ( CommonTypedDict, HNSWFlavor2, @@ -14,12 +11,11 @@ run, ) from .. import DB +from .config import RedisHNSWConfig class RedisTypedDict(TypedDict): - host: Annotated[ - str, click.option("--host", type=str, help="Db host", required=True) - ] + host: Annotated[str, click.option("--host", type=str, help="Db host", required=True)] password: Annotated[str, click.option("--password", type=str, help="Db password")] port: Annotated[int, click.option("--port", type=int, default=6379, help="Db Port")] ssl: Annotated[ @@ -52,27 +48,25 @@ class RedisTypedDict(TypedDict): ] -class RedisHNSWTypedDict(CommonTypedDict, RedisTypedDict, HNSWFlavor2): - ... +class RedisHNSWTypedDict(CommonTypedDict, RedisTypedDict, HNSWFlavor2): ... @cli.command() @click_parameter_decorators_from_typed_dict(RedisHNSWTypedDict) def Redis(**parameters: Unpack[RedisHNSWTypedDict]): from .config import RedisConfig + run( db=DB.Redis, db_config=RedisConfig( db_label=parameters["db_label"], - password=SecretStr(parameters["password"]) - if parameters["password"] - else None, + password=SecretStr(parameters["password"]) if parameters["password"] else None, host=SecretStr(parameters["host"]), port=parameters["port"], ssl=parameters["ssl"], ssl_ca_certs=parameters["ssl_ca_certs"], cmd=parameters["cmd"], - ), + ), db_case_config=RedisHNSWConfig( M=parameters["m"], efConstruction=parameters["ef_construction"], diff --git a/vectordb_bench/backend/clients/redis/config.py b/vectordb_bench/backend/clients/redis/config.py index 55a7a4159..db0127f84 100644 --- a/vectordb_bench/backend/clients/redis/config.py +++ b/vectordb_bench/backend/clients/redis/config.py @@ -1,10 +1,12 @@ -from pydantic import SecretStr, BaseModel -from ..api import DBConfig, DBCaseConfig, MetricType, IndexType +from pydantic import BaseModel, SecretStr + +from ..api import DBCaseConfig, DBConfig, IndexType, MetricType + class RedisConfig(DBConfig): password: SecretStr | None = None host: SecretStr - port: int | None = None + port: int | None = None def to_dict(self) -> dict: return { @@ -12,7 +14,6 @@ def to_dict(self) -> dict: "port": self.port, "password": self.password.get_secret_value() if self.password is not None else None, } - class RedisIndexConfig(BaseModel): @@ -24,7 +25,8 @@ def parse_metric(self) -> str: if not self.metric_type: return "" return self.metric_type.value - + + class RedisHNSWConfig(RedisIndexConfig, DBCaseConfig): M: int efConstruction: int diff --git a/vectordb_bench/backend/clients/redis/redis.py b/vectordb_bench/backend/clients/redis/redis.py index cf51fcc48..139850d2f 100644 --- a/vectordb_bench/backend/clients/redis/redis.py +++ b/vectordb_bench/backend/clients/redis/redis.py @@ -1,36 +1,40 @@ import logging from contextlib import contextmanager from typing import Any -from ..api import VectorDB, DBCaseConfig + +import numpy as np import redis -from redis.commands.search.field import TagField, VectorField, NumericField +from redis.commands.search.field import NumericField, TagField, VectorField from redis.commands.search.indexDefinition import IndexDefinition, IndexType from redis.commands.search.query import Query -import numpy as np +from ..api import DBCaseConfig, VectorDB log = logging.getLogger(__name__) -INDEX_NAME = "index" # Vector Index Name +INDEX_NAME = "index" # Vector Index Name + class Redis(VectorDB): def __init__( - self, - dim: int, - db_config: dict, - db_case_config: DBCaseConfig, - drop_old: bool = False, - - **kwargs - ): - + self, + dim: int, + db_config: dict, + db_case_config: DBCaseConfig, + drop_old: bool = False, + **kwargs, + ): self.db_config = db_config self.case_config = db_case_config self.collection_name = INDEX_NAME # Create a redis connection, if db has password configured, add it to the connection here and in init(): - password=self.db_config["password"] - conn = redis.Redis(host=self.db_config["host"], port=self.db_config["port"], password=password, db=0) - + password = self.db_config["password"] + conn = redis.Redis( + host=self.db_config["host"], + port=self.db_config["port"], + password=password, + db=0, + ) if drop_old: try: @@ -39,7 +43,7 @@ def __init__( except redis.exceptions.ResponseError: drop_old = False log.info(f"Redis client drop_old collection: {self.collection_name}") - + self.make_index(dim, conn) conn.close() conn = None @@ -48,18 +52,20 @@ def make_index(self, vector_dimensions: int, conn: redis.Redis): try: # check to see if index exists conn.ft(INDEX_NAME).info() - except: + except Exception: schema = ( - TagField("id"), - NumericField("metadata"), - VectorField("vector", # Vector Field Name - "HNSW", { # Vector Index Type: FLAT or HNSW - "TYPE": "FLOAT32", # FLOAT32 or FLOAT64 - "DIM": vector_dimensions, # Number of Vector Dimensions - "DISTANCE_METRIC": "COSINE", # Vector Search Distance Metric + TagField("id"), + NumericField("metadata"), + VectorField( + "vector", # Vector Field Name + "HNSW", # Vector Index Type: FLAT or HNSW + { + "TYPE": "FLOAT32", # FLOAT32 or FLOAT64 + "DIM": vector_dimensions, # Number of Vector Dimensions + "DISTANCE_METRIC": "COSINE", # Vector Search Distance Metric "M": self.case_config.index_param()["params"]["M"], "EF_CONSTRUCTION": self.case_config.index_param()["params"]["efConstruction"], - } + }, ), ) @@ -69,23 +75,25 @@ def make_index(self, vector_dimensions: int, conn: redis.Redis): rs.create_index(schema, definition=definition) @contextmanager - def init(self): - """ create and destory connections to database. + def init(self) -> None: + """create and destory connections to database. Examples: >>> with self.init(): >>> self.insert_embeddings() """ - self.conn = redis.Redis(host=self.db_config["host"], port=self.db_config["port"], password=self.db_config["password"], db=0) + self.conn = redis.Redis( + host=self.db_config["host"], + port=self.db_config["port"], + password=self.db_config["password"], + db=0, + ) yield self.conn.close() self.conn = None - def ready_to_search(self) -> bool: """Check if the database is ready to search.""" - pass - def ready_to_load(self) -> bool: pass @@ -93,7 +101,6 @@ def ready_to_load(self) -> bool: def optimize(self) -> None: pass - def insert_embeddings( self, embeddings: list[list[float]], @@ -104,27 +111,30 @@ def insert_embeddings( Should call self.init() first. """ - batch_size = 1000 # Adjust this as needed, but don't make too big + batch_size = 1000 # Adjust this as needed, but don't make too big try: with self.conn.pipeline(transaction=False) as pipe: for i, embedding in enumerate(embeddings): - embedding = np.array(embedding).astype(np.float32) - pipe.hset(metadata[i], mapping = { - "id": str(metadata[i]), - "metadata": metadata[i], - "vector": embedding.tobytes(), - }) + ndarr_emb = np.array(embedding).astype(np.float32) + pipe.hset( + metadata[i], + mapping={ + "id": str(metadata[i]), + "metadata": metadata[i], + "vector": ndarr_emb.tobytes(), + }, + ) # Execute the pipe so we don't keep too much in memory at once if i % batch_size == 0: - res = pipe.execute() + _ = pipe.execute() - res = pipe.execute() + _ = pipe.execute() result_len = i + 1 except Exception as e: return 0, e - + return result_len, None - + def search_embedding( self, query: list[float], @@ -132,26 +142,53 @@ def search_embedding( filters: dict | None = None, timeout: int | None = None, **kwargs: Any, - ) -> (list[int]): + ) -> list[int]: assert self.conn is not None - + query_vector = np.array(query).astype(np.float32).tobytes() ef_runtime = self.case_config.search_param()["params"]["ef"] - query_obj = Query(f"*=>[KNN {k} @vector $vec EF_RUNTIME {ef_runtime} as score]").sort_by("score").return_fields("id", "score").paging(0, k).dialect(2) + query_obj = ( + Query(f"*=>[KNN {k} @vector $vec EF_RUNTIME {ef_runtime} as score]") + .sort_by("score") + .return_fields("id", "score") + .paging(0, k) + .dialect(2) + ) query_params = {"vec": query_vector} - + if filters: # benchmark test filters of format: {'metadata': '>=10000', 'id': 10000} # gets exact match for id, and range for metadata if they exist in filters id_value = filters.get("id") metadata_value = filters.get("metadata") if id_value and metadata_value: - query_obj = Query(f"(@metadata:[{metadata_value} +inf] @id:{ {id_value} })=>[KNN {k} @vector $vec EF_RUNTIME {ef_runtime} as score]").sort_by("score").return_fields("id", "score").paging(0, k).dialect(2) + query_obj = ( + Query( + f"(@metadata:[{metadata_value} +inf] @id:{ {id_value} })=>[KNN {k} ", + f"@vector $vec EF_RUNTIME {ef_runtime} as score]", + ) + .sort_by("score") + .return_fields("id", "score") + .paging(0, k) + .dialect(2) + ) elif id_value: - #gets exact match for id - query_obj = Query(f"@id:{ {id_value} }=>[KNN {k} @vector $vec EF_RUNTIME {ef_runtime} as score]").sort_by("score").return_fields("id", "score").paging(0, k).dialect(2) - else: #metadata only case, greater than or equal to metadata value - query_obj = Query(f"@metadata:[{metadata_value} +inf]=>[KNN {k} @vector $vec EF_RUNTIME {ef_runtime} as score]").sort_by("score").return_fields("id", "score").paging(0, k).dialect(2) + # gets exact match for id + query_obj = ( + Query(f"@id:{ {id_value} }=>[KNN {k} @vector $vec EF_RUNTIME {ef_runtime} as score]") + .sort_by("score") + .return_fields("id", "score") + .paging(0, k) + .dialect(2) + ) + else: # metadata only case, greater than or equal to metadata value + query_obj = ( + Query(f"@metadata:[{metadata_value} +inf]=>[KNN {k} @vector $vec EF_RUNTIME {ef_runtime} as score]") + .sort_by("score") + .return_fields("id", "score") + .paging(0, k) + .dialect(2) + ) res = self.conn.ft(INDEX_NAME).search(query_obj, query_params) # doc in res of format {'id': '9831', 'payload': None, 'score': '1.19209289551e-07'} return [int(doc["id"]) for doc in res.docs] diff --git a/vectordb_bench/backend/clients/test/cli.py b/vectordb_bench/backend/clients/test/cli.py index f06f33492..e5cd4c78b 100644 --- a/vectordb_bench/backend/clients/test/cli.py +++ b/vectordb_bench/backend/clients/test/cli.py @@ -10,8 +10,7 @@ from ..test.config import TestConfig, TestIndexConfig -class TestTypedDict(CommonTypedDict): - ... +class TestTypedDict(CommonTypedDict): ... @cli.command() diff --git a/vectordb_bench/backend/clients/test/config.py b/vectordb_bench/backend/clients/test/config.py index 01a77e000..351d7bcac 100644 --- a/vectordb_bench/backend/clients/test/config.py +++ b/vectordb_bench/backend/clients/test/config.py @@ -1,6 +1,6 @@ -from pydantic import BaseModel, SecretStr +from pydantic import BaseModel -from ..api import DBCaseConfig, DBConfig, IndexType, MetricType +from ..api import DBCaseConfig, DBConfig, MetricType class TestConfig(DBConfig): diff --git a/vectordb_bench/backend/clients/test/test.py b/vectordb_bench/backend/clients/test/test.py index 78732eb1e..ee5a523f3 100644 --- a/vectordb_bench/backend/clients/test/test.py +++ b/vectordb_bench/backend/clients/test/test.py @@ -1,6 +1,7 @@ import logging +from collections.abc import Generator from contextlib import contextmanager -from typing import Any, Generator, Optional, Tuple +from typing import Any from ..api import DBCaseConfig, VectorDB @@ -43,11 +44,10 @@ def insert_embeddings( embeddings: list[list[float]], metadata: list[int], **kwargs: Any, - ) -> Tuple[int, Optional[Exception]]: + ) -> tuple[int, Exception | None]: """Insert embeddings into the database. Should call self.init() first. """ - raise RuntimeError("Not implemented") return len(metadata), None def search_embedding( @@ -58,5 +58,4 @@ def search_embedding( timeout: int | None = None, **kwargs: Any, ) -> list[int]: - raise NotImplementedError - return [i for i in range(k)] + return list(range(k)) diff --git a/vectordb_bench/backend/clients/weaviate_cloud/cli.py b/vectordb_bench/backend/clients/weaviate_cloud/cli.py index b6f16011b..181898c74 100644 --- a/vectordb_bench/backend/clients/weaviate_cloud/cli.py +++ b/vectordb_bench/backend/clients/weaviate_cloud/cli.py @@ -14,7 +14,8 @@ class WeaviateTypedDict(CommonTypedDict): api_key: Annotated[ - str, click.option("--api-key", type=str, help="Weaviate api key", required=True) + str, + click.option("--api-key", type=str, help="Weaviate api key", required=True), ] url: Annotated[ str, @@ -34,8 +35,6 @@ def Weaviate(**parameters: Unpack[WeaviateTypedDict]): api_key=SecretStr(parameters["api_key"]), url=SecretStr(parameters["url"]), ), - db_case_config=WeaviateIndexConfig( - ef=256, efConstruction=256, maxConnections=16 - ), + db_case_config=WeaviateIndexConfig(ef=256, efConstruction=256, maxConnections=16), **parameters, ) diff --git a/vectordb_bench/backend/clients/weaviate_cloud/config.py b/vectordb_bench/backend/clients/weaviate_cloud/config.py index 8b2f3ab81..4c58167d4 100644 --- a/vectordb_bench/backend/clients/weaviate_cloud/config.py +++ b/vectordb_bench/backend/clients/weaviate_cloud/config.py @@ -1,6 +1,6 @@ from pydantic import BaseModel, SecretStr -from ..api import DBConfig, DBCaseConfig, MetricType +from ..api import DBCaseConfig, DBConfig, MetricType class WeaviateConfig(DBConfig): @@ -23,7 +23,7 @@ class WeaviateIndexConfig(BaseModel, DBCaseConfig): def parse_metric(self) -> str: if self.metric_type == MetricType.L2: return "l2-squared" - elif self.metric_type == MetricType.IP: + if self.metric_type == MetricType.IP: return "dot" return "cosine" diff --git a/vectordb_bench/backend/clients/weaviate_cloud/weaviate_cloud.py b/vectordb_bench/backend/clients/weaviate_cloud/weaviate_cloud.py index d8fde5f09..b42f70af1 100644 --- a/vectordb_bench/backend/clients/weaviate_cloud/weaviate_cloud.py +++ b/vectordb_bench/backend/clients/weaviate_cloud/weaviate_cloud.py @@ -1,13 +1,13 @@ """Wrapper around the Weaviate vector database over VectorDB""" import logging -from typing import Iterable +from collections.abc import Iterable from contextlib import contextmanager import weaviate from weaviate.exceptions import WeaviateBaseError -from ..api import VectorDB, DBCaseConfig +from ..api import DBCaseConfig, VectorDB log = logging.getLogger(__name__) @@ -23,7 +23,13 @@ def __init__( **kwargs, ): """Initialize wrapper around the weaviate vector database.""" - db_config.update({"auth_client_secret": weaviate.AuthApiKey(api_key=db_config.get("auth_client_secret"))}) + db_config.update( + { + "auth_client_secret": weaviate.AuthApiKey( + api_key=db_config.get("auth_client_secret"), + ), + }, + ) self.db_config = db_config self.case_config = db_case_config self.collection_name = collection_name @@ -33,6 +39,7 @@ def __init__( self._index_name = "vector_idx" from weaviate import Client + client = Client(**db_config) if drop_old: try: @@ -40,7 +47,7 @@ def __init__( log.info(f"weaviate client drop_old collection: {self.collection_name}") client.schema.delete_class(self.collection_name) except WeaviateBaseError as e: - log.warning(f"Failed to drop collection: {self.collection_name} error: {str(e)}") + log.warning(f"Failed to drop collection: {self.collection_name} error: {e!s}") raise e from None self._create_collection(client) client = None @@ -54,20 +61,23 @@ def init(self) -> None: >>> self.search_embedding() """ from weaviate import Client + self.client = Client(**self.db_config) yield self.client = None - del(self.client) + del self.client def ready_to_load(self): """Should call insert first, do nothing""" - pass def optimize(self): assert self.client.schema.exists(self.collection_name) - self.client.schema.update_config(self.collection_name, {"vectorIndexConfig": self.case_config.search_param() } ) + self.client.schema.update_config( + self.collection_name, + {"vectorIndexConfig": self.case_config.search_param()}, + ) - def _create_collection(self, client): + def _create_collection(self, client: weaviate.Client) -> None: if not client.schema.exists(self.collection_name): log.info(f"Create collection: {self.collection_name}") class_obj = { @@ -78,13 +88,13 @@ def _create_collection(self, client): "dataType": ["int"], "name": self._scalar_field, }, - ] + ], } class_obj["vectorIndexConfig"] = self.case_config.index_param() try: client.schema.create_class(class_obj) except WeaviateBaseError as e: - log.warning(f"Failed to create collection: {self.collection_name} error: {str(e)}") + log.warning(f"Failed to create collection: {self.collection_name} error: {e!s}") raise e from None def insert_embeddings( @@ -102,15 +112,17 @@ def insert_embeddings( batch.dynamic = True res = [] for i in range(len(metadata)): - res.append(batch.add_data_object( - {self._scalar_field: metadata[i]}, - class_name=self.collection_name, - vector=embeddings[i] - )) + res.append( + batch.add_data_object( + {self._scalar_field: metadata[i]}, + class_name=self.collection_name, + vector=embeddings[i], + ), + ) insert_count += 1 return (len(res), None) except WeaviateBaseError as e: - log.warning(f"Failed to insert data, error: {str(e)}") + log.warning(f"Failed to insert data, error: {e!s}") return (insert_count, e) def search_embedding( @@ -125,12 +137,17 @@ def search_embedding( """ assert self.client.schema.exists(self.collection_name) - query_obj = self.client.query.get(self.collection_name, [self._scalar_field]).with_additional("distance").with_near_vector({"vector": query}).with_limit(k) + query_obj = ( + self.client.query.get(self.collection_name, [self._scalar_field]) + .with_additional("distance") + .with_near_vector({"vector": query}) + .with_limit(k) + ) if filters: where_filter = { "path": "key", "operator": "GreaterThanEqual", - "valueInt": filters.get('id') + "valueInt": filters.get("id"), } query_obj = query_obj.with_where(where_filter) @@ -138,7 +155,4 @@ def search_embedding( res = query_obj.do() # Organize results. - ret = [result[self._scalar_field] for result in res["data"]["Get"][self.collection_name]] - - return ret - + return [result[self._scalar_field] for result in res["data"]["Get"][self.collection_name]] diff --git a/vectordb_bench/backend/clients/zilliz_cloud/cli.py b/vectordb_bench/backend/clients/zilliz_cloud/cli.py index 31618f4ec..810a4175e 100644 --- a/vectordb_bench/backend/clients/zilliz_cloud/cli.py +++ b/vectordb_bench/backend/clients/zilliz_cloud/cli.py @@ -1,33 +1,36 @@ +import os from typing import Annotated, Unpack import click -import os from pydantic import SecretStr +from vectordb_bench.backend.clients import DB from vectordb_bench.cli.cli import ( CommonTypedDict, cli, click_parameter_decorators_from_typed_dict, run, ) -from vectordb_bench.backend.clients import DB class ZillizTypedDict(CommonTypedDict): uri: Annotated[ - str, click.option("--uri", type=str, help="uri connection string", required=True) + str, + click.option("--uri", type=str, help="uri connection string", required=True), ] user_name: Annotated[ - str, click.option("--user-name", type=str, help="Db username", required=True) + str, + click.option("--user-name", type=str, help="Db username", required=True), ] password: Annotated[ str, - click.option("--password", - type=str, - help="Zilliz password", - default=lambda: os.environ.get("ZILLIZ_PASSWORD", ""), - show_default="$ZILLIZ_PASSWORD", - ), + click.option( + "--password", + type=str, + help="Zilliz password", + default=lambda: os.environ.get("ZILLIZ_PASSWORD", ""), + show_default="$ZILLIZ_PASSWORD", + ), ] level: Annotated[ str, @@ -38,7 +41,7 @@ class ZillizTypedDict(CommonTypedDict): @cli.command() @click_parameter_decorators_from_typed_dict(ZillizTypedDict) def ZillizAutoIndex(**parameters: Unpack[ZillizTypedDict]): - from .config import ZillizCloudConfig, AutoIndexConfig + from .config import AutoIndexConfig, ZillizCloudConfig run( db=DB.ZillizCloud, diff --git a/vectordb_bench/backend/clients/zilliz_cloud/config.py b/vectordb_bench/backend/clients/zilliz_cloud/config.py index ee60b397f..9f113dbda 100644 --- a/vectordb_bench/backend/clients/zilliz_cloud/config.py +++ b/vectordb_bench/backend/clients/zilliz_cloud/config.py @@ -1,7 +1,7 @@ from pydantic import SecretStr from ..api import DBCaseConfig, DBConfig -from ..milvus.config import MilvusIndexConfig, IndexType +from ..milvus.config import IndexType, MilvusIndexConfig class ZillizCloudConfig(DBConfig): @@ -33,7 +33,5 @@ def search_param(self) -> dict: "metric_type": self.parse_metric(), "params": { "level": self.level, - } + }, } - - diff --git a/vectordb_bench/backend/clients/zilliz_cloud/zilliz_cloud.py b/vectordb_bench/backend/clients/zilliz_cloud/zilliz_cloud.py index 36f7fb204..4ce15545a 100644 --- a/vectordb_bench/backend/clients/zilliz_cloud/zilliz_cloud.py +++ b/vectordb_bench/backend/clients/zilliz_cloud/zilliz_cloud.py @@ -1,7 +1,7 @@ """Wrapper around the ZillizCloud vector database over VectorDB""" -from ..milvus.milvus import Milvus from ..api import DBCaseConfig +from ..milvus.milvus import Milvus class ZillizCloud(Milvus): diff --git a/vectordb_bench/backend/data_source.py b/vectordb_bench/backend/data_source.py index 9e2f172b4..b98dc7d7a 100644 --- a/vectordb_bench/backend/data_source.py +++ b/vectordb_bench/backend/data_source.py @@ -1,12 +1,12 @@ import logging import pathlib import typing +from abc import ABC, abstractmethod from enum import Enum + from tqdm import tqdm -import os -from abc import ABC, abstractmethod -from .. import config +from vectordb_bench import config logging.getLogger("s3fs").setLevel(logging.CRITICAL) @@ -14,6 +14,7 @@ DatasetReader = typing.TypeVar("DatasetReader") + class DatasetSource(Enum): S3 = "S3" AliyunOSS = "AliyunOSS" @@ -25,6 +26,8 @@ def reader(self) -> DatasetReader: if self == DatasetSource.AliyunOSS: return AliyunOSSReader() + return None + class DatasetReader(ABC): source: DatasetSource @@ -39,7 +42,6 @@ def read(self, dataset: str, files: list[str], local_ds_root: pathlib.Path): files(list[str]): all filenames of the dataset local_ds_root(pathlib.Path): whether to write the remote data. """ - pass @abstractmethod def validate_file(self, remote: pathlib.Path, local: pathlib.Path) -> bool: @@ -52,15 +54,18 @@ class AliyunOSSReader(DatasetReader): def __init__(self): import oss2 + self.bucket = oss2.Bucket(oss2.AnonymousAuth(), self.remote_root, "benchmark", True) def validate_file(self, remote: pathlib.Path, local: pathlib.Path) -> bool: info = self.bucket.get_object_meta(remote.as_posix()) # check size equal - remote_size, local_size = info.content_length, os.path.getsize(local) + remote_size, local_size = info.content_length, local.stat().st_size if remote_size != local_size: - log.info(f"local file: {local} size[{local_size}] not match with remote size[{remote_size}]") + log.info( + f"local file: {local} size[{local_size}] not match with remote size[{remote_size}]", + ) return False return True @@ -70,7 +75,13 @@ def read(self, dataset: str, files: list[str], local_ds_root: pathlib.Path): if not local_ds_root.exists(): log.info(f"local dataset root path not exist, creating it: {local_ds_root}") local_ds_root.mkdir(parents=True) - downloads = [(pathlib.PurePosixPath("benchmark", dataset, f), local_ds_root.joinpath(f)) for f in files] + downloads = [ + ( + pathlib.PurePosixPath("benchmark", dataset, f), + local_ds_root.joinpath(f), + ) + for f in files + ] else: for file in files: @@ -78,7 +89,9 @@ def read(self, dataset: str, files: list[str], local_ds_root: pathlib.Path): local_file = local_ds_root.joinpath(file) if (not local_file.exists()) or (not self.validate_file(remote_file, local_file)): - log.info(f"local file: {local_file} not match with remote: {remote_file}; add to downloading list") + log.info( + f"local file: {local_file} not match with remote: {remote_file}; add to downloading list", + ) downloads.append((remote_file, local_file)) if len(downloads) == 0: @@ -92,17 +105,14 @@ def read(self, dataset: str, files: list[str], local_ds_root: pathlib.Path): log.info(f"Succeed to download all files, downloaded file count = {len(downloads)}") - class AwsS3Reader(DatasetReader): source: DatasetSource = DatasetSource.S3 remote_root: str = config.AWS_S3_URL def __init__(self): import s3fs - self.fs = s3fs.S3FileSystem( - anon=True, - client_kwargs={'region_name': 'us-west-2'} - ) + + self.fs = s3fs.S3FileSystem(anon=True, client_kwargs={"region_name": "us-west-2"}) def ls_all(self, dataset: str): dataset_root_dir = pathlib.Path(self.remote_root, dataset) @@ -112,7 +122,6 @@ def ls_all(self, dataset: str): log.info(n) return names - def read(self, dataset: str, files: list[str], local_ds_root: pathlib.Path): downloads = [] if not local_ds_root.exists(): @@ -126,7 +135,9 @@ def read(self, dataset: str, files: list[str], local_ds_root: pathlib.Path): local_file = local_ds_root.joinpath(file) if (not local_file.exists()) or (not self.validate_file(remote_file, local_file)): - log.info(f"local file: {local_file} not match with remote: {remote_file}; add to downloading list") + log.info( + f"local file: {local_file} not match with remote: {remote_file}; add to downloading list", + ) downloads.append(remote_file) if len(downloads) == 0: @@ -139,15 +150,16 @@ def read(self, dataset: str, files: list[str], local_ds_root: pathlib.Path): log.info(f"Succeed to download all files, downloaded file count = {len(downloads)}") - def validate_file(self, remote: pathlib.Path, local: pathlib.Path) -> bool: # info() uses ls() inside, maybe we only need to ls once info = self.fs.info(remote) # check size equal - remote_size, local_size = info.get("size"), os.path.getsize(local) + remote_size, local_size = info.get("size"), local.stat().st_size if remote_size != local_size: - log.info(f"local file: {local} size[{local_size}] not match with remote size[{remote_size}]") + log.info( + f"local file: {local} size[{local_size}] not match with remote size[{remote_size}]", + ) return False return True diff --git a/vectordb_bench/backend/dataset.py b/vectordb_bench/backend/dataset.py index d62d96684..62700b0fa 100644 --- a/vectordb_bench/backend/dataset.py +++ b/vectordb_bench/backend/dataset.py @@ -4,25 +4,30 @@ >>> Dataset.Cohere.get(100_000) """ -from collections import namedtuple import logging import pathlib +import typing from enum import Enum + import pandas as pd -from pydantic import validator, PrivateAttr import polars as pl from pyarrow.parquet import ParquetFile +from pydantic import PrivateAttr, validator + +from vectordb_bench import config +from vectordb_bench.base import BaseModel -from ..base import BaseModel -from .. import config -from ..backend.clients import MetricType from . import utils -from .data_source import DatasetSource, DatasetReader +from .clients import MetricType +from .data_source import DatasetReader, DatasetSource log = logging.getLogger(__name__) -SizeLabel = namedtuple('SizeLabel', ['size', 'label', 'file_count']) +class SizeLabel(typing.NamedTuple): + size: int + label: str + file_count: int class BaseDataset(BaseModel): @@ -33,12 +38,13 @@ class BaseDataset(BaseModel): use_shuffled: bool with_gt: bool = False _size_label: dict[int, SizeLabel] = PrivateAttr() - isCustom: bool = False + is_custom: bool = False @validator("size") - def verify_size(cls, v): + def verify_size(cls, v: int): if v not in cls._size_label: - raise ValueError(f"Size {v} not supported for the dataset, expected: {cls._size_label.keys()}") + msg = f"Size {v} not supported for the dataset, expected: {cls._size_label.keys()}" + raise ValueError(msg) return v @property @@ -53,13 +59,14 @@ def dir_name(self) -> str: def file_count(self) -> int: return self._size_label.get(self.size).file_count + class CustomDataset(BaseDataset): dir: str file_num: int - isCustom: bool = True + is_custom: bool = True @validator("size") - def verify_size(cls, v): + def verify_size(cls, v: int): return v @property @@ -102,7 +109,7 @@ class Cohere(BaseDataset): dim: int = 768 metric_type: MetricType = MetricType.COSINE use_shuffled: bool = config.USE_SHUFFLED_DATA - with_gt: bool = True, + with_gt: bool = (True,) _size_label: dict = { 100_000: SizeLabel(100_000, "SMALL", 1), 1_000_000: SizeLabel(1_000_000, "MEDIUM", 1), @@ -124,7 +131,11 @@ class SIFT(BaseDataset): metric_type: MetricType = MetricType.L2 use_shuffled: bool = False _size_label: dict = { - 500_000: SizeLabel(500_000, "SMALL", 1,), + 500_000: SizeLabel( + 500_000, + "SMALL", + 1, + ), 5_000_000: SizeLabel(5_000_000, "MEDIUM", 1), # 50_000_000: SizeLabel(50_000_000, "LARGE", 50), } @@ -135,7 +146,7 @@ class OpenAI(BaseDataset): dim: int = 1536 metric_type: MetricType = MetricType.COSINE use_shuffled: bool = config.USE_SHUFFLED_DATA - with_gt: bool = True, + with_gt: bool = (True,) _size_label: dict = { 50_000: SizeLabel(50_000, "SMALL", 1), 500_000: SizeLabel(500_000, "MEDIUM", 1), @@ -153,13 +164,14 @@ class DatasetManager(BaseModel): >>> for data in cohere: >>> print(data.columns) """ - data: BaseDataset + + data: BaseDataset test_data: pd.DataFrame | None = None gt_data: pd.DataFrame | None = None - train_files : list[str] = [] + train_files: list[str] = [] reader: DatasetReader | None = None - def __eq__(self, obj): + def __eq__(self, obj: any): if isinstance(obj, DatasetManager): return self.data.name == obj.data.name and self.data.label == obj.data.label return False @@ -169,22 +181,27 @@ def set_reader(self, reader: DatasetReader): @property def data_dir(self) -> pathlib.Path: - """ data local directory: config.DATASET_LOCAL_DIR/{dataset_name}/{dataset_dirname} + """data local directory: config.DATASET_LOCAL_DIR/{dataset_name}/{dataset_dirname} Examples: >>> sift_s = Dataset.SIFT.manager(500_000) >>> sift_s.relative_path '/tmp/vectordb_bench/dataset/sift/sift_small_500k/' """ - return pathlib.Path(config.DATASET_LOCAL_DIR, self.data.name.lower(), self.data.dir_name.lower()) + return pathlib.Path( + config.DATASET_LOCAL_DIR, + self.data.name.lower(), + self.data.dir_name.lower(), + ) def __iter__(self): return DataSetIterator(self) # TODO passing use_shuffle from outside - def prepare(self, - source: DatasetSource=DatasetSource.S3, - filters: int | float | str | None = None, + def prepare( + self, + source: DatasetSource = DatasetSource.S3, + filters: float | str | None = None, ) -> bool: """Download the dataset from DatasetSource url = f"{source}/{self.data.dir_name}" @@ -208,7 +225,7 @@ def prepare(self, gt_file, test_file = utils.compose_gt_file(filters), "test.parquet" all_files.extend([gt_file, test_file]) - if not self.data.isCustom: + if not self.data.is_custom: source.reader().read( dataset=self.data.dir_name.lower(), files=all_files, @@ -220,7 +237,7 @@ def prepare(self, self.gt_data = self._read_file(gt_file) prefix = "shuffle_train" if use_shuffled else "train" - self.train_files = sorted([f.name for f in self.data_dir.glob(f'{prefix}*.parquet')]) + self.train_files = sorted([f.name for f in self.data_dir.glob(f"{prefix}*.parquet")]) log.debug(f"{self.data.name}: available train files {self.train_files}") return True @@ -241,7 +258,7 @@ def __init__(self, dataset: DatasetManager): self._ds = dataset self._idx = 0 # file number self._cur = None - self._sub_idx = [0 for i in range(len(self._ds.train_files))] # iter num for each file + self._sub_idx = [0 for i in range(len(self._ds.train_files))] # iter num for each file def __iter__(self): return self @@ -250,7 +267,9 @@ def _get_iter(self, file_name: str): p = pathlib.Path(self._ds.data_dir, file_name) log.info(f"Get iterator for {p.name}") if not p.exists(): - raise IndexError(f"No such file {p}") + msg = f"No such file: {p}" + log.warning(msg) + raise IndexError(msg) return ParquetFile(p, memory_map=True, pre_buffer=True).iter_batches(config.NUM_PER_BATCH) def __next__(self) -> pd.DataFrame: @@ -281,6 +300,7 @@ class Dataset(Enum): >>> Dataset.COHERE.manager(100_000) >>> Dataset.COHERE.get(100_000) """ + LAION = LAION GIST = GIST COHERE = Cohere diff --git a/vectordb_bench/backend/result_collector.py b/vectordb_bench/backend/result_collector.py index d4c073c1a..a4119a5fd 100644 --- a/vectordb_bench/backend/result_collector.py +++ b/vectordb_bench/backend/result_collector.py @@ -1,7 +1,7 @@ +import logging import pathlib -from ..models import TestResult -import logging +from vectordb_bench.models import TestResult log = logging.getLogger(__name__) @@ -14,7 +14,6 @@ def collect(cls, result_dir: pathlib.Path) -> list[TestResult]: if not result_dir.exists() or len(list(result_dir.rglob(reg))) == 0: return [] - for json_file in result_dir.rglob(reg): file_result = TestResult.read_file(json_file, trans_unit=True) diff --git a/vectordb_bench/backend/runner/__init__.py b/vectordb_bench/backend/runner/__init__.py index 77bb25d67..b83df6f99 100644 --- a/vectordb_bench/backend/runner/__init__.py +++ b/vectordb_bench/backend/runner/__init__.py @@ -1,12 +1,10 @@ from .mp_runner import ( MultiProcessingSearchRunner, ) - -from .serial_runner import SerialSearchRunner, SerialInsertRunner - +from .serial_runner import SerialInsertRunner, SerialSearchRunner __all__ = [ - 'MultiProcessingSearchRunner', - 'SerialSearchRunner', - 'SerialInsertRunner', + "MultiProcessingSearchRunner", + "SerialInsertRunner", + "SerialSearchRunner", ] diff --git a/vectordb_bench/backend/runner/mp_runner.py b/vectordb_bench/backend/runner/mp_runner.py index 8f35dcd8d..5b69b5481 100644 --- a/vectordb_bench/backend/runner/mp_runner.py +++ b/vectordb_bench/backend/runner/mp_runner.py @@ -1,27 +1,29 @@ -import time -import traceback import concurrent +import logging import multiprocessing as mp import random -import logging -from typing import Iterable +import time +import traceback +from collections.abc import Iterable + import numpy as np -from ..clients import api -from ... import config +from ... import config +from ..clients import api NUM_PER_BATCH = config.NUM_PER_BATCH log = logging.getLogger(__name__) class MultiProcessingSearchRunner: - """ multiprocessing search runner + """multiprocessing search runner Args: k(int): search topk, default to 100 concurrency(Iterable): concurrencies, default [1, 5, 10, 15, 20, 25, 30, 35] duration(int): duration for each concurency, default to 30s """ + def __init__( self, db: api.VectorDB, @@ -40,7 +42,12 @@ def __init__( self.test_data = test_data log.debug(f"test dataset columns: {len(test_data)}") - def search(self, test_data: list[list[float]], q: mp.Queue, cond: mp.Condition) -> tuple[int, float]: + def search( + self, + test_data: list[list[float]], + q: mp.Queue, + cond: mp.Condition, + ) -> tuple[int, float]: # sync all process q.put(1) with cond: @@ -71,24 +78,27 @@ def search(self, test_data: list[list[float]], q: mp.Queue, cond: mp.Condition) idx = idx + 1 if idx < num - 1 else 0 if count % 500 == 0: - log.debug(f"({mp.current_process().name:16}) search_count: {count}, latest_latency={time.perf_counter()-s}") + log.debug( + f"({mp.current_process().name:16}) ", + f"search_count: {count}, latest_latency={time.perf_counter()-s}", + ) total_dur = round(time.perf_counter() - start_time, 4) log.info( f"{mp.current_process().name:16} search {self.duration}s: " - f"actual_dur={total_dur}s, count={count}, qps in this process: {round(count / total_dur, 4):3}" - ) + f"actual_dur={total_dur}s, count={count}, qps in this process: {round(count / total_dur, 4):3}", + ) return (count, total_dur, latencies) @staticmethod def get_mp_context(): mp_start_method = "spawn" - log.debug(f"MultiProcessingSearchRunner get multiprocessing start method: {mp_start_method}") + log.debug( + f"MultiProcessingSearchRunner get multiprocessing start method: {mp_start_method}", + ) return mp.get_context(mp_start_method) - - def _run_all_concurrencies_mem_efficient(self): max_qps = 0 conc_num_list = [] @@ -99,8 +109,13 @@ def _run_all_concurrencies_mem_efficient(self): for conc in self.concurrencies: with mp.Manager() as m: q, cond = m.Queue(), m.Condition() - with concurrent.futures.ProcessPoolExecutor(mp_context=self.get_mp_context(), max_workers=conc) as executor: - log.info(f"Start search {self.duration}s in concurrency {conc}, filters: {self.filters}") + with concurrent.futures.ProcessPoolExecutor( + mp_context=self.get_mp_context(), + max_workers=conc, + ) as executor: + log.info( + f"Start search {self.duration}s in concurrency {conc}, filters: {self.filters}", + ) future_iter = [executor.submit(self.search, self.test_data, q, cond) for i in range(conc)] # Sync all processes while q.qsize() < conc: @@ -109,7 +124,9 @@ def _run_all_concurrencies_mem_efficient(self): with cond: cond.notify_all() - log.info(f"Syncing all process and start concurrency search, concurrency={conc}") + log.info( + f"Syncing all process and start concurrency search, concurrency={conc}", + ) start = time.perf_counter() all_count = sum([r.result()[0] for r in future_iter]) @@ -123,13 +140,19 @@ def _run_all_concurrencies_mem_efficient(self): conc_qps_list.append(qps) conc_latency_p99_list.append(latency_p99) conc_latency_avg_list.append(latency_avg) - log.info(f"End search in concurrency {conc}: dur={cost}s, total_count={all_count}, qps={qps}") + log.info( + f"End search in concurrency {conc}: dur={cost}s, total_count={all_count}, qps={qps}", + ) if qps > max_qps: max_qps = qps - log.info(f"Update largest qps with concurrency {conc}: current max_qps={max_qps}") + log.info( + f"Update largest qps with concurrency {conc}: current max_qps={max_qps}", + ) except Exception as e: - log.warning(f"Fail to search all concurrencies: {self.concurrencies}, max_qps before failure={max_qps}, reason={e}") + log.warning( + f"Fail to search all concurrencies: {self.concurrencies}, max_qps before failure={max_qps}, reason={e}", + ) traceback.print_exc() # No results available, raise exception @@ -139,7 +162,13 @@ def _run_all_concurrencies_mem_efficient(self): finally: self.stop() - return max_qps, conc_num_list, conc_qps_list, conc_latency_p99_list, conc_latency_avg_list + return ( + max_qps, + conc_num_list, + conc_qps_list, + conc_latency_p99_list, + conc_latency_avg_list, + ) def run(self) -> float: """ @@ -160,9 +189,16 @@ def _run_by_dur(self, duration: int) -> float: for conc in self.concurrencies: with mp.Manager() as m: q, cond = m.Queue(), m.Condition() - with concurrent.futures.ProcessPoolExecutor(mp_context=self.get_mp_context(), max_workers=conc) as executor: - log.info(f"Start search_by_dur {duration}s in concurrency {conc}, filters: {self.filters}") - future_iter = [executor.submit(self.search_by_dur, duration, self.test_data, q, cond) for i in range(conc)] + with concurrent.futures.ProcessPoolExecutor( + mp_context=self.get_mp_context(), + max_workers=conc, + ) as executor: + log.info( + f"Start search_by_dur {duration}s in concurrency {conc}, filters: {self.filters}", + ) + future_iter = [ + executor.submit(self.search_by_dur, duration, self.test_data, q, cond) for i in range(conc) + ] # Sync all processes while q.qsize() < conc: sleep_t = conc if conc < 10 else 10 @@ -170,20 +206,28 @@ def _run_by_dur(self, duration: int) -> float: with cond: cond.notify_all() - log.info(f"Syncing all process and start concurrency search, concurrency={conc}") + log.info( + f"Syncing all process and start concurrency search, concurrency={conc}", + ) start = time.perf_counter() all_count = sum([r.result() for r in future_iter]) cost = time.perf_counter() - start qps = round(all_count / cost, 4) - log.info(f"End search in concurrency {conc}: dur={cost}s, total_count={all_count}, qps={qps}") + log.info( + f"End search in concurrency {conc}: dur={cost}s, total_count={all_count}, qps={qps}", + ) if qps > max_qps: max_qps = qps - log.info(f"Update largest qps with concurrency {conc}: current max_qps={max_qps}") + log.info( + f"Update largest qps with concurrency {conc}: current max_qps={max_qps}", + ) except Exception as e: - log.warning(f"Fail to search all concurrencies: {self.concurrencies}, max_qps before failure={max_qps}, reason={e}") + log.warning( + f"Fail to search all concurrencies: {self.concurrencies}, max_qps before failure={max_qps}, reason={e}", + ) traceback.print_exc() # No results available, raise exception @@ -195,8 +239,13 @@ def _run_by_dur(self, duration: int) -> float: return max_qps - - def search_by_dur(self, dur: int, test_data: list[list[float]], q: mp.Queue, cond: mp.Condition) -> int: + def search_by_dur( + self, + dur: int, + test_data: list[list[float]], + q: mp.Queue, + cond: mp.Condition, + ) -> int: # sync all process q.put(1) with cond: @@ -225,13 +274,15 @@ def search_by_dur(self, dur: int, test_data: list[list[float]], q: mp.Queue, con idx = idx + 1 if idx < num - 1 else 0 if count % 500 == 0: - log.debug(f"({mp.current_process().name:16}) search_count: {count}, latest_latency={time.perf_counter()-s}") + log.debug( + f"({mp.current_process().name:16}) search_count: {count}, ", + f"latest_latency={time.perf_counter()-s}", + ) total_dur = round(time.perf_counter() - start_time, 4) log.debug( f"{mp.current_process().name:16} search {self.duration}s: " - f"actual_dur={total_dur}s, count={count}, qps in this process: {round(count / total_dur, 4):3}" - ) + f"actual_dur={total_dur}s, count={count}, qps in this process: {round(count / total_dur, 4):3}", + ) return count - diff --git a/vectordb_bench/backend/runner/rate_runner.py b/vectordb_bench/backend/runner/rate_runner.py index d77c0fd15..0145af4ce 100644 --- a/vectordb_bench/backend/runner/rate_runner.py +++ b/vectordb_bench/backend/runner/rate_runner.py @@ -1,36 +1,36 @@ +import concurrent import logging +import multiprocessing as mp import time -import concurrent from concurrent.futures import ThreadPoolExecutor -import multiprocessing as mp - +from vectordb_bench import config from vectordb_bench.backend.clients import api from vectordb_bench.backend.dataset import DataSetIterator from vectordb_bench.backend.utils import time_it -from vectordb_bench import config from .util import get_data + log = logging.getLogger(__name__) class RatedMultiThreadingInsertRunner: def __init__( self, - rate: int, # numRows per second + rate: int, # numRows per second db: api.VectorDB, dataset_iter: DataSetIterator, normalize: bool = False, timeout: float | None = None, ): - self.timeout = timeout if isinstance(timeout, (int, float)) else None + self.timeout = timeout if isinstance(timeout, int | float) else None self.dataset = dataset_iter self.db = db self.normalize = normalize self.insert_rate = rate self.batch_rate = rate // config.NUM_PER_BATCH - def send_insert_task(self, db, emb: list[list[float]], metadata: list[str]): + def send_insert_task(self, db: api.VectorDB, emb: list[list[float]], metadata: list[str]): db.insert_embeddings(emb, metadata) @time_it @@ -43,7 +43,9 @@ def submit_by_rate() -> bool: rate = self.batch_rate for data in self.dataset: emb, metadata = get_data(data, self.normalize) - executing_futures.append(executor.submit(self.send_insert_task, self.db, emb, metadata)) + executing_futures.append( + executor.submit(self.send_insert_task, self.db, emb, metadata), + ) rate -= 1 if rate == 0: @@ -66,19 +68,26 @@ def submit_by_rate() -> bool: done, not_done = concurrent.futures.wait( executing_futures, timeout=wait_interval, - return_when=concurrent.futures.FIRST_EXCEPTION) + return_when=concurrent.futures.FIRST_EXCEPTION, + ) if len(not_done) > 0: - log.warning(f"Failed to finish all tasks in 1s, [{len(not_done)}/{len(executing_futures)}] tasks are not done, waited={wait_interval:.2f}, trying to wait in the next round") + log.warning( + f"Failed to finish all tasks in 1s, [{len(not_done)}/{len(executing_futures)}] ", + f"tasks are not done, waited={wait_interval:.2f}, trying to wait in the next round", + ) executing_futures = list(not_done) else: - log.debug(f"Finished {len(executing_futures)} insert-{config.NUM_PER_BATCH} task in 1s, wait_interval={wait_interval:.2f}") + log.debug( + f"Finished {len(executing_futures)} insert-{config.NUM_PER_BATCH} ", + f"task in 1s, wait_interval={wait_interval:.2f}", + ) executing_futures = [] except Exception as e: - log.warn(f"task error, terminating, err={e}") - q.put(None, block=True) - executor.shutdown(wait=True, cancel_futures=True) - raise e + log.warning(f"task error, terminating, err={e}") + q.put(None, block=True) + executor.shutdown(wait=True, cancel_futures=True) + raise e from e dur = time.perf_counter() - start_time if dur < 1: @@ -87,10 +96,12 @@ def submit_by_rate() -> bool: # wait for all tasks in executing_futures to complete if len(executing_futures) > 0: try: - done, _ = concurrent.futures.wait(executing_futures, - return_when=concurrent.futures.FIRST_EXCEPTION) + done, _ = concurrent.futures.wait( + executing_futures, + return_when=concurrent.futures.FIRST_EXCEPTION, + ) except Exception as e: - log.warn(f"task error, terminating, err={e}") + log.warning(f"task error, terminating, err={e}") q.put(None, block=True) executor.shutdown(wait=True, cancel_futures=True) - raise e + raise e from e diff --git a/vectordb_bench/backend/runner/read_write_runner.py b/vectordb_bench/backend/runner/read_write_runner.py index fd425b227..e916f45d6 100644 --- a/vectordb_bench/backend/runner/read_write_runner.py +++ b/vectordb_bench/backend/runner/read_write_runner.py @@ -1,16 +1,18 @@ +import concurrent import logging -from typing import Iterable +import math import multiprocessing as mp -import concurrent +from collections.abc import Iterable + import numpy as np -import math -from .mp_runner import MultiProcessingSearchRunner -from .serial_runner import SerialSearchRunner -from .rate_runner import RatedMultiThreadingInsertRunner from vectordb_bench.backend.clients import api from vectordb_bench.backend.dataset import DatasetManager +from .mp_runner import MultiProcessingSearchRunner +from .rate_runner import RatedMultiThreadingInsertRunner +from .serial_runner import SerialSearchRunner + log = logging.getLogger(__name__) @@ -24,8 +26,14 @@ def __init__( k: int = 100, filters: dict | None = None, concurrencies: Iterable[int] = (1, 15, 50), - search_stage: Iterable[float] = (0.5, 0.6, 0.7, 0.8, 0.9), # search from insert portion, 0.0 means search from the start - read_dur_after_write: int = 300, # seconds, search duration when insertion is done + search_stage: Iterable[float] = ( + 0.5, + 0.6, + 0.7, + 0.8, + 0.9, + ), # search from insert portion, 0.0 means search from the start + read_dur_after_write: int = 300, # seconds, search duration when insertion is done timeout: float | None = None, ): self.insert_rate = insert_rate @@ -36,7 +44,10 @@ def __init__( self.search_stage = sorted(search_stage) self.read_dur_after_write = read_dur_after_write - log.info(f"Init runner, concurencys={concurrencies}, search_stage={search_stage}, stage_search_dur={read_dur_after_write}") + log.info( + f"Init runner, concurencys={concurrencies}, search_stage={search_stage}, ", + f"stage_search_dur={read_dur_after_write}", + ) test_emb = np.stack(dataset.test_data["emb"]) if normalize: @@ -76,8 +87,13 @@ def run_search(self): log.info("Search after write - Serial search start") res, ssearch_dur = self.serial_search_runner.run() recall, ndcg, p99_latency = res - log.info(f"Search after write - Serial search - recall={recall}, ndcg={ndcg}, p99={p99_latency}, dur={ssearch_dur:.4f}") - log.info(f"Search after wirte - Conc search start, dur for each conc={self.read_dur_after_write}") + log.info( + f"Search after write - Serial search - recall={recall}, ndcg={ndcg}, p99={p99_latency}, ", + f"dur={ssearch_dur:.4f}", + ) + log.info( + f"Search after wirte - Conc search start, dur for each conc={self.read_dur_after_write}", + ) max_qps = self.run_by_dur(self.read_dur_after_write) log.info(f"Search after wirte - Conc search finished, max_qps={max_qps}") @@ -86,7 +102,10 @@ def run_search(self): def run_read_write(self): with mp.Manager() as m: q = m.Queue() - with concurrent.futures.ProcessPoolExecutor(mp_context=mp.get_context("spawn"), max_workers=2) as executor: + with concurrent.futures.ProcessPoolExecutor( + mp_context=mp.get_context("spawn"), + max_workers=2, + ) as executor: read_write_futures = [] read_write_futures.append(executor.submit(self.run_with_rate, q)) read_write_futures.append(executor.submit(self.run_search_by_sig, q)) @@ -107,10 +126,10 @@ def run_read_write(self): except Exception as e: log.warning(f"Read and write error: {e}") executor.shutdown(wait=True, cancel_futures=True) - raise e + raise e from e log.info("Concurrent read write all done") - def run_search_by_sig(self, q): + def run_search_by_sig(self, q: mp.Queue): """ Args: q: multiprocessing queue @@ -122,15 +141,14 @@ def run_search_by_sig(self, q): total_batch = math.ceil(self.data_volume / self.insert_rate) recall, ndcg, p99_latency = None, None, None - def wait_next_target(start, target_batch) -> bool: + def wait_next_target(start: int, target_batch: int) -> bool: """Return False when receive True or None""" while start < target_batch: sig = q.get(block=True) if sig is None or sig is True: return False - else: - start += 1 + start += 1 return True for idx, stage in enumerate(self.search_stage): @@ -139,19 +157,24 @@ def wait_next_target(start, target_batch) -> bool: got = wait_next_target(start_batch, target_batch) if got is False: - log.warning(f"Abnormal exit, target_batch={target_batch}, start_batch={start_batch}") - return + log.warning( + f"Abnormal exit, target_batch={target_batch}, start_batch={start_batch}", + ) + return None log.info(f"Insert {perc}% done, total batch={total_batch}") log.info(f"[{target_batch}/{total_batch}] Serial search - {perc}% start") res, ssearch_dur = self.serial_search_runner.run() recall, ndcg, p99_latency = res - log.info(f"[{target_batch}/{total_batch}] Serial search - {perc}% done, recall={recall}, ndcg={ndcg}, p99={p99_latency}, dur={ssearch_dur:.4f}") + log.info( + f"[{target_batch}/{total_batch}] Serial search - {perc}% done, recall={recall}, ", + f"ndcg={ndcg}, p99={p99_latency}, dur={ssearch_dur:.4f}", + ) # Search duration for non-last search stage is carefully calculated. # If duration for each concurrency is less than 30s, runner will raise error. if idx < len(self.search_stage) - 1: - total_dur_between_stages = self.data_volume * (self.search_stage[idx + 1] - stage) // self.insert_rate + total_dur_between_stages = self.data_volume * (self.search_stage[idx + 1] - stage) // self.insert_rate csearch_dur = total_dur_between_stages - ssearch_dur # Try to leave room for init process executors @@ -159,14 +182,19 @@ def wait_next_target(start, target_batch) -> bool: each_conc_search_dur = csearch_dur / len(self.concurrencies) if each_conc_search_dur < 30: - warning_msg = f"Results might be inaccurate, duration[{csearch_dur:.4f}] left for conc-search is too short, total available dur={total_dur_between_stages}, serial_search_cost={ssearch_dur}." + warning_msg = ( + f"Results might be inaccurate, duration[{csearch_dur:.4f}] left for conc-search is too short, ", + f"total available dur={total_dur_between_stages}, serial_search_cost={ssearch_dur}.", + ) log.warning(warning_msg) # The last stage else: each_conc_search_dur = 60 - log.info(f"[{target_batch}/{total_batch}] Concurrent search - {perc}% start, dur={each_conc_search_dur:.4f}") + log.info( + f"[{target_batch}/{total_batch}] Concurrent search - {perc}% start, dur={each_conc_search_dur:.4f}", + ) max_qps = self.run_by_dur(each_conc_search_dur) result.append((perc, max_qps, recall, ndcg, p99_latency)) diff --git a/vectordb_bench/backend/runner/serial_runner.py b/vectordb_bench/backend/runner/serial_runner.py index 13270e21a..7eb59432b 100644 --- a/vectordb_bench/backend/runner/serial_runner.py +++ b/vectordb_bench/backend/runner/serial_runner.py @@ -1,20 +1,21 @@ -import time -import logging -import traceback import concurrent -import multiprocessing as mp +import logging import math -import psutil +import multiprocessing as mp +import time +import traceback import numpy as np import pandas as pd +import psutil -from ..clients import api +from vectordb_bench.backend.dataset import DatasetManager + +from ... import config from ...metric import calc_ndcg, calc_recall, get_ideal_dcg from ...models import LoadTimeoutError, PerformanceTimeoutError from .. import utils -from ... import config -from vectordb_bench.backend.dataset import DatasetManager +from ..clients import api NUM_PER_BATCH = config.NUM_PER_BATCH LOAD_MAX_TRY_COUNT = 10 @@ -22,9 +23,16 @@ log = logging.getLogger(__name__) + class SerialInsertRunner: - def __init__(self, db: api.VectorDB, dataset: DatasetManager, normalize: bool, timeout: float | None = None): - self.timeout = timeout if isinstance(timeout, (int, float)) else None + def __init__( + self, + db: api.VectorDB, + dataset: DatasetManager, + normalize: bool, + timeout: float | None = None, + ): + self.timeout = timeout if isinstance(timeout, int | float) else None self.dataset = dataset self.db = db self.normalize = normalize @@ -32,18 +40,20 @@ def __init__(self, db: api.VectorDB, dataset: DatasetManager, normalize: bool, t def task(self) -> int: count = 0 with self.db.init(): - log.info(f"({mp.current_process().name:16}) Start inserting embeddings in batch {config.NUM_PER_BATCH}") + log.info( + f"({mp.current_process().name:16}) Start inserting embeddings in batch {config.NUM_PER_BATCH}", + ) start = time.perf_counter() for data_df in self.dataset: - all_metadata = data_df['id'].tolist() + all_metadata = data_df["id"].tolist() - emb_np = np.stack(data_df['emb']) + emb_np = np.stack(data_df["emb"]) if self.normalize: log.debug("normalize the 100k train data") all_embeddings = (emb_np / np.linalg.norm(emb_np, axis=1)[:, np.newaxis]).tolist() else: all_embeddings = emb_np.tolist() - del(emb_np) + del emb_np log.debug(f"batch dataset size: {len(all_embeddings)}, {len(all_metadata)}") insert_count, error = self.db.insert_embeddings( @@ -56,30 +66,41 @@ def task(self) -> int: assert insert_count == len(all_metadata) count += insert_count if count % 100_000 == 0: - log.info(f"({mp.current_process().name:16}) Loaded {count} embeddings into VectorDB") + log.info( + f"({mp.current_process().name:16}) Loaded {count} embeddings into VectorDB", + ) - log.info(f"({mp.current_process().name:16}) Finish loading all dataset into VectorDB, dur={time.perf_counter()-start}") + log.info( + f"({mp.current_process().name:16}) Finish loading all dataset into VectorDB, ", + f"dur={time.perf_counter()-start}", + ) return count - def endless_insert_data(self, all_embeddings, all_metadata, left_id: int = 0) -> int: + def endless_insert_data(self, all_embeddings: list, all_metadata: list, left_id: int = 0) -> int: with self.db.init(): # unique id for endlessness insertion - all_metadata = [i+left_id for i in all_metadata] + all_metadata = [i + left_id for i in all_metadata] - NUM_BATCHES = math.ceil(len(all_embeddings)/NUM_PER_BATCH) - log.info(f"({mp.current_process().name:16}) Start inserting {len(all_embeddings)} embeddings in batch {NUM_PER_BATCH}") + num_batches = math.ceil(len(all_embeddings) / NUM_PER_BATCH) + log.info( + f"({mp.current_process().name:16}) Start inserting {len(all_embeddings)} ", + f"embeddings in batch {NUM_PER_BATCH}", + ) count = 0 - for batch_id in range(NUM_BATCHES): + for batch_id in range(num_batches): retry_count = 0 already_insert_count = 0 - metadata = all_metadata[batch_id*NUM_PER_BATCH : (batch_id+1)*NUM_PER_BATCH] - embeddings = all_embeddings[batch_id*NUM_PER_BATCH : (batch_id+1)*NUM_PER_BATCH] + metadata = all_metadata[batch_id * NUM_PER_BATCH : (batch_id + 1) * NUM_PER_BATCH] + embeddings = all_embeddings[batch_id * NUM_PER_BATCH : (batch_id + 1) * NUM_PER_BATCH] - log.debug(f"({mp.current_process().name:16}) batch [{batch_id:3}/{NUM_BATCHES}], Start inserting {len(metadata)} embeddings") + log.debug( + f"({mp.current_process().name:16}) batch [{batch_id:3}/{num_batches}], ", + f"Start inserting {len(metadata)} embeddings", + ) while retry_count < LOAD_MAX_TRY_COUNT: insert_count, error = self.db.insert_embeddings( - embeddings=embeddings[already_insert_count :], - metadata=metadata[already_insert_count :], + embeddings=embeddings[already_insert_count:], + metadata=metadata[already_insert_count:], ) already_insert_count += insert_count if error is not None: @@ -91,17 +112,26 @@ def endless_insert_data(self, all_embeddings, all_metadata, left_id: int = 0) -> raise error else: break - log.debug(f"({mp.current_process().name:16}) batch [{batch_id:3}/{NUM_BATCHES}], Finish inserting {len(metadata)} embeddings") + log.debug( + f"({mp.current_process().name:16}) batch [{batch_id:3}/{num_batches}], ", + f"Finish inserting {len(metadata)} embeddings", + ) assert already_insert_count == len(metadata) count += already_insert_count - log.info(f"({mp.current_process().name:16}) Finish inserting {len(all_embeddings)} embeddings in batch {NUM_PER_BATCH}") + log.info( + f"({mp.current_process().name:16}) Finish inserting {len(all_embeddings)} embeddings in ", + f"batch {NUM_PER_BATCH}", + ) return count @utils.time_it def _insert_all_batches(self) -> int: """Performance case only""" - with concurrent.futures.ProcessPoolExecutor(mp_context=mp.get_context('spawn'), max_workers=1) as executor: + with concurrent.futures.ProcessPoolExecutor( + mp_context=mp.get_context("spawn"), + max_workers=1, + ) as executor: future = executor.submit(self.task) try: count = future.result(timeout=self.timeout) @@ -121,8 +151,11 @@ def run_endlessness(self) -> int: """run forever util DB raises exception or crash""" # datasets for load tests are quite small, can fit into memory # only 1 file - data_df = [data_df for data_df in self.dataset][0] - all_embeddings, all_metadata = np.stack(data_df["emb"]).tolist(), data_df['id'].tolist() + data_df = next(iter(self.dataset)) + all_embeddings, all_metadata = ( + np.stack(data_df["emb"]).tolist(), + data_df["id"].tolist(), + ) start_time = time.perf_counter() max_load_count, times = 0, 0 @@ -130,18 +163,26 @@ def run_endlessness(self) -> int: with self.db.init(): self.db.ready_to_load() while time.perf_counter() - start_time < self.timeout: - count = self.endless_insert_data(all_embeddings, all_metadata, left_id=max_load_count) + count = self.endless_insert_data( + all_embeddings, + all_metadata, + left_id=max_load_count, + ) max_load_count += count times += 1 - log.info(f"Loaded {times} entire dataset, current max load counts={utils.numerize(max_load_count)}, {max_load_count}") + log.info( + f"Loaded {times} entire dataset, current max load counts={utils.numerize(max_load_count)}, ", + f"{max_load_count}", + ) except Exception as e: - log.info(f"Capacity case load reach limit, insertion counts={utils.numerize(max_load_count)}, {max_load_count}, err={e}") + log.info( + f"Capacity case load reach limit, insertion counts={utils.numerize(max_load_count)}, ", + f"{max_load_count}, err={e}", + ) traceback.print_exc() return max_load_count else: - msg = f"capacity case load timeout in {self.timeout}s" - log.info(msg) - raise LoadTimeoutError(msg) + raise LoadTimeoutError(self.timeout) def run(self) -> int: count, dur = self._insert_all_batches() @@ -168,7 +209,9 @@ def __init__( self.ground_truth = ground_truth def search(self, args: tuple[list, pd.DataFrame]) -> tuple[float, float, float]: - log.info(f"{mp.current_process().name:14} start search the entire test_data to get recall and latency") + log.info( + f"{mp.current_process().name:14} start search the entire test_data to get recall and latency", + ) with self.db.init(): test_data, ground_truth = args ideal_dcg = get_ideal_dcg(self.k) @@ -193,13 +236,15 @@ def search(self, args: tuple[list, pd.DataFrame]) -> tuple[float, float, float]: latencies.append(time.perf_counter() - s) - gt = ground_truth['neighbors_id'][idx] - recalls.append(calc_recall(self.k, gt[:self.k], results)) - ndcgs.append(calc_ndcg(gt[:self.k], results, ideal_dcg)) - + gt = ground_truth["neighbors_id"][idx] + recalls.append(calc_recall(self.k, gt[: self.k], results)) + ndcgs.append(calc_ndcg(gt[: self.k], results, ideal_dcg)) if len(latencies) % 100 == 0: - log.debug(f"({mp.current_process().name:14}) search_count={len(latencies):3}, latest_latency={latencies[-1]}, latest recall={recalls[-1]}") + log.debug( + f"({mp.current_process().name:14}) search_count={len(latencies):3}, ", + f"latest_latency={latencies[-1]}, latest recall={recalls[-1]}", + ) avg_latency = round(np.mean(latencies), 4) avg_recall = round(np.mean(recalls), 4) @@ -213,16 +258,14 @@ def search(self, args: tuple[list, pd.DataFrame]) -> tuple[float, float, float]: f"avg_recall={avg_recall}, " f"avg_ndcg={avg_ndcg}," f"avg_latency={avg_latency}, " - f"p99={p99}" - ) + f"p99={p99}", + ) return (avg_recall, avg_ndcg, p99) - def _run_in_subprocess(self) -> tuple[float, float]: with concurrent.futures.ProcessPoolExecutor(max_workers=1) as executor: future = executor.submit(self.search, (self.test_data, self.ground_truth)) - result = future.result() - return result + return future.result() @utils.time_it def run(self) -> tuple[float, float, float]: diff --git a/vectordb_bench/backend/runner/util.py b/vectordb_bench/backend/runner/util.py index ba1888167..50e91b04b 100644 --- a/vectordb_bench/backend/runner/util.py +++ b/vectordb_bench/backend/runner/util.py @@ -1,13 +1,14 @@ import logging -from pandas import DataFrame import numpy as np +from pandas import DataFrame log = logging.getLogger(__name__) + def get_data(data_df: DataFrame, normalize: bool) -> tuple[list[list[float]], list[str]]: - all_metadata = data_df['id'].tolist() - emb_np = np.stack(data_df['emb']) + all_metadata = data_df["id"].tolist() + emb_np = np.stack(data_df["emb"]) if normalize: log.debug("normalize the 100k train data") all_embeddings = (emb_np / np.linalg.norm(emb_np, axis=1)[:, np.newaxis]).tolist() diff --git a/vectordb_bench/backend/task_runner.py b/vectordb_bench/backend/task_runner.py index 568152ae0..e24d74f03 100644 --- a/vectordb_bench/backend/task_runner.py +++ b/vectordb_bench/backend/task_runner.py @@ -1,24 +1,20 @@ +import concurrent import logging -import psutil import traceback -import concurrent -import numpy as np from enum import Enum, auto -from . import utils -from .cases import Case, CaseLabel -from ..base import BaseModel -from ..models import TaskConfig, PerformanceTimeoutError, TaskStage +import numpy as np +import psutil -from .clients import ( - api, - MetricType -) -from ..metric import Metric -from .runner import MultiProcessingSearchRunner -from .runner import SerialSearchRunner, SerialInsertRunner -from .data_source import DatasetSource +from vectordb_bench.base import BaseModel +from vectordb_bench.metric import Metric +from vectordb_bench.models import PerformanceTimeoutError, TaskConfig, TaskStage +from . import utils +from .cases import Case, CaseLabel +from .clients import MetricType, api +from .data_source import DatasetSource +from .runner import MultiProcessingSearchRunner, SerialInsertRunner, SerialSearchRunner log = logging.getLogger(__name__) @@ -53,24 +49,39 @@ class CaseRunner(BaseModel): search_runner: MultiProcessingSearchRunner | None = None final_search_runner: MultiProcessingSearchRunner | None = None - def __eq__(self, obj): + def __eq__(self, obj: any): if isinstance(obj, CaseRunner): - return self.ca.label == CaseLabel.Performance and \ - self.config.db == obj.config.db and \ - self.config.db_case_config == obj.config.db_case_config and \ - self.ca.dataset == obj.ca.dataset + return ( + self.ca.label == CaseLabel.Performance + and self.config.db == obj.config.db + and self.config.db_case_config == obj.config.db_case_config + and self.ca.dataset == obj.ca.dataset + ) return False def display(self) -> dict: - c_dict = self.ca.dict(include={'label':True, 'filters': True,'dataset':{'data': {'name': True, 'size': True, 'dim': True, 'metric_type': True, 'label': True}} }) - c_dict['db'] = self.config.db_name + c_dict = self.ca.dict( + include={ + "label": True, + "filters": True, + "dataset": { + "data": { + "name": True, + "size": True, + "dim": True, + "metric_type": True, + "label": True, + }, + }, + }, + ) + c_dict["db"] = self.config.db_name return c_dict @property def normalize(self) -> bool: assert self.db - return self.db.need_normalize_cosine() and \ - self.ca.dataset.data.metric_type == MetricType.COSINE + return self.db.need_normalize_cosine() and self.ca.dataset.data.metric_type == MetricType.COSINE def init_db(self, drop_old: bool = True) -> None: db_cls = self.config.db.init_cls @@ -80,8 +91,7 @@ def init_db(self, drop_old: bool = True) -> None: db_config=self.config.db_config.to_dict(), db_case_config=self.config.db_case_config, drop_old=drop_old, - ) # type:ignore - + ) def _pre_run(self, drop_old: bool = True): try: @@ -89,12 +99,9 @@ def _pre_run(self, drop_old: bool = True): self.ca.dataset.prepare(self.dataset_source, filters=self.ca.filter_rate) except ModuleNotFoundError as e: log.warning( - f"pre run case error: please install client for db: {self.config.db}, error={e}" + f"pre run case error: please install client for db: {self.config.db}, error={e}", ) raise e from None - except Exception as e: - log.warning(f"pre run case error: {e}") - raise e from None def run(self, drop_old: bool = True) -> Metric: log.info("Starting run") @@ -103,12 +110,11 @@ def run(self, drop_old: bool = True) -> Metric: if self.ca.label == CaseLabel.Load: return self._run_capacity_case() - elif self.ca.label == CaseLabel.Performance: + if self.ca.label == CaseLabel.Performance: return self._run_perf_case(drop_old) - else: - msg = f"unknown case type: {self.ca.label}" - log.warning(msg) - raise ValueError(msg) + msg = f"unknown case type: {self.ca.label}" + log.warning(msg) + raise ValueError(msg) def _run_capacity_case(self) -> Metric: """run capacity cases @@ -120,7 +126,10 @@ def _run_capacity_case(self) -> Metric: log.info("Start capacity case") try: runner = SerialInsertRunner( - self.db, self.ca.dataset, self.normalize, self.ca.load_timeout + self.db, + self.ca.dataset, + self.normalize, + self.ca.load_timeout, ) count = runner.run_endlessness() except Exception as e: @@ -128,7 +137,7 @@ def _run_capacity_case(self) -> Metric: raise e from None else: log.info( - f"Capacity case loading dataset reaches VectorDB's limit: max capacity = {count}" + f"Capacity case loading dataset reaches VectorDB's limit: max capacity = {count}", ) return Metric(max_load_count=count) @@ -138,7 +147,7 @@ def _run_perf_case(self, drop_old: bool = True) -> Metric: Returns: Metric: load_duration, recall, serial_latency_p99, and, qps """ - ''' + """ if drop_old: _, load_dur = self._load_train_data() build_dur = self._optimize() @@ -153,38 +162,40 @@ def _run_perf_case(self, drop_old: bool = True) -> Metric: m.qps, m.conc_num_list, m.conc_qps_list, m.conc_latency_p99_list = self._conc_search() m.recall, m.serial_latency_p99 = self._serial_search() - ''' + """ log.info("Start performance case") try: m = Metric() if drop_old: if TaskStage.LOAD in self.config.stages: - # self._load_train_data() _, load_dur = self._load_train_data() build_dur = self._optimize() m.load_duration = round(load_dur + build_dur, 4) log.info( f"Finish loading the entire dataset into VectorDB," f" insert_duration={load_dur}, optimize_duration={build_dur}" - f" load_duration(insert + optimize) = {m.load_duration}" + f" load_duration(insert + optimize) = {m.load_duration}", ) else: log.info("Data loading skipped") - if ( - TaskStage.SEARCH_SERIAL in self.config.stages - or TaskStage.SEARCH_CONCURRENT in self.config.stages - ): + if TaskStage.SEARCH_SERIAL in self.config.stages or TaskStage.SEARCH_CONCURRENT in self.config.stages: self._init_search_runner() if TaskStage.SEARCH_CONCURRENT in self.config.stages: search_results = self._conc_search() - m.qps, m.conc_num_list, m.conc_qps_list, m.conc_latency_p99_list, m.conc_latency_avg_list = search_results + ( + m.qps, + m.conc_num_list, + m.conc_qps_list, + m.conc_latency_p99_list, + m.conc_latency_avg_list, + ) = search_results if TaskStage.SEARCH_SERIAL in self.config.stages: search_results = self._serial_search() - ''' + """ m.recall = search_results.recall m.serial_latencies = search_results.serial_latencies - ''' + """ m.recall, m.ndcg, m.serial_latency_p99 = search_results except Exception as e: @@ -199,7 +210,12 @@ def _run_perf_case(self, drop_old: bool = True) -> Metric: def _load_train_data(self): """Insert train data and get the insert_duration""" try: - runner = SerialInsertRunner(self.db, self.ca.dataset, self.normalize, self.ca.load_timeout) + runner = SerialInsertRunner( + self.db, + self.ca.dataset, + self.normalize, + self.ca.load_timeout, + ) runner.run() except Exception as e: raise e from None @@ -215,11 +231,12 @@ def _serial_search(self) -> tuple[float, float, float]: """ try: results, _ = self.serial_search_runner.run() - return results except Exception as e: - log.warning(f"search error: {str(e)}, {e}") + log.warning(f"search error: {e!s}, {e}") self.stop() - raise e from None + raise e from e + else: + return results def _conc_search(self): """Performance concurrency tests, search the test data endlessness @@ -231,7 +248,7 @@ def _conc_search(self): try: return self.search_runner.run() except Exception as e: - log.warning(f"search error: {str(e)}, {e}") + log.warning(f"search error: {e!s}, {e}") raise e from None finally: self.stop() @@ -250,7 +267,7 @@ def _optimize(self) -> float: log.warning(f"VectorDB optimize timeout in {self.ca.optimize_timeout}") for pid, _ in executor._processes.items(): psutil.Process(pid).kill() - raise PerformanceTimeoutError("Performance case optimize timeout") from e + raise PerformanceTimeoutError from e except Exception as e: log.warning(f"VectorDB optimize error: {e}") raise e from None @@ -286,6 +303,16 @@ def stop(self): self.search_runner.stop() +DATA_FORMAT = " %-14s | %-12s %-20s %7s | %-10s" +TITLE_FORMAT = (" %-14s | %-12s %-20s %7s | %-10s") % ( + "DB", + "CaseType", + "Dataset", + "Filter", + "task_label", +) + + class TaskRunner(BaseModel): run_id: str task_label: str @@ -304,18 +331,8 @@ def _get_num_by_status(self, status: RunningStatus) -> int: return sum([1 for c in self.case_runners if c.status == status]) def display(self) -> None: - DATA_FORMAT = (" %-14s | %-12s %-20s %7s | %-10s") - TITLE_FORMAT = (" %-14s | %-12s %-20s %7s | %-10s") % ( - "DB", "CaseType", "Dataset", "Filter", "task_label") - fmt = [TITLE_FORMAT] - fmt.append(DATA_FORMAT%( - "-"*11, - "-"*12, - "-"*20, - "-"*7, - "-"*7 - )) + fmt.append(DATA_FORMAT % ("-" * 11, "-" * 12, "-" * 20, "-" * 7, "-" * 7)) for f in self.case_runners: if f.ca.filter_rate != 0.0: @@ -326,13 +343,16 @@ def display(self) -> None: filters = "None" ds_str = f"{f.ca.dataset.data.name}-{f.ca.dataset.data.label}-{utils.numerize(f.ca.dataset.data.size)}" - fmt.append(DATA_FORMAT%( - f.config.db_name, - f.ca.label.name, - ds_str, - filters, - self.task_label, - )) + fmt.append( + DATA_FORMAT + % ( + f.config.db_name, + f.ca.label.name, + ds_str, + filters, + self.task_label, + ), + ) tmp_logger = logging.getLogger("no_color") for f in fmt: diff --git a/vectordb_bench/backend/utils.py b/vectordb_bench/backend/utils.py index ea11e6461..86c4faf5e 100644 --- a/vectordb_bench/backend/utils.py +++ b/vectordb_bench/backend/utils.py @@ -2,7 +2,7 @@ from functools import wraps -def numerize(n) -> str: +def numerize(n: int) -> str: """display positive number n for readability Examples: @@ -16,32 +16,34 @@ def numerize(n) -> str: "K": 1e6, "M": 1e9, "B": 1e12, - "END": float('inf'), + "END": float("inf"), } display_n, sufix = n, "" for s, base in sufix2upbound.items(): # number >= 1000B will alway have sufix 'B' if s == "END": - display_n = int(n/1e9) + display_n = int(n / 1e9) sufix = "B" break if n < base: sufix = "" if s == "EMPTY" else s - display_n = int(n/(base/1e3)) + display_n = int(n / (base / 1e3)) break return f"{display_n}{sufix}" -def time_it(func): - """ returns result and elapsed time""" +def time_it(func: any): + """returns result and elapsed time""" + @wraps(func) def inner(*args, **kwargs): pref = time.perf_counter() result = func(*args, **kwargs) delta = time.perf_counter() - pref return result, delta + return inner @@ -62,14 +64,19 @@ def compose_train_files(train_count: int, use_shuffled: bool) -> list[str]: return train_files -def compose_gt_file(filters: int | float | str | None = None) -> str: +ONE_PERCENT = 0.01 +NINETY_NINE_PERCENT = 0.99 + + +def compose_gt_file(filters: float | str | None = None) -> str: if filters is None: return "neighbors.parquet" - if filters == 0.01: + if filters == ONE_PERCENT: return "neighbors_head_1p.parquet" - if filters == 0.99: + if filters == NINETY_NINE_PERCENT: return "neighbors_tail_1p.parquet" - raise ValueError(f"Filters not supported: {filters}") + msg = f"Filters not supported: {filters}" + raise ValueError(msg) diff --git a/vectordb_bench/base.py b/vectordb_bench/base.py index 3c71fb5a7..502d5fa49 100644 --- a/vectordb_bench/base.py +++ b/vectordb_bench/base.py @@ -3,4 +3,3 @@ class BaseModel(PydanticBaseModel, arbitrary_types_allowed=True): pass - diff --git a/vectordb_bench/cli/cli.py b/vectordb_bench/cli/cli.py index e5b9a5fe2..3bb7763d8 100644 --- a/vectordb_bench/cli/cli.py +++ b/vectordb_bench/cli/cli.py @@ -1,27 +1,27 @@ import logging +import os import time +from collections.abc import Callable from concurrent.futures import wait from datetime import datetime from pprint import pformat from typing import ( Annotated, - Callable, - List, - Optional, - Type, + Any, TypedDict, Unpack, get_origin, get_type_hints, - Dict, - Any, ) + import click +from yaml import load from vectordb_bench.backend.clients.api import MetricType + from .. import config from ..backend.clients import DB -from ..interface import benchMarkRunner, global_result_future +from ..interface import benchmark_runner, global_result_future from ..models import ( CaseConfig, CaseType, @@ -31,8 +31,7 @@ TaskConfig, TaskStage, ) -import os -from yaml import load + try: from yaml import CLoader as Loader except ImportError: @@ -46,8 +45,8 @@ def click_get_defaults_from_file(ctx, param, value): else: input_file = os.path.join(config.CONFIG_LOCAL_DIR, value) try: - with open(input_file, 'r') as f: - _config: Dict[str, Dict[str, Any]] = load(f.read(), Loader=Loader) + with open(input_file) as f: + _config: dict[str, dict[str, Any]] = load(f.read(), Loader=Loader) ctx.default_map = _config.get(ctx.command.name, {}) except Exception as e: raise click.BadParameter(f"Failed to load config file: {e}") @@ -55,7 +54,7 @@ def click_get_defaults_from_file(ctx, param, value): def click_parameter_decorators_from_typed_dict( - typed_dict: Type, + typed_dict: type, ) -> Callable[[click.decorators.FC], click.decorators.FC]: """A convenience method decorator that will read in a TypedDict with parameters defined by Annotated types. from .models import CaseConfig, CaseType, DBCaseConfig, DBConfig, TaskConfig, TaskStage @@ -91,15 +90,12 @@ def foo(**parameters: Unpack[FooTypedDict]): decorators = [] for _, t in get_type_hints(typed_dict, include_extras=True).items(): assert get_origin(t) is Annotated - if ( - len(t.__metadata__) == 1 - and t.__metadata__[0].__module__ == "click.decorators" - ): + if len(t.__metadata__) == 1 and t.__metadata__[0].__module__ == "click.decorators": # happy path -- only accept Annotated[..., Union[click.option,click.argument,...]] with no additional metadata defined (len=1) decorators.append(t.__metadata__[0]) else: raise RuntimeError( - "Click-TypedDict decorator parsing must only contain root type and a click decorator like click.option. See docstring" + "Click-TypedDict decorator parsing must only contain root type and a click decorator like click.option. See docstring", ) def deco(f): @@ -132,11 +128,11 @@ def parse_task_stages( load: bool, search_serial: bool, search_concurrent: bool, -) -> List[TaskStage]: +) -> list[TaskStage]: stages = [] if load and not drop_old: raise RuntimeError("Dropping old data cannot be skipped if loading data") - elif drop_old and not load: + if drop_old and not load: raise RuntimeError("Load cannot be skipped if dropping old data") if drop_old: stages.append(TaskStage.DROP_OLD) @@ -149,12 +145,19 @@ def parse_task_stages( return stages -def check_custom_case_parameters(ctx, param, value): - if ctx.params.get("case_type") == "PerformanceCustomDataset": - if value is None: - raise click.BadParameter("Custom case parameters\ - \n--custom-case-name\n--custom-dataset-name\n--custom-dataset-dir\n--custom-dataset-size \ - \n--custom-dataset-dim\n--custom-dataset-file-count\n are required") +# ruff: noqa +def check_custom_case_parameters(ctx: any, param: any, value: any): + if ctx.params.get("case_type") == "PerformanceCustomDataset" and value is None: + raise click.BadParameter( + """ Custom case parameters +--custom-case-name +--custom-dataset-name +--custom-dataset-dir +--custom-dataset-sizes +--custom-dataset-dim +--custom-dataset-file-count +are required """, + ) return value @@ -175,7 +178,7 @@ def get_custom_case_config(parameters: dict) -> dict: "file_count": parameters["custom_dataset_file_count"], "use_shuffled": parameters["custom_dataset_use_shuffled"], "with_gt": parameters["custom_dataset_with_gt"], - } + }, } return custom_case_config @@ -186,12 +189,14 @@ def get_custom_case_config(parameters: dict) -> dict: class CommonTypedDict(TypedDict): config_file: Annotated[ bool, - click.option('--config-file', - type=click.Path(), - callback=click_get_defaults_from_file, - is_eager=True, - expose_value=False, - help='Read configuration from yaml file'), + click.option( + "--config-file", + type=click.Path(), + callback=click_get_defaults_from_file, + is_eager=True, + expose_value=False, + help="Read configuration from yaml file", + ), ] drop_old: Annotated[ bool, @@ -246,9 +251,11 @@ class CommonTypedDict(TypedDict): db_label: Annotated[ str, click.option( - "--db-label", type=str, help="Db label, default: date in ISO format", + "--db-label", + type=str, + help="Db label, default: date in ISO format", show_default=True, - default=datetime.now().isoformat() + default=datetime.now().isoformat(), ), ] dry_run: Annotated[ @@ -282,7 +289,7 @@ class CommonTypedDict(TypedDict): ), ] num_concurrency: Annotated[ - List[str], + list[str], click.option( "--num-concurrency", type=str, @@ -298,7 +305,7 @@ class CommonTypedDict(TypedDict): "--custom-case-name", help="Custom dataset case name", callback=check_custom_case_parameters, - ) + ), ] custom_case_description: Annotated[ str, @@ -307,7 +314,7 @@ class CommonTypedDict(TypedDict): help="Custom dataset case description", default="This is a customized dataset.", show_default=True, - ) + ), ] custom_case_load_timeout: Annotated[ int, @@ -316,7 +323,7 @@ class CommonTypedDict(TypedDict): help="Custom dataset case load timeout", default=36000, show_default=True, - ) + ), ] custom_case_optimize_timeout: Annotated[ int, @@ -325,7 +332,7 @@ class CommonTypedDict(TypedDict): help="Custom dataset case optimize timeout", default=36000, show_default=True, - ) + ), ] custom_dataset_name: Annotated[ str, @@ -397,60 +404,60 @@ class CommonTypedDict(TypedDict): class HNSWBaseTypedDict(TypedDict): - m: Annotated[Optional[int], click.option("--m", type=int, help="hnsw m")] + m: Annotated[int | None, click.option("--m", type=int, help="hnsw m")] ef_construction: Annotated[ - Optional[int], + int | None, click.option("--ef-construction", type=int, help="hnsw ef-construction"), ] class HNSWBaseRequiredTypedDict(TypedDict): - m: Annotated[Optional[int], click.option("--m", type=int, help="hnsw m", required=True)] + m: Annotated[int | None, click.option("--m", type=int, help="hnsw m", required=True)] ef_construction: Annotated[ - Optional[int], + int | None, click.option("--ef-construction", type=int, help="hnsw ef-construction", required=True), ] class HNSWFlavor1(HNSWBaseTypedDict): ef_search: Annotated[ - Optional[int], click.option("--ef-search", type=int, help="hnsw ef-search", is_eager=True) + int | None, + click.option("--ef-search", type=int, help="hnsw ef-search", is_eager=True), ] class HNSWFlavor2(HNSWBaseTypedDict): ef_runtime: Annotated[ - Optional[int], click.option("--ef-runtime", type=int, help="hnsw ef-runtime") + int | None, + click.option("--ef-runtime", type=int, help="hnsw ef-runtime"), ] class HNSWFlavor3(HNSWBaseRequiredTypedDict): ef_search: Annotated[ - Optional[int], click.option("--ef-search", type=int, help="hnsw ef-search", required=True) + int | None, + click.option("--ef-search", type=int, help="hnsw ef-search", required=True), ] class IVFFlatTypedDict(TypedDict): - lists: Annotated[ - Optional[int], click.option("--lists", type=int, help="ivfflat lists") - ] - probes: Annotated[ - Optional[int], click.option("--probes", type=int, help="ivfflat probes") - ] + lists: Annotated[int | None, click.option("--lists", type=int, help="ivfflat lists")] + probes: Annotated[int | None, click.option("--probes", type=int, help="ivfflat probes")] class IVFFlatTypedDictN(TypedDict): nlist: Annotated[ - Optional[int], click.option("--lists", "nlist", type=int, help="ivfflat lists", required=True) + int | None, + click.option("--lists", "nlist", type=int, help="ivfflat lists", required=True), ] nprobe: Annotated[ - Optional[int], click.option("--probes", "nprobe", type=int, help="ivfflat probes", required=True) + int | None, + click.option("--probes", "nprobe", type=int, help="ivfflat probes", required=True), ] @click.group() -def cli(): - ... +def cli(): ... def run( @@ -482,9 +489,7 @@ def run( custom_case=get_custom_case_config(parameters), ), stages=parse_task_stages( - ( - False if not parameters["load"] else parameters["drop_old"] - ), # only drop old data if loading new data + (False if not parameters["load"] else parameters["drop_old"]), # only drop old data if loading new data parameters["load"], parameters["search_serial"], parameters["search_concurrent"], @@ -493,7 +498,7 @@ def run( log.info(f"Task:\n{pformat(task)}\n") if not parameters["dry_run"]: - benchMarkRunner.run([task]) + benchmark_runner.run([task]) time.sleep(5) if global_result_future: wait([global_result_future]) diff --git a/vectordb_bench/cli/vectordbbench.py b/vectordb_bench/cli/vectordbbench.py index f9ad69ceb..5e3798691 100644 --- a/vectordb_bench/cli/vectordbbench.py +++ b/vectordb_bench/cli/vectordbbench.py @@ -1,16 +1,15 @@ -from ..backend.clients.pgvector.cli import PgVectorHNSW +from ..backend.clients.alloydb.cli import AlloyDBScaNN +from ..backend.clients.aws_opensearch.cli import AWSOpenSearch +from ..backend.clients.memorydb.cli import MemoryDB +from ..backend.clients.milvus.cli import MilvusAutoIndex +from ..backend.clients.pgdiskann.cli import PgDiskAnn from ..backend.clients.pgvecto_rs.cli import PgVectoRSHNSW, PgVectoRSIVFFlat +from ..backend.clients.pgvector.cli import PgVectorHNSW from ..backend.clients.pgvectorscale.cli import PgVectorScaleDiskAnn -from ..backend.clients.pgdiskann.cli import PgDiskAnn from ..backend.clients.redis.cli import Redis -from ..backend.clients.memorydb.cli import MemoryDB from ..backend.clients.test.cli import Test from ..backend.clients.weaviate_cloud.cli import Weaviate from ..backend.clients.zilliz_cloud.cli import ZillizAutoIndex -from ..backend.clients.milvus.cli import MilvusAutoIndex -from ..backend.clients.aws_opensearch.cli import AWSOpenSearch -from ..backend.clients.alloydb.cli import AlloyDBScaNN - from .cli import cli cli.add_command(PgVectorHNSW) diff --git a/vectordb_bench/frontend/components/check_results/charts.py b/vectordb_bench/frontend/components/check_results/charts.py index 9e869b479..0e74d2752 100644 --- a/vectordb_bench/frontend/components/check_results/charts.py +++ b/vectordb_bench/frontend/components/check_results/charts.py @@ -1,8 +1,7 @@ -from vectordb_bench.backend.cases import Case from vectordb_bench.frontend.components.check_results.expanderStyle import ( initMainExpanderStyle, ) -from vectordb_bench.metric import metricOrder, isLowerIsBetterMetric, metricUnitMap +from vectordb_bench.metric import metric_order, isLowerIsBetterMetric, metric_unit_map from vectordb_bench.frontend.config.styles import * from vectordb_bench.models import ResultLabel import plotly.express as px @@ -21,9 +20,7 @@ def drawCharts(st, allData, failedTasks, caseNames: list[str]): def showFailedDBs(st, errorDBs): failedDBs = [db for db, label in errorDBs.items() if label == ResultLabel.FAILED] - timeoutDBs = [ - db for db, label in errorDBs.items() if label == ResultLabel.OUTOFRANGE - ] + timeoutDBs = [db for db, label in errorDBs.items() if label == ResultLabel.OUTOFRANGE] showFailedText(st, "Failed", failedDBs) showFailedText(st, "Timeout", timeoutDBs) @@ -41,7 +38,7 @@ def drawChart(data, st, key_prefix: str): metricsSet = set() for d in data: metricsSet = metricsSet.union(d["metricsSet"]) - showMetrics = [metric for metric in metricOrder if metric in metricsSet] + showMetrics = [metric for metric in metric_order if metric in metricsSet] for i, metric in enumerate(showMetrics): container = st.container() @@ -72,9 +69,7 @@ def getLabelToShapeMap(data): else: usedShapes.add(labelIndexMap[label] % len(PATTERN_SHAPES)) - labelToShapeMap = { - label: getPatternShape(index) for label, index in labelIndexMap.items() - } + labelToShapeMap = {label: getPatternShape(index) for label, index in labelIndexMap.items()} return labelToShapeMap @@ -96,11 +91,9 @@ def drawMetricChart(data, metric, st, key: str): xpadding = (xmax - xmin) / 16 xpadding_multiplier = 1.8 xrange = [xmin, xmax + xpadding * xpadding_multiplier] - unit = metricUnitMap.get(metric, "") + unit = metric_unit_map.get(metric, "") labelToShapeMap = getLabelToShapeMap(dataWithMetric) - categoryorder = ( - "total descending" if isLowerIsBetterMetric(metric) else "total ascending" - ) + categoryorder = "total descending" if isLowerIsBetterMetric(metric) else "total ascending" fig = px.bar( dataWithMetric, x=metric, @@ -137,18 +130,14 @@ def drawMetricChart(data, metric, st, key: str): color="#333", size=12, ), - marker=dict( - pattern=dict(fillmode="overlay", fgcolor="#fff", fgopacity=1, size=7) - ), + marker=dict(pattern=dict(fillmode="overlay", fgcolor="#fff", fgopacity=1, size=7)), texttemplate="%{x:,.4~r}" + unit, ) fig.update_layout( margin=dict(l=0, r=0, t=48, b=12, pad=8), bargap=0.25, showlegend=False, - legend=dict( - orientation="h", yanchor="bottom", y=1, xanchor="right", x=1, title="" - ), + legend=dict(orientation="h", yanchor="bottom", y=1, xanchor="right", x=1, title=""), # legend=dict(orientation="v", title=""), yaxis={"categoryorder": categoryorder}, title=dict( diff --git a/vectordb_bench/frontend/components/check_results/data.py b/vectordb_bench/frontend/components/check_results/data.py index b3cac21e1..94d1b4eab 100644 --- a/vectordb_bench/frontend/components/check_results/data.py +++ b/vectordb_bench/frontend/components/check_results/data.py @@ -1,6 +1,5 @@ from collections import defaultdict from dataclasses import asdict -from vectordb_bench.backend.cases import Case from vectordb_bench.metric import isLowerIsBetterMetric from vectordb_bench.models import CaseResult, ResultLabel @@ -24,10 +23,7 @@ def getFilterTasks( task for task in tasks if task.task_config.db_name in dbNames - and task.task_config.case_config.case_id.case_cls( - task.task_config.case_config.custom_case - ).name - in caseNames + and task.task_config.case_config.case_id.case_cls(task.task_config.case_config.custom_case).name in caseNames ] return filterTasks @@ -39,9 +35,7 @@ def mergeTasks(tasks: list[CaseResult]): db = task.task_config.db.value db_label = task.task_config.db_config.db_label or "" version = task.task_config.db_config.version or "" - case = task.task_config.case_config.case_id.case_cls( - task.task_config.case_config.custom_case - ) + case = task.task_config.case_config.case_id.case_cls(task.task_config.case_config.custom_case) dbCaseMetricsMap[db_name][case.name] = { "db": db, "db_label": db_label, @@ -86,9 +80,7 @@ def mergeTasks(tasks: list[CaseResult]): def mergeMetrics(metrics_1: dict, metrics_2: dict) -> dict: metrics = {**metrics_1} for key, value in metrics_2.items(): - metrics[key] = ( - getBetterMetric(key, value, metrics[key]) if key in metrics else value - ) + metrics[key] = getBetterMetric(key, value, metrics[key]) if key in metrics else value return metrics @@ -99,11 +91,7 @@ def getBetterMetric(metric, value_1, value_2): return value_2 if value_2 < 1e-7: return value_1 - return ( - min(value_1, value_2) - if isLowerIsBetterMetric(metric) - else max(value_1, value_2) - ) + return min(value_1, value_2) if isLowerIsBetterMetric(metric) else max(value_1, value_2) except Exception: return value_1 diff --git a/vectordb_bench/frontend/components/check_results/filters.py b/vectordb_bench/frontend/components/check_results/filters.py index e60efb2e1..129c1d5ae 100644 --- a/vectordb_bench/frontend/components/check_results/filters.py +++ b/vectordb_bench/frontend/components/check_results/filters.py @@ -20,23 +20,17 @@ def getshownData(results: list[TestResult], st): shownResults = getshownResults(results, st) showDBNames, showCaseNames = getShowDbsAndCases(shownResults, st) - shownData, failedTasks = getChartData( - shownResults, showDBNames, showCaseNames) + shownData, failedTasks = getChartData(shownResults, showDBNames, showCaseNames) return shownData, failedTasks, showCaseNames def getshownResults(results: list[TestResult], st) -> list[CaseResult]: resultSelectOptions = [ - result.task_label - if result.task_label != result.run_id - else f"res-{result.run_id[:4]}" - for result in results + result.task_label if result.task_label != result.run_id else f"res-{result.run_id[:4]}" for result in results ] if len(resultSelectOptions) == 0: - st.write( - "There are no results to display. Please wait for the task to complete or run a new task." - ) + st.write("There are no results to display. Please wait for the task to complete or run a new task.") return [] selectedResultSelectedOptions = st.multiselect( @@ -58,13 +52,12 @@ def getShowDbsAndCases(result: list[CaseResult], st) -> tuple[list[str], list[st allDbNames = list(set({res.task_config.db_name for res in result})) allDbNames.sort() allCases: list[Case] = [ - res.task_config.case_config.case_id.case_cls( - res.task_config.case_config.custom_case) - for res in result + res.task_config.case_config.case_id.case_cls(res.task_config.case_config.custom_case) for res in result ] allCaseNameSet = set({case.name for case in allCases}) - allCaseNames = [case_name for case_name in CASE_NAME_ORDER if case_name in allCaseNameSet] + \ - [case_name for case_name in allCaseNameSet if case_name not in CASE_NAME_ORDER] + allCaseNames = [case_name for case_name in CASE_NAME_ORDER if case_name in allCaseNameSet] + [ + case_name for case_name in allCaseNameSet if case_name not in CASE_NAME_ORDER + ] # DB Filter dbFilterContainer = st.container() @@ -120,8 +113,7 @@ def filterView(container, header, options, col, optionLables=None): ) if optionLables is None: optionLables = options - isActive = {option: st.session_state[selectAllState] - for option in optionLables} + isActive = {option: st.session_state[selectAllState] for option in optionLables} for i, option in enumerate(optionLables): isActive[option] = columns[i % col].checkbox( optionLables[i], diff --git a/vectordb_bench/frontend/components/check_results/nav.py b/vectordb_bench/frontend/components/check_results/nav.py index ad070ab35..f95e43d7a 100644 --- a/vectordb_bench/frontend/components/check_results/nav.py +++ b/vectordb_bench/frontend/components/check_results/nav.py @@ -7,15 +7,15 @@ def NavToRunTest(st): navClick = st.button("Run Your Test   >") if navClick: switch_page("run test") - - + + def NavToQuriesPerDollar(st): st.subheader("Compare qps with price.") navClick = st.button("QP$ (Quries per Dollar)   >") if navClick: switch_page("quries_per_dollar") - - + + def NavToResults(st, key="nav-to-results"): navClick = st.button("<   Back to Results", key=key) if navClick: diff --git a/vectordb_bench/frontend/components/check_results/priceTable.py b/vectordb_bench/frontend/components/check_results/priceTable.py index 06d7a6a6b..f2c0ae001 100644 --- a/vectordb_bench/frontend/components/check_results/priceTable.py +++ b/vectordb_bench/frontend/components/check_results/priceTable.py @@ -7,9 +7,7 @@ def priceTable(container, data): - dbAndLabelSet = { - (d["db"], d["db_label"]) for d in data if d["db"] != DB.Milvus.value - } + dbAndLabelSet = {(d["db"], d["db_label"]) for d in data if d["db"] != DB.Milvus.value} dbAndLabelList = list(dbAndLabelSet) dbAndLabelList.sort() diff --git a/vectordb_bench/frontend/components/check_results/stPageConfig.py b/vectordb_bench/frontend/components/check_results/stPageConfig.py index c03cceab4..9521be285 100644 --- a/vectordb_bench/frontend/components/check_results/stPageConfig.py +++ b/vectordb_bench/frontend/components/check_results/stPageConfig.py @@ -9,10 +9,11 @@ def initResultsPageConfig(st): # initial_sidebar_state="collapsed", ) + def initRunTestPageConfig(st): st.set_page_config( page_title=PAGE_TITLE, page_icon=FAVICON, # layout="wide", initial_sidebar_state="collapsed", - ) \ No newline at end of file + ) diff --git a/vectordb_bench/frontend/components/concurrent/charts.py b/vectordb_bench/frontend/components/concurrent/charts.py index 11379c4b3..83e2961d6 100644 --- a/vectordb_bench/frontend/components/concurrent/charts.py +++ b/vectordb_bench/frontend/components/concurrent/charts.py @@ -14,24 +14,24 @@ def drawChartsByCase(allData, showCaseNames: list[str], st, latency_type: str): data = [ { "conc_num": caseData["conc_num_list"][i], - "qps": caseData["conc_qps_list"][i] - if 0 <= i < len(caseData["conc_qps_list"]) - else 0, - "latency_p99": caseData["conc_latency_p99_list"][i] * 1000 - if 0 <= i < len(caseData["conc_latency_p99_list"]) - else 0, - "latency_avg": caseData["conc_latency_avg_list"][i] * 1000 - if 0 <= i < len(caseData["conc_latency_avg_list"]) - else 0, + "qps": (caseData["conc_qps_list"][i] if 0 <= i < len(caseData["conc_qps_list"]) else 0), + "latency_p99": ( + caseData["conc_latency_p99_list"][i] * 1000 + if 0 <= i < len(caseData["conc_latency_p99_list"]) + else 0 + ), + "latency_avg": ( + caseData["conc_latency_avg_list"][i] * 1000 + if 0 <= i < len(caseData["conc_latency_avg_list"]) + else 0 + ), "db_name": caseData["db_name"], "db": caseData["db"], } for caseData in caseDataList for i in range(len(caseData["conc_num_list"])) ] - drawChart( - data, chartContainer, key=f"{caseName}-qps-p99", x_metric=latency_type - ) + drawChart(data, chartContainer, key=f"{caseName}-qps-p99", x_metric=latency_type) def getRange(metric, data, padding_multipliers): diff --git a/vectordb_bench/frontend/components/custom/displayCustomCase.py b/vectordb_bench/frontend/components/custom/displayCustomCase.py index 3f5266051..ac111c883 100644 --- a/vectordb_bench/frontend/components/custom/displayCustomCase.py +++ b/vectordb_bench/frontend/components/custom/displayCustomCase.py @@ -1,4 +1,3 @@ - from vectordb_bench.frontend.components.custom.getCustomConfig import CustomCaseConfig @@ -6,26 +5,33 @@ def displayCustomCase(customCase: CustomCaseConfig, st, key): columns = st.columns([1, 2]) customCase.dataset_config.name = columns[0].text_input( - "Name", key=f"{key}_name", value=customCase.dataset_config.name) + "Name", key=f"{key}_name", value=customCase.dataset_config.name + ) customCase.name = f"{customCase.dataset_config.name} (Performace Case)" customCase.dataset_config.dir = columns[1].text_input( - "Folder Path", key=f"{key}_dir", value=customCase.dataset_config.dir) + "Folder Path", key=f"{key}_dir", value=customCase.dataset_config.dir + ) columns = st.columns(4) customCase.dataset_config.dim = columns[0].number_input( - "dim", key=f"{key}_dim", value=customCase.dataset_config.dim) + "dim", key=f"{key}_dim", value=customCase.dataset_config.dim + ) customCase.dataset_config.size = columns[1].number_input( - "size", key=f"{key}_size", value=customCase.dataset_config.size) + "size", key=f"{key}_size", value=customCase.dataset_config.size + ) customCase.dataset_config.metric_type = columns[2].selectbox( - "metric type", key=f"{key}_metric_type", options=["L2", "Cosine", "IP"]) + "metric type", key=f"{key}_metric_type", options=["L2", "Cosine", "IP"] + ) customCase.dataset_config.file_count = columns[3].number_input( - "train file count", key=f"{key}_file_count", value=customCase.dataset_config.file_count) + "train file count", key=f"{key}_file_count", value=customCase.dataset_config.file_count + ) columns = st.columns(4) customCase.dataset_config.use_shuffled = columns[0].checkbox( - "use shuffled data", key=f"{key}_use_shuffled", value=customCase.dataset_config.use_shuffled) + "use shuffled data", key=f"{key}_use_shuffled", value=customCase.dataset_config.use_shuffled + ) customCase.dataset_config.with_gt = columns[1].checkbox( - "with groundtruth", key=f"{key}_with_gt", value=customCase.dataset_config.with_gt) + "with groundtruth", key=f"{key}_with_gt", value=customCase.dataset_config.with_gt + ) - customCase.description = st.text_area( - "description", key=f"{key}_description", value=customCase.description) + customCase.description = st.text_area("description", key=f"{key}_description", value=customCase.description) diff --git a/vectordb_bench/frontend/components/custom/displaypPrams.py b/vectordb_bench/frontend/components/custom/displaypPrams.py index a300b45e1..b677e5909 100644 --- a/vectordb_bench/frontend/components/custom/displaypPrams.py +++ b/vectordb_bench/frontend/components/custom/displaypPrams.py @@ -1,5 +1,6 @@ def displayParams(st): - st.markdown(""" + st.markdown( + """ - `Folder Path` - The path to the folder containing all the files. Please ensure that all files in the folder are in the `Parquet` format. - Vectors data files: The file must be named `train.parquet` and should have two columns: `id` as an incrementing `int` and `emb` as an array of `float32`. - Query test vectors: The file must be named `test.parquet` and should have two columns: `id` as an incrementing `int` and `emb` as an array of `float32`. @@ -8,4 +9,5 @@ def displayParams(st): - `Train File Count` - If the vector file is too large, you can consider splitting it into multiple files. The naming format for the split files should be `train-[index]-of-[file_count].parquet`. For example, `train-01-of-10.parquet` represents the second file (0-indexed) among 10 split files. - `Use Shuffled Data` - If you check this option, the vector data files need to be modified. VectorDBBench will load the data labeled with `shuffle`. For example, use `shuffle_train.parquet` instead of `train.parquet` and `shuffle_train-04-of-10.parquet` instead of `train-04-of-10.parquet`. The `id` column in the shuffled data can be in any order. -""") +""" + ) diff --git a/vectordb_bench/frontend/components/custom/getCustomConfig.py b/vectordb_bench/frontend/components/custom/getCustomConfig.py index ede664b6a..668bfb6d5 100644 --- a/vectordb_bench/frontend/components/custom/getCustomConfig.py +++ b/vectordb_bench/frontend/components/custom/getCustomConfig.py @@ -32,8 +32,7 @@ def get_custom_configs(): def save_custom_configs(custom_configs: list[CustomDatasetConfig]): with open(config.CUSTOM_CONFIG_DIR, "w") as f: - json.dump([custom_config.dict() - for custom_config in custom_configs], f, indent=4) + json.dump([custom_config.dict() for custom_config in custom_configs], f, indent=4) def generate_custom_case(): diff --git a/vectordb_bench/frontend/components/custom/initStyle.py b/vectordb_bench/frontend/components/custom/initStyle.py index 0e5129248..7ecbbf24e 100644 --- a/vectordb_bench/frontend/components/custom/initStyle.py +++ b/vectordb_bench/frontend/components/custom/initStyle.py @@ -12,4 +12,4 @@ def initStyle(st): */ """, unsafe_allow_html=True, - ) \ No newline at end of file + ) diff --git a/vectordb_bench/frontend/components/get_results/saveAsImage.py b/vectordb_bench/frontend/components/get_results/saveAsImage.py index 9820e9575..4be2baec1 100644 --- a/vectordb_bench/frontend/components/get_results/saveAsImage.py +++ b/vectordb_bench/frontend/components/get_results/saveAsImage.py @@ -9,10 +9,12 @@ def load_unpkg(src: str) -> str: return requests.get(src).text + def getResults(container, pageName="vectordb_bench"): container.subheader("Get results") saveAsImage(container, pageName) + def saveAsImage(container, pageName): html2canvasJS = load_unpkg(HTML_2_CANVAS_URL) container.write() diff --git a/vectordb_bench/frontend/components/run_test/caseSelector.py b/vectordb_bench/frontend/components/run_test/caseSelector.py index b25618271..e3d28238d 100644 --- a/vectordb_bench/frontend/components/run_test/caseSelector.py +++ b/vectordb_bench/frontend/components/run_test/caseSelector.py @@ -1,6 +1,4 @@ - from vectordb_bench.frontend.config.styles import * -from vectordb_bench.backend.cases import CaseType from vectordb_bench.frontend.config.dbCaseConfigs import * from collections import defaultdict @@ -23,8 +21,7 @@ def caseSelector(st, activedDbList: list[DB]): dbToCaseConfigs = defaultdict(lambda: defaultdict(dict)) caseClusters = UI_CASE_CLUSTERS + [get_custom_case_cluter()] for caseCluster in caseClusters: - activedCaseList += caseClusterExpander( - st, caseCluster, dbToCaseClusterConfigs, activedDbList) + activedCaseList += caseClusterExpander(st, caseCluster, dbToCaseClusterConfigs, activedDbList) for db in dbToCaseClusterConfigs: for uiCaseItem in dbToCaseClusterConfigs[db]: for case in uiCaseItem.cases: @@ -40,8 +37,7 @@ def caseClusterExpander(st, caseCluster: UICaseItemCluster, dbToCaseClusterConfi if uiCaseItem.isLine: addHorizontalLine(expander) else: - activedCases += caseItemCheckbox(expander, - dbToCaseClusterConfigs, uiCaseItem, activedDbList) + activedCases += caseItemCheckbox(expander, dbToCaseClusterConfigs, uiCaseItem, activedDbList) return activedCases @@ -53,9 +49,7 @@ def caseItemCheckbox(st, dbToCaseClusterConfigs, uiCaseItem: UICaseItem, actived ) if selected: - caseConfigSetting( - st.container(), dbToCaseClusterConfigs, uiCaseItem, activedDbList - ) + caseConfigSetting(st.container(), dbToCaseClusterConfigs, uiCaseItem, activedDbList) return uiCaseItem.cases if selected else [] diff --git a/vectordb_bench/frontend/components/run_test/dbConfigSetting.py b/vectordb_bench/frontend/components/run_test/dbConfigSetting.py index 257608413..800e6dede 100644 --- a/vectordb_bench/frontend/components/run_test/dbConfigSetting.py +++ b/vectordb_bench/frontend/components/run_test/dbConfigSetting.py @@ -42,10 +42,7 @@ def dbConfigSettingItem(st, activeDb: DB): # db config (unique) for key, property in properties.items(): - if ( - key not in dbConfigClass.common_short_configs() - and key not in dbConfigClass.common_long_configs() - ): + if key not in dbConfigClass.common_short_configs() and key not in dbConfigClass.common_long_configs(): column = columns[idx % DB_CONFIG_SETTING_COLUMNS] idx += 1 dbConfig[key] = column.text_input( diff --git a/vectordb_bench/frontend/components/run_test/dbSelector.py b/vectordb_bench/frontend/components/run_test/dbSelector.py index ccf0168c6..e20ee5059 100644 --- a/vectordb_bench/frontend/components/run_test/dbSelector.py +++ b/vectordb_bench/frontend/components/run_test/dbSelector.py @@ -22,7 +22,7 @@ def dbSelector(st): dbIsActived[db] = column.checkbox(db.name) try: column.image(DB_TO_ICON.get(db, "")) - except MediaFileStorageError as e: + except MediaFileStorageError: column.warning(f"{db.name} image not available") pass activedDbList = [db for db in DB_LIST if dbIsActived[db]] diff --git a/vectordb_bench/frontend/components/run_test/generateTasks.py b/vectordb_bench/frontend/components/run_test/generateTasks.py index 828913f30..d8a678ffc 100644 --- a/vectordb_bench/frontend/components/run_test/generateTasks.py +++ b/vectordb_bench/frontend/components/run_test/generateTasks.py @@ -7,13 +7,13 @@ def generate_tasks(activedDbList: list[DB], dbConfigs, activedCaseList: list[Cas for db in activedDbList: for case in activedCaseList: task = TaskConfig( - db=db.value, - db_config=dbConfigs[db], - case_config=case, - db_case_config=db.case_config_cls( - allCaseConfigs[db][case].get(CaseConfigParamType.IndexType, None) - )(**{key.value: value for key, value in allCaseConfigs[db][case].items()}), - ) + db=db.value, + db_config=dbConfigs[db], + case_config=case, + db_case_config=db.case_config_cls(allCaseConfigs[db][case].get(CaseConfigParamType.IndexType, None))( + **{key.value: value for key, value in allCaseConfigs[db][case].items()} + ), + ) tasks.append(task) - + return tasks diff --git a/vectordb_bench/frontend/components/run_test/submitTask.py b/vectordb_bench/frontend/components/run_test/submitTask.py index f824dd9d9..426095397 100644 --- a/vectordb_bench/frontend/components/run_test/submitTask.py +++ b/vectordb_bench/frontend/components/run_test/submitTask.py @@ -1,6 +1,6 @@ from datetime import datetime -from vectordb_bench.frontend.config.styles import * -from vectordb_bench.interface import benchMarkRunner +from vectordb_bench.frontend.config import styles +from vectordb_bench.interface import benchmark_runner def submitTask(st, tasks, isAllValid): @@ -27,10 +27,8 @@ def submitTask(st, tasks, isAllValid): def taskLabelInput(st): defaultTaskLabel = datetime.now().strftime("%Y%m%d%H") - columns = st.columns(TASK_LABEL_INPUT_COLUMNS) - taskLabel = columns[0].text_input( - "task_label", defaultTaskLabel, label_visibility="collapsed" - ) + columns = st.columns(styles.TASK_LABEL_INPUT_COLUMNS) + taskLabel = columns[0].text_input("task_label", defaultTaskLabel, label_visibility="collapsed") return taskLabel @@ -46,10 +44,8 @@ def advancedSettings(st): ) container = st.columns([1, 2]) - k = container[0].number_input("k",min_value=1, value=100, label_visibility="collapsed") - container[1].caption( - "K value for number of nearest neighbors to search" - ) + k = container[0].number_input("k", min_value=1, value=100, label_visibility="collapsed") + container[1].caption("K value for number of nearest neighbors to search") return index_already_exists, use_aliyun, k @@ -58,20 +54,20 @@ def controlPanel(st, tasks, taskLabel, isAllValid): index_already_exists, use_aliyun, k = advancedSettings(st) def runHandler(): - benchMarkRunner.set_drop_old(not index_already_exists) + benchmark_runner.set_drop_old(not index_already_exists) for task in tasks: task.case_config.k = k - benchMarkRunner.set_download_address(use_aliyun) - benchMarkRunner.run(tasks, taskLabel) + benchmark_runner.set_download_address(use_aliyun) + benchmark_runner.run(tasks, taskLabel) def stopHandler(): - benchMarkRunner.stop_running() + benchmark_runner.stop_running() - isRunning = benchMarkRunner.has_running() + isRunning = benchmark_runner.has_running() if isRunning: - currentTaskId = benchMarkRunner.get_current_task_id() - tasksCount = benchMarkRunner.get_tasks_count() + currentTaskId = benchmark_runner.get_current_task_id() + tasksCount = benchmark_runner.get_tasks_count() text = f":running: Running Task {currentTaskId} / {tasksCount}" st.progress(currentTaskId / tasksCount, text=text) @@ -89,7 +85,7 @@ def stopHandler(): ) else: - errorText = benchMarkRunner.latest_error or "" + errorText = benchmark_runner.latest_error or "" if len(errorText) > 0: st.error(errorText) disabled = True if len(tasks) == 0 or not isAllValid else False diff --git a/vectordb_bench/frontend/components/tables/data.py b/vectordb_bench/frontend/components/tables/data.py index 96134c7ff..fbe83f197 100644 --- a/vectordb_bench/frontend/components/tables/data.py +++ b/vectordb_bench/frontend/components/tables/data.py @@ -1,12 +1,11 @@ from dataclasses import asdict -from vectordb_bench.backend.cases import CaseType -from vectordb_bench.interface import benchMarkRunner +from vectordb_bench.interface import benchmark_runner from vectordb_bench.models import CaseResult, ResultLabel import pandas as pd def getNewResults(): - allResults = benchMarkRunner.get_results() + allResults = benchmark_runner.get_results() newResults: list[CaseResult] = [] for res in allResults: @@ -14,7 +13,6 @@ def getNewResults(): for result in results: if result.label == ResultLabel.NORMAL: newResults.append(result) - df = pd.DataFrame(formatData(newResults)) return df @@ -26,7 +24,6 @@ def formatData(caseResults: list[CaseResult]): db = caseResult.task_config.db.value db_label = caseResult.task_config.db_config.db_label case_config = caseResult.task_config.case_config - db_case_config = caseResult.task_config.db_case_config case = case_config.case_id.case_cls() filter_rate = case.filter_rate dataset = case.dataset.data.name @@ -41,4 +38,4 @@ def formatData(caseResults: list[CaseResult]): **metrics, } ) - return data \ No newline at end of file + return data diff --git a/vectordb_bench/frontend/config/dbCaseConfigs.py b/vectordb_bench/frontend/config/dbCaseConfigs.py index 7f076a2dd..e004f2ba7 100644 --- a/vectordb_bench/frontend/config/dbCaseConfigs.py +++ b/vectordb_bench/frontend/config/dbCaseConfigs.py @@ -33,9 +33,9 @@ class UICaseItem(BaseModel): def __init__( self, isLine: bool = False, - case_id: CaseType = None, - custom_case: dict = {}, - cases: list[CaseConfig] = [], + case_id: CaseType | None = None, + custom_case: dict | None = None, + cases: list[CaseConfig] | None = None, label: str = "", description: str = "", caseLabel: CaseLabel = CaseLabel.Performance, @@ -70,17 +70,13 @@ class UICaseItemCluster(BaseModel): def get_custom_case_items() -> list[UICaseItem]: custom_configs = get_custom_configs() return [ - UICaseItem( - case_id=CaseType.PerformanceCustomDataset, custom_case=custom_config.dict() - ) + UICaseItem(case_id=CaseType.PerformanceCustomDataset, custom_case=custom_config.dict()) for custom_config in custom_configs ] def get_custom_case_cluter() -> UICaseItemCluster: - return UICaseItemCluster( - label="Custom Search Performance Test", uiCaseItems=get_custom_case_items() - ) + return UICaseItemCluster(label="Custom Search Performance Test", uiCaseItems=get_custom_case_items()) UI_CASE_CLUSTERS: list[UICaseItemCluster] = [ @@ -224,8 +220,7 @@ class CaseConfigInput(BaseModel): "max": 300, "value": 32, }, - isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) - == IndexType.DISKANN.value, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.DISKANN.value, ) CaseConfigParamInput_l_value_ib = CaseConfigInput( @@ -236,8 +231,7 @@ class CaseConfigInput(BaseModel): "max": 300, "value": 50, }, - isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) - == IndexType.DISKANN.value, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.DISKANN.value, ) CaseConfigParamInput_l_value_is = CaseConfigInput( @@ -248,8 +242,7 @@ class CaseConfigInput(BaseModel): "max": 300, "value": 40, }, - isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) - == IndexType.DISKANN.value, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.DISKANN.value, ) CaseConfigParamInput_num_neighbors = CaseConfigInput( @@ -260,8 +253,7 @@ class CaseConfigInput(BaseModel): "max": 300, "value": 50, }, - isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) - == IndexType.STREAMING_DISKANN.value, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.STREAMING_DISKANN.value, ) CaseConfigParamInput_search_list_size = CaseConfigInput( @@ -272,8 +264,7 @@ class CaseConfigInput(BaseModel): "max": 300, "value": 100, }, - isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) - == IndexType.STREAMING_DISKANN.value, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.STREAMING_DISKANN.value, ) CaseConfigParamInput_max_alpha = CaseConfigInput( @@ -284,8 +275,7 @@ class CaseConfigInput(BaseModel): "max": 2.0, "value": 1.2, }, - isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) - == IndexType.STREAMING_DISKANN.value, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.STREAMING_DISKANN.value, ) CaseConfigParamInput_num_dimensions = CaseConfigInput( @@ -296,8 +286,7 @@ class CaseConfigInput(BaseModel): "max": 2000, "value": 0, }, - isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) - == IndexType.STREAMING_DISKANN.value, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.STREAMING_DISKANN.value, ) CaseConfigParamInput_query_search_list_size = CaseConfigInput( @@ -308,8 +297,7 @@ class CaseConfigInput(BaseModel): "max": 150, "value": 100, }, - isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) - == IndexType.STREAMING_DISKANN.value, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.STREAMING_DISKANN.value, ) @@ -321,8 +309,7 @@ class CaseConfigInput(BaseModel): "max": 150, "value": 50, }, - isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) - == IndexType.STREAMING_DISKANN.value, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.STREAMING_DISKANN.value, ) CaseConfigParamInput_IndexType_PgVector = CaseConfigInput( @@ -358,8 +345,7 @@ class CaseConfigInput(BaseModel): "max": 64, "value": 30, }, - isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) - == IndexType.HNSW.value, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.HNSW.value, ) CaseConfigParamInput_m = CaseConfigInput( @@ -370,8 +356,7 @@ class CaseConfigInput(BaseModel): "max": 64, "value": 16, }, - isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) - == IndexType.HNSW.value, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.HNSW.value, ) @@ -383,8 +368,7 @@ class CaseConfigInput(BaseModel): "max": 512, "value": 360, }, - isDisplayed=lambda config: config[CaseConfigParamType.IndexType] - == IndexType.HNSW.value, + isDisplayed=lambda config: config[CaseConfigParamType.IndexType] == IndexType.HNSW.value, ) CaseConfigParamInput_EFConstruction_Weaviate = CaseConfigInput( @@ -480,8 +464,7 @@ class CaseConfigInput(BaseModel): "max": 2000, "value": 300, }, - isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) - == IndexType.HNSW.value, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.HNSW.value, ) CaseConfigParamInput_EFSearch_PgVectoRS = CaseConfigInput( @@ -492,8 +475,7 @@ class CaseConfigInput(BaseModel): "max": 65535, "value": 100, }, - isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) - == IndexType.HNSW.value, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.HNSW.value, ) CaseConfigParamInput_EFConstruction_PgVector = CaseConfigInput( @@ -504,8 +486,7 @@ class CaseConfigInput(BaseModel): "max": 1024, "value": 256, }, - isDisplayed=lambda config: config[CaseConfigParamType.IndexType] - == IndexType.HNSW.value, + isDisplayed=lambda config: config[CaseConfigParamType.IndexType] == IndexType.HNSW.value, ) @@ -537,8 +518,7 @@ class CaseConfigInput(BaseModel): "max": MAX_STREAMLIT_INT, "value": 100, }, - isDisplayed=lambda config: config[CaseConfigParamType.IndexType] - == IndexType.HNSW.value, + isDisplayed=lambda config: config[CaseConfigParamType.IndexType] == IndexType.HNSW.value, ) CaseConfigParamInput_EF_Weaviate = CaseConfigInput( @@ -565,8 +545,7 @@ class CaseConfigInput(BaseModel): "max": MAX_STREAMLIT_INT, "value": 100, }, - isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) - == IndexType.DISKANN.value, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.DISKANN.value, ) CaseConfigParamInput_Nlist = CaseConfigInput( @@ -611,8 +590,7 @@ class CaseConfigInput(BaseModel): "max": 65536, "value": 0, }, - isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) - in [IndexType.GPU_IVF_PQ.value], + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) in [IndexType.GPU_IVF_PQ.value], ) @@ -624,8 +602,7 @@ class CaseConfigInput(BaseModel): "max": 65536, "value": 8, }, - isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) - in [IndexType.GPU_IVF_PQ.value], + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) in [IndexType.GPU_IVF_PQ.value], ) CaseConfigParamInput_intermediate_graph_degree = CaseConfigInput( @@ -636,8 +613,7 @@ class CaseConfigInput(BaseModel): "max": 65536, "value": 64, }, - isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) - in [IndexType.GPU_CAGRA.value], + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) in [IndexType.GPU_CAGRA.value], ) CaseConfigParamInput_graph_degree = CaseConfigInput( @@ -648,8 +624,7 @@ class CaseConfigInput(BaseModel): "max": 65536, "value": 32, }, - isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) - in [IndexType.GPU_CAGRA.value], + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) in [IndexType.GPU_CAGRA.value], ) CaseConfigParamInput_itopk_size = CaseConfigInput( @@ -660,8 +635,7 @@ class CaseConfigInput(BaseModel): "max": 65536, "value": 128, }, - isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) - in [IndexType.GPU_CAGRA.value], + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) in [IndexType.GPU_CAGRA.value], ) CaseConfigParamInput_team_size = CaseConfigInput( @@ -672,8 +646,7 @@ class CaseConfigInput(BaseModel): "max": 65536, "value": 0, }, - isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) - in [IndexType.GPU_CAGRA.value], + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) in [IndexType.GPU_CAGRA.value], ) CaseConfigParamInput_search_width = CaseConfigInput( @@ -684,8 +657,7 @@ class CaseConfigInput(BaseModel): "max": 65536, "value": 4, }, - isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) - in [IndexType.GPU_CAGRA.value], + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) in [IndexType.GPU_CAGRA.value], ) CaseConfigParamInput_min_iterations = CaseConfigInput( @@ -696,8 +668,7 @@ class CaseConfigInput(BaseModel): "max": 65536, "value": 0, }, - isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) - in [IndexType.GPU_CAGRA.value], + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) in [IndexType.GPU_CAGRA.value], ) CaseConfigParamInput_max_iterations = CaseConfigInput( @@ -708,8 +679,7 @@ class CaseConfigInput(BaseModel): "max": 65536, "value": 0, }, - isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) - in [IndexType.GPU_CAGRA.value], + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) in [IndexType.GPU_CAGRA.value], ) CaseConfigParamInput_build_algo = CaseConfigInput( @@ -718,8 +688,7 @@ class CaseConfigInput(BaseModel): inputConfig={ "options": ["IVF_PQ", "NN_DESCENT"], }, - isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) - in [IndexType.GPU_CAGRA.value], + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) in [IndexType.GPU_CAGRA.value], ) @@ -762,8 +731,7 @@ class CaseConfigInput(BaseModel): "max": 65536, "value": 10, }, - isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) - in [IndexType.IVFFlat.value], + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) in [IndexType.IVFFlat.value], ) CaseConfigParamInput_Probes = CaseConfigInput( @@ -784,8 +752,7 @@ class CaseConfigInput(BaseModel): "max": 65536, "value": 10, }, - isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) - == IndexType.IVFFlat.value, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.IVFFlat.value, ) CaseConfigParamInput_Probes_PgVector = CaseConfigInput( @@ -796,8 +763,7 @@ class CaseConfigInput(BaseModel): "max": 65536, "value": 1, }, - isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) - == IndexType.IVFFlat.value, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.IVFFlat.value, ) CaseConfigParamInput_EFSearch_PgVector = CaseConfigInput( @@ -808,8 +774,7 @@ class CaseConfigInput(BaseModel): "max": 2048, "value": 256, }, - isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) - == IndexType.HNSW.value, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.HNSW.value, ) @@ -845,8 +810,7 @@ class CaseConfigInput(BaseModel): inputConfig={ "options": ["x4", "x8", "x16", "x32", "x64"], }, - isDisplayed=lambda config: config.get(CaseConfigParamType.quantizationType, None) - == "product" + isDisplayed=lambda config: config.get(CaseConfigParamType.quantizationType, None) == "product" and config.get(CaseConfigParamType.IndexType, None) in [ IndexType.HNSW.value, @@ -885,8 +849,7 @@ class CaseConfigInput(BaseModel): inputConfig={ "value": False, }, - isDisplayed=lambda config: config.get(CaseConfigParamType.quantizationType, None) - == "bit" + isDisplayed=lambda config: config.get(CaseConfigParamType.quantizationType, None) == "bit", ) CaseConfigParamInput_quantized_fetch_limit_PgVector = CaseConfigInput( @@ -899,8 +862,8 @@ class CaseConfigInput(BaseModel): "max": 1000, "value": 200, }, - isDisplayed=lambda config: config.get(CaseConfigParamType.quantizationType, None) - == "bit" and config.get(CaseConfigParamType.reranking, False) + isDisplayed=lambda config: config.get(CaseConfigParamType.quantizationType, None) == "bit" + and config.get(CaseConfigParamType.reranking, False), ) @@ -908,12 +871,10 @@ class CaseConfigInput(BaseModel): label=CaseConfigParamType.rerankingMetric, inputType=InputType.Option, inputConfig={ - "options": [ - metric.value for metric in MetricType if metric.value not in ["HAMMING", "JACCARD"] - ], + "options": [metric.value for metric in MetricType if metric.value not in ["HAMMING", "JACCARD"]], }, - isDisplayed=lambda config: config.get(CaseConfigParamType.quantizationType, None) - == "bit" and config.get(CaseConfigParamType.reranking, False) + isDisplayed=lambda config: config.get(CaseConfigParamType.quantizationType, None) == "bit" + and config.get(CaseConfigParamType.reranking, False), ) @@ -1131,7 +1092,10 @@ class CaseConfigInput(BaseModel): CaseConfigParamInput_NumCandidates_ES, ] -AWSOpensearchLoadingConfig = [CaseConfigParamInput_EFConstruction_AWSOpensearch, CaseConfigParamInput_M_AWSOpensearch] +AWSOpensearchLoadingConfig = [ + CaseConfigParamInput_EFConstruction_AWSOpensearch, + CaseConfigParamInput_M_AWSOpensearch, +] AWSOpenSearchPerformanceConfig = [ CaseConfigParamInput_EFConstruction_AWSOpensearch, CaseConfigParamInput_M_AWSOpensearch, @@ -1250,7 +1214,10 @@ class CaseConfigInput(BaseModel): CaseConfigParamInput_max_parallel_workers_AlloyDB, ] -AliyunElasticsearchLoadingConfig = [CaseConfigParamInput_EFConstruction_AliES, CaseConfigParamInput_M_AliES] +AliyunElasticsearchLoadingConfig = [ + CaseConfigParamInput_EFConstruction_AliES, + CaseConfigParamInput_M_AliES, +] AliyunElasticsearchPerformanceConfig = [ CaseConfigParamInput_EFConstruction_AliES, CaseConfigParamInput_M_AliES, diff --git a/vectordb_bench/frontend/pages/concurrent.py b/vectordb_bench/frontend/pages/concurrent.py index 941675436..045239494 100644 --- a/vectordb_bench/frontend/pages/concurrent.py +++ b/vectordb_bench/frontend/pages/concurrent.py @@ -9,7 +9,7 @@ from vectordb_bench.frontend.components.concurrent.charts import drawChartsByCase from vectordb_bench.frontend.components.get_results.saveAsImage import getResults from vectordb_bench.frontend.config.styles import FAVICON -from vectordb_bench.interface import benchMarkRunner +from vectordb_bench.interface import benchmark_runner from vectordb_bench.models import TestResult @@ -25,7 +25,7 @@ def main(): # header drawHeaderIcon(st) - allResults = benchMarkRunner.get_results() + allResults = benchmark_runner.get_results() def check_conc_data(res: TestResult): case_results = res.results @@ -57,9 +57,7 @@ def check_conc_data(res: TestResult): # main latency_type = st.radio("Latency Type", options=["latency_p99", "latency_avg"]) - drawChartsByCase( - shownData, showCaseNames, st.container(), latency_type=latency_type - ) + drawChartsByCase(shownData, showCaseNames, st.container(), latency_type=latency_type) # footer footer(st.container()) diff --git a/vectordb_bench/frontend/pages/custom.py b/vectordb_bench/frontend/pages/custom.py index 28c249f78..4f6beed91 100644 --- a/vectordb_bench/frontend/pages/custom.py +++ b/vectordb_bench/frontend/pages/custom.py @@ -1,13 +1,21 @@ +from functools import partial import streamlit as st from vectordb_bench.frontend.components.check_results.headerIcon import drawHeaderIcon -from vectordb_bench.frontend.components.custom.displayCustomCase import displayCustomCase +from vectordb_bench.frontend.components.custom.displayCustomCase import ( + displayCustomCase, +) from vectordb_bench.frontend.components.custom.displaypPrams import displayParams -from vectordb_bench.frontend.components.custom.getCustomConfig import CustomCaseConfig, generate_custom_case, get_custom_configs, save_custom_configs +from vectordb_bench.frontend.components.custom.getCustomConfig import ( + CustomCaseConfig, + generate_custom_case, + get_custom_configs, + save_custom_configs, +) from vectordb_bench.frontend.components.custom.initStyle import initStyle from vectordb_bench.frontend.config.styles import FAVICON, PAGE_TITLE -class CustomCaseManager(): +class CustomCaseManager: customCaseItems: list[CustomCaseConfig] def __init__(self): @@ -52,12 +60,25 @@ def main(): columns = expander.columns(8) columns[0].button( - "Save", key=f"{key}_", type="secondary", on_click=lambda: customCaseManager.save()) - columns[1].button(":red[Delete]", key=f"{key}_delete", type="secondary", - on_click=lambda: customCaseManager.deleteCase(idx)) - - st.button("\+ New Dataset", key=f"add_custom_configs", - type="primary", on_click=lambda: customCaseManager.addCase()) + "Save", + key=f"{key}_", + type="secondary", + on_click=lambda: customCaseManager.save(), + ) + columns[1].button( + ":red[Delete]", + key=f"{key}_delete", + type="secondary", + # B023 + on_click=partial(lambda idx: customCaseManager.deleteCase(idx), idx=idx), + ) + + st.button( + "\+ New Dataset", + key="add_custom_configs", + type="primary", + on_click=lambda: customCaseManager.addCase(), + ) if __name__ == "__main__": diff --git a/vectordb_bench/frontend/pages/quries_per_dollar.py b/vectordb_bench/frontend/pages/quries_per_dollar.py index 4a45181de..2822f2864 100644 --- a/vectordb_bench/frontend/pages/quries_per_dollar.py +++ b/vectordb_bench/frontend/pages/quries_per_dollar.py @@ -15,8 +15,8 @@ from vectordb_bench.frontend.components.check_results.charts import drawMetricChart from vectordb_bench.frontend.components.check_results.filters import getshownData from vectordb_bench.frontend.components.get_results.saveAsImage import getResults -from vectordb_bench.frontend.config.styles import * -from vectordb_bench.interface import benchMarkRunner + +from vectordb_bench.interface import benchmark_runner from vectordb_bench.metric import QURIES_PER_DOLLAR_METRIC @@ -27,7 +27,7 @@ def main(): # header drawHeaderIcon(st) - allResults = benchMarkRunner.get_results() + allResults = benchmark_runner.get_results() st.title("Vector DB Benchmark (QP$)") diff --git a/vectordb_bench/frontend/pages/run_test.py b/vectordb_bench/frontend/pages/run_test.py index 1297743ae..3da8ea2c0 100644 --- a/vectordb_bench/frontend/pages/run_test.py +++ b/vectordb_bench/frontend/pages/run_test.py @@ -15,10 +15,10 @@ def main(): # set page config initRunTestPageConfig(st) - + # init style initStyle(st) - + # header drawHeaderIcon(st) @@ -48,11 +48,7 @@ def main(): activedCaseList, allCaseConfigs = caseSelector(caseSelectorContainer, activedDbList) # generate tasks - tasks = ( - generate_tasks(activedDbList, dbConfigs, activedCaseList, allCaseConfigs) - if isAllValid - else [] - ) + tasks = generate_tasks(activedDbList, dbConfigs, activedCaseList, allCaseConfigs) if isAllValid else [] # submit submitContainer = st.container() diff --git a/vectordb_bench/frontend/utils.py b/vectordb_bench/frontend/utils.py index 787b67d03..407dd497d 100644 --- a/vectordb_bench/frontend/utils.py +++ b/vectordb_bench/frontend/utils.py @@ -18,5 +18,5 @@ def addHorizontalLine(st): def generate_random_string(length): letters = string.ascii_letters + string.digits - result = ''.join(random.choice(letters) for _ in range(length)) + result = "".join(random.choice(letters) for _ in range(length)) return result diff --git a/vectordb_bench/frontend/vdb_benchmark.py b/vectordb_bench/frontend/vdb_benchmark.py index c76a2f3de..cf261bf1d 100644 --- a/vectordb_bench/frontend/vdb_benchmark.py +++ b/vectordb_bench/frontend/vdb_benchmark.py @@ -11,8 +11,8 @@ from vectordb_bench.frontend.components.check_results.charts import drawCharts from vectordb_bench.frontend.components.check_results.filters import getshownData from vectordb_bench.frontend.components.get_results.saveAsImage import getResults -from vectordb_bench.frontend.config.styles import * -from vectordb_bench.interface import benchMarkRunner + +from vectordb_bench.interface import benchmark_runner def main(): @@ -22,7 +22,7 @@ def main(): # header drawHeaderIcon(st) - allResults = benchMarkRunner.get_results() + allResults = benchmark_runner.get_results() st.title("Vector Database Benchmark") st.caption( @@ -32,9 +32,7 @@ def main(): # results selector and filter resultSelectorContainer = st.sidebar.container() - shownData, failedTasks, showCaseNames = getshownData( - allResults, resultSelectorContainer - ) + shownData, failedTasks, showCaseNames = getshownData(allResults, resultSelectorContainer) resultSelectorContainer.divider() diff --git a/vectordb_bench/interface.py b/vectordb_bench/interface.py index c765d0d63..615a9600d 100644 --- a/vectordb_bench/interface.py +++ b/vectordb_bench/interface.py @@ -5,6 +5,7 @@ import signal import traceback import uuid +from collections.abc import Callable from enum import Enum from multiprocessing.connection import Connection @@ -16,8 +17,15 @@ from .backend.result_collector import ResultCollector from .backend.task_runner import TaskRunner from .metric import Metric -from .models import (CaseResult, LoadTimeoutError, PerformanceTimeoutError, - ResultLabel, TaskConfig, TaskStage, TestResult) +from .models import ( + CaseResult, + LoadTimeoutError, + PerformanceTimeoutError, + ResultLabel, + TaskConfig, + TaskStage, + TestResult, +) log = logging.getLogger(__name__) @@ -37,11 +45,9 @@ def __init__(self): self.drop_old: bool = True self.dataset_source: DatasetSource = DatasetSource.S3 - def set_drop_old(self, drop_old: bool): self.drop_old = drop_old - def set_download_address(self, use_aliyun: bool): if use_aliyun: self.dataset_source = DatasetSource.AliyunOSS @@ -59,7 +65,9 @@ def run(self, tasks: list[TaskConfig], task_label: str | None = None) -> bool: log.warning("Empty tasks submitted") return False - log.debug(f"tasks: {tasks}, task_label: {task_label}, dataset source: {self.dataset_source}") + log.debug( + f"tasks: {tasks}, task_label: {task_label}, dataset source: {self.dataset_source}", + ) # Generate run_id run_id = uuid.uuid4().hex @@ -70,7 +78,12 @@ def run(self, tasks: list[TaskConfig], task_label: str | None = None) -> bool: self.latest_error = "" try: - self.running_task = Assembler.assemble_all(run_id, task_label, tasks, self.dataset_source) + self.running_task = Assembler.assemble_all( + run_id, + task_label, + tasks, + self.dataset_source, + ) self.running_task.display() except ModuleNotFoundError as e: msg = f"Please install client for database, error={e}" @@ -119,7 +132,7 @@ def get_tasks_count(self) -> int: return 0 def get_current_task_id(self) -> int: - """ the index of current running task + """the index of current running task return -1 if not running """ if not self.running_task: @@ -153,18 +166,18 @@ def _async_task_v2(self, running_task: TaskRunner, send_conn: Connection) -> Non task_config=runner.config, ) - # drop_old = False if latest_runner and runner == latest_runner else config.DROP_OLD - # drop_old = config.DROP_OLD drop_old = TaskStage.DROP_OLD in runner.config.stages - if latest_runner and runner == latest_runner: - drop_old = False - elif not self.drop_old: + if (latest_runner and runner == latest_runner) or not self.drop_old: drop_old = False try: - log.info(f"[{idx+1}/{running_task.num_cases()}] start case: {runner.display()}, drop_old={drop_old}") + log.info( + f"[{idx+1}/{running_task.num_cases()}] start case: {runner.display()}, drop_old={drop_old}", + ) case_res.metrics = runner.run(drop_old) - log.info(f"[{idx+1}/{running_task.num_cases()}] finish case: {runner.display()}, " - f"result={case_res.metrics}, label={case_res.label}") + log.info( + f"[{idx+1}/{running_task.num_cases()}] finish case: {runner.display()}, " + f"result={case_res.metrics}, label={case_res.label}", + ) # cache the latest succeeded runner latest_runner = runner @@ -176,12 +189,16 @@ def _async_task_v2(self, running_task: TaskRunner, send_conn: Connection) -> Non if not drop_old: case_res.metrics.load_duration = cached_load_duration if cached_load_duration else 0.0 except (LoadTimeoutError, PerformanceTimeoutError) as e: - log.warning(f"[{idx+1}/{running_task.num_cases()}] case {runner.display()} failed to run, reason={e}") + log.warning( + f"[{idx+1}/{running_task.num_cases()}] case {runner.display()} failed to run, reason={e}", + ) case_res.label = ResultLabel.OUTOFRANGE continue except Exception as e: - log.warning(f"[{idx+1}/{running_task.num_cases()}] case {runner.display()} failed to run, reason={e}") + log.warning( + f"[{idx+1}/{running_task.num_cases()}] case {runner.display()} failed to run, reason={e}", + ) traceback.print_exc() case_res.label = ResultLabel.FAILED continue @@ -200,10 +217,14 @@ def _async_task_v2(self, running_task: TaskRunner, send_conn: Connection) -> Non send_conn.send((SIGNAL.SUCCESS, None)) send_conn.close() - log.info(f"Success to finish task: label={running_task.task_label}, run_id={running_task.run_id}") + log.info( + f"Success to finish task: label={running_task.task_label}, run_id={running_task.run_id}", + ) except Exception as e: - err_msg = f"An error occurs when running task={running_task.task_label}, run_id={running_task.run_id}, err={e}" + err_msg = ( + f"An error occurs when running task={running_task.task_label}, run_id={running_task.run_id}, err={e}" + ) traceback.print_exc() log.warning(err_msg) send_conn.send((SIGNAL.ERROR, err_msg)) @@ -226,16 +247,26 @@ def _clear_running_task(self): self.receive_conn.close() self.receive_conn = None - def _run_async(self, conn: Connection) -> bool: - log.info(f"task submitted: id={self.running_task.run_id}, {self.running_task.task_label}, case number: {len(self.running_task.case_runners)}") + log.info( + f"task submitted: id={self.running_task.run_id}, {self.running_task.task_label}, ", + f"case number: {len(self.running_task.case_runners)}", + ) global global_result_future - executor = concurrent.futures.ProcessPoolExecutor(max_workers=1, mp_context=mp.get_context("spawn")) + executor = concurrent.futures.ProcessPoolExecutor( + max_workers=1, + mp_context=mp.get_context("spawn"), + ) global_result_future = executor.submit(self._async_task_v2, self.running_task, conn) return True - def kill_proc_tree(self, sig=signal.SIGTERM, timeout=None, on_terminate=None): + def kill_proc_tree( + self, + sig: int = signal.SIGTERM, + timeout: float | None = None, + on_terminate: Callable | None = None, + ): """Kill a process tree (including grandchildren) with signal "sig" and return a (gone, still_alive) tuple. "on_terminate", if specified, is a callback function which is @@ -248,12 +279,11 @@ def kill_proc_tree(self, sig=signal.SIGTERM, timeout=None, on_terminate=None): p.send_signal(sig) except psutil.NoSuchProcess: pass - gone, alive = psutil.wait_procs(children, timeout=timeout, - callback=on_terminate) + gone, alive = psutil.wait_procs(children, timeout=timeout, callback=on_terminate) for p in alive: log.warning(f"force killing child process: {p}") p.kill() -benchMarkRunner = BenchMarkRunner() +benchmark_runner = BenchMarkRunner() diff --git a/vectordb_bench/log_util.py b/vectordb_bench/log_util.py index b923bdcd2..d75688137 100644 --- a/vectordb_bench/log_util.py +++ b/vectordb_bench/log_util.py @@ -1,102 +1,97 @@ import logging from logging import config -def init(log_level): - LOGGING = { - 'version': 1, - 'disable_existing_loggers': False, - 'formatters': { - 'default': { - 'format': '%(asctime)s | %(levelname)s |%(message)s (%(filename)s:%(lineno)s)', + +def init(log_level: str): + log_config = { + "version": 1, + "disable_existing_loggers": False, + "formatters": { + "default": { + "format": "%(asctime)s | %(levelname)s |%(message)s (%(filename)s:%(lineno)s)", }, - 'colorful_console': { - 'format': '%(asctime)s | %(levelname)s: %(message)s (%(filename)s:%(lineno)s) (%(process)s)', - '()': ColorfulFormatter, + "colorful_console": { + "format": "%(asctime)s | %(levelname)s: %(message)s (%(filename)s:%(lineno)s) (%(process)s)", + "()": ColorfulFormatter, }, }, - 'handlers': { - 'console': { - 'class': 'logging.StreamHandler', - 'formatter': 'colorful_console', + "handlers": { + "console": { + "class": "logging.StreamHandler", + "formatter": "colorful_console", }, - 'no_color_console': { - 'class': 'logging.StreamHandler', - 'formatter': 'default', + "no_color_console": { + "class": "logging.StreamHandler", + "formatter": "default", }, }, - 'loggers': { - 'vectordb_bench': { - 'handlers': ['console'], - 'level': log_level, - 'propagate': False + "loggers": { + "vectordb_bench": { + "handlers": ["console"], + "level": log_level, + "propagate": False, }, - 'no_color': { - 'handlers': ['no_color_console'], - 'level': log_level, - 'propagate': False + "no_color": { + "handlers": ["no_color_console"], + "level": log_level, + "propagate": False, }, }, - 'propagate': False, + "propagate": False, } - config.dictConfig(LOGGING) + config.dictConfig(log_config) -class colors: - HEADER= '\033[95m' - INFO= '\033[92m' - DEBUG= '\033[94m' - WARNING= '\033[93m' - ERROR= '\033[95m' - CRITICAL= '\033[91m' - ENDC= '\033[0m' +class colors: + HEADER = "\033[95m" + INFO = "\033[92m" + DEBUG = "\033[94m" + WARNING = "\033[93m" + ERROR = "\033[95m" + CRITICAL = "\033[91m" + ENDC = "\033[0m" COLORS = { - 'INFO': colors.INFO, - 'INFOM': colors.INFO, - 'DEBUG': colors.DEBUG, - 'DEBUGM': colors.DEBUG, - 'WARNING': colors.WARNING, - 'WARNINGM': colors.WARNING, - 'CRITICAL': colors.CRITICAL, - 'CRITICALM': colors.CRITICAL, - 'ERROR': colors.ERROR, - 'ERRORM': colors.ERROR, - 'ENDC': colors.ENDC, + "INFO": colors.INFO, + "INFOM": colors.INFO, + "DEBUG": colors.DEBUG, + "DEBUGM": colors.DEBUG, + "WARNING": colors.WARNING, + "WARNINGM": colors.WARNING, + "CRITICAL": colors.CRITICAL, + "CRITICALM": colors.CRITICAL, + "ERROR": colors.ERROR, + "ERRORM": colors.ERROR, + "ENDC": colors.ENDC, } class ColorFulFormatColMixin: - def format_col(self, message_str, level_name): - if level_name in COLORS.keys(): - message_str = COLORS[level_name] + message_str + COLORS['ENDC'] - return message_str - - def formatTime(self, record, datefmt=None): - ret = super().formatTime(record, datefmt) - return ret + def format_col(self, message: str, level_name: str): + if level_name in COLORS: + message = COLORS[level_name] + message + COLORS["ENDC"] + return message class ColorfulLogRecordProxy(logging.LogRecord): - def __init__(self, record): + def __init__(self, record: any): self._record = record - msg_level = record.levelname + 'M' + msg_level = record.levelname + "M" self.msg = f"{COLORS[msg_level]}{record.msg}{COLORS['ENDC']}" self.filename = record.filename - self.lineno = f'{record.lineno}' - self.process = f'{record.process}' + self.lineno = f"{record.lineno}" + self.process = f"{record.process}" self.levelname = f"{COLORS[record.levelname]}{record.levelname}{COLORS['ENDC']}" - def __getattr__(self, attr): + def __getattr__(self, attr: any): if attr not in self.__dict__: return getattr(self._record, attr) return getattr(self, attr) class ColorfulFormatter(ColorFulFormatColMixin, logging.Formatter): - def format(self, record): + def format(self, record: any): proxy = ColorfulLogRecordProxy(record) - message_str = super().format(proxy) - - return message_str + return super().format(proxy) diff --git a/vectordb_bench/metric.py b/vectordb_bench/metric.py index 9f083a5c6..e0b6cff0e 100644 --- a/vectordb_bench/metric.py +++ b/vectordb_bench/metric.py @@ -1,8 +1,7 @@ import logging -import numpy as np - from dataclasses import dataclass, field +import numpy as np log = logging.getLogger(__name__) @@ -33,19 +32,19 @@ class Metric: QPS_METRIC = "qps" RECALL_METRIC = "recall" -metricUnitMap = { +metric_unit_map = { LOAD_DURATION_METRIC: "s", SERIAL_LATENCY_P99_METRIC: "ms", MAX_LOAD_COUNT_METRIC: "K", QURIES_PER_DOLLAR_METRIC: "K", } -lowerIsBetterMetricList = [ +lower_is_better_metrics = [ LOAD_DURATION_METRIC, SERIAL_LATENCY_P99_METRIC, ] -metricOrder = [ +metric_order = [ QPS_METRIC, RECALL_METRIC, LOAD_DURATION_METRIC, @@ -55,7 +54,7 @@ class Metric: def isLowerIsBetterMetric(metric: str) -> bool: - return metric in lowerIsBetterMetricList + return metric in lower_is_better_metrics def calc_recall(count: int, ground_truth: list[int], got: list[int]) -> float: @@ -70,7 +69,7 @@ def calc_recall(count: int, ground_truth: list[int], got: list[int]) -> float: def get_ideal_dcg(k: int): ideal_dcg = 0 for i in range(k): - ideal_dcg += 1 / np.log2(i+2) + ideal_dcg += 1 / np.log2(i + 2) return ideal_dcg @@ -78,8 +77,8 @@ def get_ideal_dcg(k: int): def calc_ndcg(ground_truth: list[int], got: list[int], ideal_dcg: float) -> float: dcg = 0 ground_truth = list(ground_truth) - for id in set(got): - if id in ground_truth: - idx = ground_truth.index(id) - dcg += 1 / np.log2(idx+2) + for got_id in set(got): + if got_id in ground_truth: + idx = ground_truth.index(got_id) + dcg += 1 / np.log2(idx + 2) return dcg / ideal_dcg diff --git a/vectordb_bench/models.py b/vectordb_bench/models.py index 648fb1727..49bb04ae0 100644 --- a/vectordb_bench/models.py +++ b/vectordb_bench/models.py @@ -2,29 +2,31 @@ import pathlib from datetime import date, datetime from enum import Enum, StrEnum, auto -from typing import List, Self +from typing import Self import ujson +from . import config +from .backend.cases import CaseType from .backend.clients import ( DB, - DBConfig, DBCaseConfig, + DBConfig, ) -from .backend.cases import CaseType from .base import BaseModel -from . import config from .metric import Metric log = logging.getLogger(__name__) class LoadTimeoutError(TimeoutError): - pass + def __init__(self, duration: int): + super().__init__(f"capacity case load timeout in {duration}s") class PerformanceTimeoutError(TimeoutError): - pass + def __init__(self): + super().__init__("Performance case optimize timeout") class CaseConfigParamType(Enum): @@ -92,7 +94,7 @@ class CustomizedCase(BaseModel): class ConcurrencySearchConfig(BaseModel): - num_concurrency: List[int] = config.NUM_CONCURRENCY + num_concurrency: list[int] = config.NUM_CONCURRENCY concurrency_duration: int = config.CONCURRENCY_DURATION @@ -146,7 +148,7 @@ class TaskConfig(BaseModel): db_config: DBConfig db_case_config: DBCaseConfig case_config: CaseConfig - stages: List[TaskStage] = ALL_TASK_STAGES + stages: list[TaskStage] = ALL_TASK_STAGES @property def db_name(self): @@ -210,26 +212,23 @@ def write_db_file(self, result_dir: pathlib.Path, partial: Self, db: str): log.info(f"local result directory not exist, creating it: {result_dir}") result_dir.mkdir(parents=True) - file_name = self.file_fmt.format( - date.today().strftime("%Y%m%d"), partial.task_label, db - ) + file_name = self.file_fmt.format(date.today().strftime("%Y%m%d"), partial.task_label, db) result_file = result_dir.joinpath(file_name) if result_file.exists(): - log.warning( - f"Replacing existing result with the same file_name: {result_file}" - ) + log.warning(f"Replacing existing result with the same file_name: {result_file}") log.info(f"write results to disk {result_file}") - with open(result_file, "w") as f: + with pathlib.Path(result_file).open("w") as f: b = partial.json(exclude={"db_config": {"password", "api_key"}}) f.write(b) @classmethod def read_file(cls, full_path: pathlib.Path, trans_unit: bool = False) -> Self: if not full_path.exists(): - raise ValueError(f"No such file: {full_path}") + msg = f"No such file: {full_path}" + raise ValueError(msg) - with open(full_path) as f: + with pathlib.Path(full_path).open("r") as f: test_result = ujson.loads(f.read()) if "task_label" not in test_result: test_result["task_label"] = test_result["run_id"] @@ -248,19 +247,16 @@ def read_file(cls, full_path: pathlib.Path, trans_unit: bool = False) -> Self: if trans_unit: cur_max_count = case_result["metrics"]["max_load_count"] case_result["metrics"]["max_load_count"] = ( - cur_max_count / 1000 - if int(cur_max_count) > 0 - else cur_max_count + cur_max_count / 1000 if int(cur_max_count) > 0 else cur_max_count ) cur_latency = case_result["metrics"]["serial_latency_p99"] case_result["metrics"]["serial_latency_p99"] = ( cur_latency * 1000 if cur_latency > 0 else cur_latency ) - c = TestResult.validate(test_result) - - return c + return TestResult.validate(test_result) + # ruff: noqa def display(self, dbs: list[DB] | None = None): filter_list = dbs if dbs and isinstance(dbs, list) else None sorted_results = sorted( @@ -273,31 +269,18 @@ def display(self, dbs: list[DB] | None = None): reverse=True, ) - filtered_results = [ - r - for r in sorted_results - if not filter_list or r.task_config.db not in filter_list - ] + filtered_results = [r for r in sorted_results if not filter_list or r.task_config.db not in filter_list] - def append_return(x, y): + def append_return(x: any, y: any): x.append(y) return x max_db = max(map(len, [f.task_config.db.name for f in filtered_results])) - max_db_labels = ( - max(map(len, [f.task_config.db_config.db_label for f in filtered_results])) - + 3 - ) - max_case = max( - map(len, [f.task_config.case_config.case_id.name for f in filtered_results]) - ) - max_load_dur = ( - max(map(len, [str(f.metrics.load_duration) for f in filtered_results])) + 3 - ) + max_db_labels = max(map(len, [f.task_config.db_config.db_label for f in filtered_results])) + 3 + max_case = max(map(len, [f.task_config.case_config.case_id.name for f in filtered_results])) + max_load_dur = max(map(len, [str(f.metrics.load_duration) for f in filtered_results])) + 3 max_qps = max(map(len, [str(f.metrics.qps) for f in filtered_results])) + 3 - max_recall = ( - max(map(len, [str(f.metrics.recall) for f in filtered_results])) + 3 - ) + max_recall = max(map(len, [str(f.metrics.recall) for f in filtered_results])) + 3 max_db_labels = 8 if max_db_labels < 8 else max_db_labels max_load_dur = 11 if max_load_dur < 11 else max_load_dur @@ -356,7 +339,7 @@ def append_return(x, y): f.metrics.recall, f.metrics.max_load_count, f.label.value, - ) + ), ) tmp_logger = logging.getLogger("no_color") From 4940bf7efb7bb1efc888c18319e29ecc062afcb5 Mon Sep 17 00:00:00 2001 From: "min.tian" Date: Thu, 9 Jan 2025 15:45:06 +0800 Subject: [PATCH 132/327] fix bug Signed-off-by: min.tian --- vectordb_bench/backend/assembler.py | 4 ++-- vectordb_bench/interface.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/vectordb_bench/backend/assembler.py b/vectordb_bench/backend/assembler.py index b81d315c2..b28a7847d 100644 --- a/vectordb_bench/backend/assembler.py +++ b/vectordb_bench/backend/assembler.py @@ -53,8 +53,8 @@ def assemble_all( _ = k.init_cls # sort by dataset size - for k, _ in db2runner: - db2runner[k].sort(key=lambda x: x.ca.dataset.data.size) + for _, runner in db2runner.items(): + runner.sort(key=lambda x: x.ca.dataset.data.size) all_runners = [] all_runners.extend(load_runners) diff --git a/vectordb_bench/interface.py b/vectordb_bench/interface.py index 615a9600d..ebe12d2e6 100644 --- a/vectordb_bench/interface.py +++ b/vectordb_bench/interface.py @@ -249,7 +249,7 @@ def _clear_running_task(self): def _run_async(self, conn: Connection) -> bool: log.info( - f"task submitted: id={self.running_task.run_id}, {self.running_task.task_label}, ", + f"task submitted: id={self.running_task.run_id}, {self.running_task.task_label}, " f"case number: {len(self.running_task.case_runners)}", ) global global_result_future From 6814767cdd154632801f09a6343122fcf1d7178a Mon Sep 17 00:00:00 2001 From: yangxuan Date: Fri, 10 Jan 2025 14:21:24 +0800 Subject: [PATCH 133/327] fix: Unable to run vebbench and cli fix: remove comma of logging str fix cli unable to run #444 Signed-off-by: yangxuan --- pyproject.toml | 1 + vectordb_bench/backend/clients/__init__.py | 14 +++++- .../backend/clients/memorydb/cli.py | 4 +- .../backend/clients/pgvecto_rs/pgvecto_rs.py | 9 +--- .../backend/clients/pgvector/pgvector.py | 4 +- .../clients/pgvectorscale/pgvectorscale.py | 4 +- .../clients/qdrant_cloud/qdrant_cloud.py | 4 +- vectordb_bench/backend/clients/test/cli.py | 2 +- vectordb_bench/backend/data_source.py | 16 ++---- vectordb_bench/backend/runner/mp_runner.py | 50 ++++++------------- vectordb_bench/backend/runner/rate_runner.py | 8 +-- .../backend/runner/read_write_runner.py | 24 ++++----- .../backend/runner/serial_runner.py | 46 ++++++++--------- vectordb_bench/backend/task_runner.py | 26 ++-------- vectordb_bench/interface.py | 27 ++++------ 15 files changed, 88 insertions(+), 151 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 312940634..6259bcea6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -133,6 +133,7 @@ lint.ignore = [ "RUF017", "C416", "PLW0603", + "COM812", ] # Allow autofix for all enabled rules (when `--fix`) is provided. diff --git a/vectordb_bench/backend/clients/__init__.py b/vectordb_bench/backend/clients/__init__.py index 773cd4948..e796aa069 100644 --- a/vectordb_bench/backend/clients/__init__.py +++ b/vectordb_bench/backend/clients/__init__.py @@ -42,7 +42,7 @@ class DB(Enum): AliyunOpenSearch = "AliyunOpenSearch" @property - def init_cls(self) -> type[VectorDB]: # noqa: PLR0911, PLR0912 + def init_cls(self) -> type[VectorDB]: # noqa: PLR0911, PLR0912, C901 """Import while in use""" if self == DB.Milvus: from .milvus.milvus import Milvus @@ -129,11 +129,16 @@ def init_cls(self) -> type[VectorDB]: # noqa: PLR0911, PLR0912 return AliyunOpenSearch + if self == DB.Test: + from .test.test import Test + + return Test + msg = f"Unknown DB: {self.name}" raise ValueError(msg) @property - def config_cls(self) -> type[DBConfig]: # noqa: PLR0911, PLR0912 + def config_cls(self) -> type[DBConfig]: # noqa: PLR0911, PLR0912, C901 """Import while in use""" if self == DB.Milvus: from .milvus.config import MilvusConfig @@ -220,6 +225,11 @@ def config_cls(self) -> type[DBConfig]: # noqa: PLR0911, PLR0912 return AliyunOpenSearchConfig + if self == DB.Test: + from .test.config import TestConfig + + return TestConfig + msg = f"Unknown DB: {self.name}" raise ValueError(msg) diff --git a/vectordb_bench/backend/clients/memorydb/cli.py b/vectordb_bench/backend/clients/memorydb/cli.py index ae00bfd17..568eec2a3 100644 --- a/vectordb_bench/backend/clients/memorydb/cli.py +++ b/vectordb_bench/backend/clients/memorydb/cli.py @@ -43,8 +43,8 @@ class MemoryDBTypedDict(TypedDict): show_default=True, default=False, help=( - "Cluster Mode Disabled (CMD), use this flag when testing locally on a single node instance.", - " In production, MemoryDB only supports cluster mode (CME)", + "Cluster Mode Disabled (CMD), use this flag when testing locally on a single node instance." + " In production, MemoryDB only supports cluster mode (CME)" ), ), ] diff --git a/vectordb_bench/backend/clients/pgvecto_rs/pgvecto_rs.py b/vectordb_bench/backend/clients/pgvecto_rs/pgvecto_rs.py index fc4f17807..64e95a1be 100644 --- a/vectordb_bench/backend/clients/pgvecto_rs/pgvecto_rs.py +++ b/vectordb_bench/backend/clients/pgvecto_rs/pgvecto_rs.py @@ -200,10 +200,7 @@ def _create_index(self): self.cursor.execute(index_create_sql) self.conn.commit() except Exception as e: - log.warning( - f"Failed to create pgvecto.rs index {self._index_name} \ - at table {self.table_name} error: {e}", - ) + log.warning(f"Failed to create pgvecto.rs index {self._index_name} at table {self.table_name} error: {e}") raise e from None def _create_table(self, dim: int): @@ -258,9 +255,7 @@ def insert_embeddings( return len(metadata), None except Exception as e: - log.warning( - f"Failed to insert data into pgvecto.rs table ({self.table_name}), error: {e}", - ) + log.warning(f"Failed to insert data into pgvecto.rs table ({self.table_name}), error: {e}") return 0, e def search_embedding( diff --git a/vectordb_bench/backend/clients/pgvector/pgvector.py b/vectordb_bench/backend/clients/pgvector/pgvector.py index 62a7971bb..bd024175c 100644 --- a/vectordb_bench/backend/clients/pgvector/pgvector.py +++ b/vectordb_bench/backend/clients/pgvector/pgvector.py @@ -415,9 +415,7 @@ def insert_embeddings( return len(metadata), None except Exception as e: - log.warning( - f"Failed to insert data into pgvector table ({self.table_name}), error: {e}", - ) + log.warning(f"Failed to insert data into pgvector table ({self.table_name}), error: {e}") return 0, e def search_embedding( diff --git a/vectordb_bench/backend/clients/pgvectorscale/pgvectorscale.py b/vectordb_bench/backend/clients/pgvectorscale/pgvectorscale.py index 981accc2e..ca7d809b4 100644 --- a/vectordb_bench/backend/clients/pgvectorscale/pgvectorscale.py +++ b/vectordb_bench/backend/clients/pgvectorscale/pgvectorscale.py @@ -255,9 +255,7 @@ def insert_embeddings( return len(metadata), None except Exception as e: - log.warning( - f"Failed to insert data into pgvector table ({self.table_name}), error: {e}", - ) + log.warning(f"Failed to insert data into pgvector table ({self.table_name}), error: {e}") return 0, e def search_embedding( diff --git a/vectordb_bench/backend/clients/qdrant_cloud/qdrant_cloud.py b/vectordb_bench/backend/clients/qdrant_cloud/qdrant_cloud.py index 0861e8938..a0d146a73 100644 --- a/vectordb_bench/backend/clients/qdrant_cloud/qdrant_cloud.py +++ b/vectordb_bench/backend/clients/qdrant_cloud/qdrant_cloud.py @@ -76,8 +76,8 @@ def optimize(self): continue if info.status == CollectionStatus.GREEN: msg = ( - f"Stored vectors: {info.vectors_count}, Indexed vectors: {info.indexed_vectors_count}, ", - f"Collection status: {info.indexed_vectors_count}", + f"Stored vectors: {info.vectors_count}, Indexed vectors: {info.indexed_vectors_count}, " + f"Collection status: {info.indexed_vectors_count}" ) log.info(msg) return diff --git a/vectordb_bench/backend/clients/test/cli.py b/vectordb_bench/backend/clients/test/cli.py index e5cd4c78b..2dcc4c407 100644 --- a/vectordb_bench/backend/clients/test/cli.py +++ b/vectordb_bench/backend/clients/test/cli.py @@ -17,7 +17,7 @@ class TestTypedDict(CommonTypedDict): ... @click_parameter_decorators_from_typed_dict(TestTypedDict) def Test(**parameters: Unpack[TestTypedDict]): run( - db=DB.NewClient, + db=DB.Test, db_config=TestConfig(db_label=parameters["db_label"]), db_case_config=TestIndexConfig(), **parameters, diff --git a/vectordb_bench/backend/data_source.py b/vectordb_bench/backend/data_source.py index b98dc7d7a..139d2e308 100644 --- a/vectordb_bench/backend/data_source.py +++ b/vectordb_bench/backend/data_source.py @@ -63,9 +63,7 @@ def validate_file(self, remote: pathlib.Path, local: pathlib.Path) -> bool: # check size equal remote_size, local_size = info.content_length, local.stat().st_size if remote_size != local_size: - log.info( - f"local file: {local} size[{local_size}] not match with remote size[{remote_size}]", - ) + log.info(f"local file: {local} size[{local_size}] not match with remote size[{remote_size}]") return False return True @@ -89,9 +87,7 @@ def read(self, dataset: str, files: list[str], local_ds_root: pathlib.Path): local_file = local_ds_root.joinpath(file) if (not local_file.exists()) or (not self.validate_file(remote_file, local_file)): - log.info( - f"local file: {local_file} not match with remote: {remote_file}; add to downloading list", - ) + log.info(f"local file: {local_file} not match with remote: {remote_file}; add to downloading list") downloads.append((remote_file, local_file)) if len(downloads) == 0: @@ -135,9 +131,7 @@ def read(self, dataset: str, files: list[str], local_ds_root: pathlib.Path): local_file = local_ds_root.joinpath(file) if (not local_file.exists()) or (not self.validate_file(remote_file, local_file)): - log.info( - f"local file: {local_file} not match with remote: {remote_file}; add to downloading list", - ) + log.info(f"local file: {local_file} not match with remote: {remote_file}; add to downloading list") downloads.append(remote_file) if len(downloads) == 0: @@ -157,9 +151,7 @@ def validate_file(self, remote: pathlib.Path, local: pathlib.Path) -> bool: # check size equal remote_size, local_size = info.get("size"), local.stat().st_size if remote_size != local_size: - log.info( - f"local file: {local} size[{local_size}] not match with remote size[{remote_size}]", - ) + log.info(f"local file: {local} size[{local_size}] not match with remote size[{remote_size}]") return False return True diff --git a/vectordb_bench/backend/runner/mp_runner.py b/vectordb_bench/backend/runner/mp_runner.py index 5b69b5481..687a0ecd7 100644 --- a/vectordb_bench/backend/runner/mp_runner.py +++ b/vectordb_bench/backend/runner/mp_runner.py @@ -79,14 +79,14 @@ def search( if count % 500 == 0: log.debug( - f"({mp.current_process().name:16}) ", - f"search_count: {count}, latest_latency={time.perf_counter()-s}", + f"({mp.current_process().name:16}) " + f"search_count: {count}, latest_latency={time.perf_counter()-s}" ) total_dur = round(time.perf_counter() - start_time, 4) log.info( f"{mp.current_process().name:16} search {self.duration}s: " - f"actual_dur={total_dur}s, count={count}, qps in this process: {round(count / total_dur, 4):3}", + f"actual_dur={total_dur}s, count={count}, qps in this process: {round(count / total_dur, 4):3}" ) return (count, total_dur, latencies) @@ -94,9 +94,7 @@ def search( @staticmethod def get_mp_context(): mp_start_method = "spawn" - log.debug( - f"MultiProcessingSearchRunner get multiprocessing start method: {mp_start_method}", - ) + log.debug(f"MultiProcessingSearchRunner get multiprocessing start method: {mp_start_method}") return mp.get_context(mp_start_method) def _run_all_concurrencies_mem_efficient(self): @@ -113,9 +111,7 @@ def _run_all_concurrencies_mem_efficient(self): mp_context=self.get_mp_context(), max_workers=conc, ) as executor: - log.info( - f"Start search {self.duration}s in concurrency {conc}, filters: {self.filters}", - ) + log.info(f"Start search {self.duration}s in concurrency {conc}, filters: {self.filters}") future_iter = [executor.submit(self.search, self.test_data, q, cond) for i in range(conc)] # Sync all processes while q.qsize() < conc: @@ -124,9 +120,7 @@ def _run_all_concurrencies_mem_efficient(self): with cond: cond.notify_all() - log.info( - f"Syncing all process and start concurrency search, concurrency={conc}", - ) + log.info(f"Syncing all process and start concurrency search, concurrency={conc}") start = time.perf_counter() all_count = sum([r.result()[0] for r in future_iter]) @@ -140,18 +134,14 @@ def _run_all_concurrencies_mem_efficient(self): conc_qps_list.append(qps) conc_latency_p99_list.append(latency_p99) conc_latency_avg_list.append(latency_avg) - log.info( - f"End search in concurrency {conc}: dur={cost}s, total_count={all_count}, qps={qps}", - ) + log.info(f"End search in concurrency {conc}: dur={cost}s, total_count={all_count}, qps={qps}") if qps > max_qps: max_qps = qps - log.info( - f"Update largest qps with concurrency {conc}: current max_qps={max_qps}", - ) + log.info(f"Update largest qps with concurrency {conc}: current max_qps={max_qps}") except Exception as e: log.warning( - f"Fail to search all concurrencies: {self.concurrencies}, max_qps before failure={max_qps}, reason={e}", + f"Fail to search, concurrencies: {self.concurrencies}, max_qps before failure={max_qps}, reason={e}" ) traceback.print_exc() @@ -193,9 +183,7 @@ def _run_by_dur(self, duration: int) -> float: mp_context=self.get_mp_context(), max_workers=conc, ) as executor: - log.info( - f"Start search_by_dur {duration}s in concurrency {conc}, filters: {self.filters}", - ) + log.info(f"Start search_by_dur {duration}s in concurrency {conc}, filters: {self.filters}") future_iter = [ executor.submit(self.search_by_dur, duration, self.test_data, q, cond) for i in range(conc) ] @@ -206,24 +194,18 @@ def _run_by_dur(self, duration: int) -> float: with cond: cond.notify_all() - log.info( - f"Syncing all process and start concurrency search, concurrency={conc}", - ) + log.info(f"Syncing all process and start concurrency search, concurrency={conc}") start = time.perf_counter() all_count = sum([r.result() for r in future_iter]) cost = time.perf_counter() - start qps = round(all_count / cost, 4) - log.info( - f"End search in concurrency {conc}: dur={cost}s, total_count={all_count}, qps={qps}", - ) + log.info(f"End search in concurrency {conc}: dur={cost}s, total_count={all_count}, qps={qps}") if qps > max_qps: max_qps = qps - log.info( - f"Update largest qps with concurrency {conc}: current max_qps={max_qps}", - ) + log.info(f"Update largest qps with concurrency {conc}: current max_qps={max_qps}") except Exception as e: log.warning( f"Fail to search all concurrencies: {self.concurrencies}, max_qps before failure={max_qps}, reason={e}", @@ -275,14 +257,14 @@ def search_by_dur( if count % 500 == 0: log.debug( - f"({mp.current_process().name:16}) search_count: {count}, ", - f"latest_latency={time.perf_counter()-s}", + f"({mp.current_process().name:16}) search_count: {count}, " + f"latest_latency={time.perf_counter()-s}" ) total_dur = round(time.perf_counter() - start_time, 4) log.debug( f"{mp.current_process().name:16} search {self.duration}s: " - f"actual_dur={total_dur}s, count={count}, qps in this process: {round(count / total_dur, 4):3}", + f"actual_dur={total_dur}s, count={count}, qps in this process: {round(count / total_dur, 4):3}" ) return count diff --git a/vectordb_bench/backend/runner/rate_runner.py b/vectordb_bench/backend/runner/rate_runner.py index 0145af4ce..4b32bcd9f 100644 --- a/vectordb_bench/backend/runner/rate_runner.py +++ b/vectordb_bench/backend/runner/rate_runner.py @@ -73,14 +73,14 @@ def submit_by_rate() -> bool: if len(not_done) > 0: log.warning( - f"Failed to finish all tasks in 1s, [{len(not_done)}/{len(executing_futures)}] ", - f"tasks are not done, waited={wait_interval:.2f}, trying to wait in the next round", + f"Failed to finish all tasks in 1s, [{len(not_done)}/{len(executing_futures)}] " + f"tasks are not done, waited={wait_interval:.2f}, trying to wait in the next round" ) executing_futures = list(not_done) else: log.debug( - f"Finished {len(executing_futures)} insert-{config.NUM_PER_BATCH} ", - f"task in 1s, wait_interval={wait_interval:.2f}", + f"Finished {len(executing_futures)} insert-{config.NUM_PER_BATCH} " + f"task in 1s, wait_interval={wait_interval:.2f}" ) executing_futures = [] except Exception as e: diff --git a/vectordb_bench/backend/runner/read_write_runner.py b/vectordb_bench/backend/runner/read_write_runner.py index e916f45d6..d7584459a 100644 --- a/vectordb_bench/backend/runner/read_write_runner.py +++ b/vectordb_bench/backend/runner/read_write_runner.py @@ -45,8 +45,8 @@ def __init__( self.read_dur_after_write = read_dur_after_write log.info( - f"Init runner, concurencys={concurrencies}, search_stage={search_stage}, ", - f"stage_search_dur={read_dur_after_write}", + f"Init runner, concurencys={concurrencies}, search_stage={search_stage}, " + f"stage_search_dur={read_dur_after_write}" ) test_emb = np.stack(dataset.test_data["emb"]) @@ -88,12 +88,10 @@ def run_search(self): res, ssearch_dur = self.serial_search_runner.run() recall, ndcg, p99_latency = res log.info( - f"Search after write - Serial search - recall={recall}, ndcg={ndcg}, p99={p99_latency}, ", + f"Search after write - Serial search - recall={recall}, ndcg={ndcg}, p99={p99_latency}, " f"dur={ssearch_dur:.4f}", ) - log.info( - f"Search after wirte - Conc search start, dur for each conc={self.read_dur_after_write}", - ) + log.info(f"Search after wirte - Conc search start, dur for each conc={self.read_dur_after_write}") max_qps = self.run_by_dur(self.read_dur_after_write) log.info(f"Search after wirte - Conc search finished, max_qps={max_qps}") @@ -157,9 +155,7 @@ def wait_next_target(start: int, target_batch: int) -> bool: got = wait_next_target(start_batch, target_batch) if got is False: - log.warning( - f"Abnormal exit, target_batch={target_batch}, start_batch={start_batch}", - ) + log.warning(f"Abnormal exit, target_batch={target_batch}, start_batch={start_batch}") return None log.info(f"Insert {perc}% done, total batch={total_batch}") @@ -167,8 +163,8 @@ def wait_next_target(start: int, target_batch: int) -> bool: res, ssearch_dur = self.serial_search_runner.run() recall, ndcg, p99_latency = res log.info( - f"[{target_batch}/{total_batch}] Serial search - {perc}% done, recall={recall}, ", - f"ndcg={ndcg}, p99={p99_latency}, dur={ssearch_dur:.4f}", + f"[{target_batch}/{total_batch}] Serial search - {perc}% done, recall={recall}, " + f"ndcg={ndcg}, p99={p99_latency}, dur={ssearch_dur:.4f}" ) # Search duration for non-last search stage is carefully calculated. @@ -183,8 +179,8 @@ def wait_next_target(start: int, target_batch: int) -> bool: each_conc_search_dur = csearch_dur / len(self.concurrencies) if each_conc_search_dur < 30: warning_msg = ( - f"Results might be inaccurate, duration[{csearch_dur:.4f}] left for conc-search is too short, ", - f"total available dur={total_dur_between_stages}, serial_search_cost={ssearch_dur}.", + f"Results might be inaccurate, duration[{csearch_dur:.4f}] left for conc-search is too short, " + f"total available dur={total_dur_between_stages}, serial_search_cost={ssearch_dur}." ) log.warning(warning_msg) @@ -193,7 +189,7 @@ def wait_next_target(start: int, target_batch: int) -> bool: each_conc_search_dur = 60 log.info( - f"[{target_batch}/{total_batch}] Concurrent search - {perc}% start, dur={each_conc_search_dur:.4f}", + f"[{target_batch}/{total_batch}] Concurrent search - {perc}% start, dur={each_conc_search_dur:.4f}" ) max_qps = self.run_by_dur(each_conc_search_dur) result.append((perc, max_qps, recall, ndcg, p99_latency)) diff --git a/vectordb_bench/backend/runner/serial_runner.py b/vectordb_bench/backend/runner/serial_runner.py index 7eb59432b..08d42e14c 100644 --- a/vectordb_bench/backend/runner/serial_runner.py +++ b/vectordb_bench/backend/runner/serial_runner.py @@ -40,9 +40,7 @@ def __init__( def task(self) -> int: count = 0 with self.db.init(): - log.info( - f"({mp.current_process().name:16}) Start inserting embeddings in batch {config.NUM_PER_BATCH}", - ) + log.info(f"({mp.current_process().name:16}) Start inserting embeddings in batch {config.NUM_PER_BATCH}") start = time.perf_counter() for data_df in self.dataset: all_metadata = data_df["id"].tolist() @@ -66,13 +64,11 @@ def task(self) -> int: assert insert_count == len(all_metadata) count += insert_count if count % 100_000 == 0: - log.info( - f"({mp.current_process().name:16}) Loaded {count} embeddings into VectorDB", - ) + log.info(f"({mp.current_process().name:16}) Loaded {count} embeddings into VectorDB") log.info( - f"({mp.current_process().name:16}) Finish loading all dataset into VectorDB, ", - f"dur={time.perf_counter()-start}", + f"({mp.current_process().name:16}) Finish loading all dataset into VectorDB, " + f"dur={time.perf_counter()-start}" ) return count @@ -83,8 +79,8 @@ def endless_insert_data(self, all_embeddings: list, all_metadata: list, left_id: num_batches = math.ceil(len(all_embeddings) / NUM_PER_BATCH) log.info( - f"({mp.current_process().name:16}) Start inserting {len(all_embeddings)} ", - f"embeddings in batch {NUM_PER_BATCH}", + f"({mp.current_process().name:16}) Start inserting {len(all_embeddings)} " + f"embeddings in batch {NUM_PER_BATCH}" ) count = 0 for batch_id in range(num_batches): @@ -94,8 +90,8 @@ def endless_insert_data(self, all_embeddings: list, all_metadata: list, left_id: embeddings = all_embeddings[batch_id * NUM_PER_BATCH : (batch_id + 1) * NUM_PER_BATCH] log.debug( - f"({mp.current_process().name:16}) batch [{batch_id:3}/{num_batches}], ", - f"Start inserting {len(metadata)} embeddings", + f"({mp.current_process().name:16}) batch [{batch_id:3}/{num_batches}], " + f"Start inserting {len(metadata)} embeddings" ) while retry_count < LOAD_MAX_TRY_COUNT: insert_count, error = self.db.insert_embeddings( @@ -113,15 +109,15 @@ def endless_insert_data(self, all_embeddings: list, all_metadata: list, left_id: else: break log.debug( - f"({mp.current_process().name:16}) batch [{batch_id:3}/{num_batches}], ", - f"Finish inserting {len(metadata)} embeddings", + f"({mp.current_process().name:16}) batch [{batch_id:3}/{num_batches}], " + f"Finish inserting {len(metadata)} embeddings" ) assert already_insert_count == len(metadata) count += already_insert_count log.info( - f"({mp.current_process().name:16}) Finish inserting {len(all_embeddings)} embeddings in ", - f"batch {NUM_PER_BATCH}", + f"({mp.current_process().name:16}) Finish inserting {len(all_embeddings)} embeddings in " + f"batch {NUM_PER_BATCH}" ) return count @@ -171,13 +167,13 @@ def run_endlessness(self) -> int: max_load_count += count times += 1 log.info( - f"Loaded {times} entire dataset, current max load counts={utils.numerize(max_load_count)}, ", - f"{max_load_count}", + f"Loaded {times} entire dataset, current max load counts={utils.numerize(max_load_count)}, " + f"{max_load_count}" ) except Exception as e: log.info( - f"Capacity case load reach limit, insertion counts={utils.numerize(max_load_count)}, ", - f"{max_load_count}, err={e}", + f"Capacity case load reach limit, insertion counts={utils.numerize(max_load_count)}, " + f"{max_load_count}, err={e}" ) traceback.print_exc() return max_load_count @@ -209,9 +205,7 @@ def __init__( self.ground_truth = ground_truth def search(self, args: tuple[list, pd.DataFrame]) -> tuple[float, float, float]: - log.info( - f"{mp.current_process().name:14} start search the entire test_data to get recall and latency", - ) + log.info(f"{mp.current_process().name:14} start search the entire test_data to get recall and latency") with self.db.init(): test_data, ground_truth = args ideal_dcg = get_ideal_dcg(self.k) @@ -242,8 +236,8 @@ def search(self, args: tuple[list, pd.DataFrame]) -> tuple[float, float, float]: if len(latencies) % 100 == 0: log.debug( - f"({mp.current_process().name:14}) search_count={len(latencies):3}, ", - f"latest_latency={latencies[-1]}, latest recall={recalls[-1]}", + f"({mp.current_process().name:14}) search_count={len(latencies):3}, " + f"latest_latency={latencies[-1]}, latest recall={recalls[-1]}" ) avg_latency = round(np.mean(latencies), 4) @@ -258,7 +252,7 @@ def search(self, args: tuple[list, pd.DataFrame]) -> tuple[float, float, float]: f"avg_recall={avg_recall}, " f"avg_ndcg={avg_ndcg}," f"avg_latency={avg_latency}, " - f"p99={p99}", + f"p99={p99}" ) return (avg_recall, avg_ndcg, p99) diff --git a/vectordb_bench/backend/task_runner.py b/vectordb_bench/backend/task_runner.py index e24d74f03..e8be9f07d 100644 --- a/vectordb_bench/backend/task_runner.py +++ b/vectordb_bench/backend/task_runner.py @@ -98,9 +98,7 @@ def _pre_run(self, drop_old: bool = True): self.init_db(drop_old) self.ca.dataset.prepare(self.dataset_source, filters=self.ca.filter_rate) except ModuleNotFoundError as e: - log.warning( - f"pre run case error: please install client for db: {self.config.db}, error={e}", - ) + log.warning(f"pre run case error: please install client for db: {self.config.db}, error={e}") raise e from None def run(self, drop_old: bool = True) -> Metric: @@ -136,9 +134,7 @@ def _run_capacity_case(self) -> Metric: log.warning(f"Failed to run capacity case, reason = {e}") raise e from None else: - log.info( - f"Capacity case loading dataset reaches VectorDB's limit: max capacity = {count}", - ) + log.info(f"Capacity case loading dataset reaches VectorDB's limit: max capacity = {count}") return Metric(max_load_count=count) def _run_perf_case(self, drop_old: bool = True) -> Metric: @@ -147,22 +143,6 @@ def _run_perf_case(self, drop_old: bool = True) -> Metric: Returns: Metric: load_duration, recall, serial_latency_p99, and, qps """ - """ - if drop_old: - _, load_dur = self._load_train_data() - build_dur = self._optimize() - m.load_duration = round(load_dur+build_dur, 4) - log.info( - f"Finish loading the entire dataset into VectorDB," - f" insert_duration={load_dur}, optimize_duration={build_dur}" - f" load_duration(insert + optimize) = {m.load_duration}" - ) - - self._init_search_runner() - - m.qps, m.conc_num_list, m.conc_qps_list, m.conc_latency_p99_list = self._conc_search() - m.recall, m.serial_latency_p99 = self._serial_search() - """ log.info("Start performance case") try: @@ -175,7 +155,7 @@ def _run_perf_case(self, drop_old: bool = True) -> Metric: log.info( f"Finish loading the entire dataset into VectorDB," f" insert_duration={load_dur}, optimize_duration={build_dur}" - f" load_duration(insert + optimize) = {m.load_duration}", + f" load_duration(insert + optimize) = {m.load_duration}" ) else: log.info("Data loading skipped") diff --git a/vectordb_bench/interface.py b/vectordb_bench/interface.py index ebe12d2e6..2e573fdc7 100644 --- a/vectordb_bench/interface.py +++ b/vectordb_bench/interface.py @@ -65,9 +65,7 @@ def run(self, tasks: list[TaskConfig], task_label: str | None = None) -> bool: log.warning("Empty tasks submitted") return False - log.debug( - f"tasks: {tasks}, task_label: {task_label}, dataset source: {self.dataset_source}", - ) + log.debug(f"tasks: {tasks}, task_label: {task_label}, dataset source: {self.dataset_source}") # Generate run_id run_id = uuid.uuid4().hex @@ -169,14 +167,13 @@ def _async_task_v2(self, running_task: TaskRunner, send_conn: Connection) -> Non drop_old = TaskStage.DROP_OLD in runner.config.stages if (latest_runner and runner == latest_runner) or not self.drop_old: drop_old = False + num_cases = running_task.num_cases() try: - log.info( - f"[{idx+1}/{running_task.num_cases()}] start case: {runner.display()}, drop_old={drop_old}", - ) + log.info(f"[{idx+1}/{num_cases}] start case: {runner.display()}, drop_old={drop_old}") case_res.metrics = runner.run(drop_old) log.info( - f"[{idx+1}/{running_task.num_cases()}] finish case: {runner.display()}, " - f"result={case_res.metrics}, label={case_res.label}", + f"[{idx+1}/{num_cases}] finish case: {runner.display()}, " + f"result={case_res.metrics}, label={case_res.label}" ) # cache the latest succeeded runner @@ -189,16 +186,12 @@ def _async_task_v2(self, running_task: TaskRunner, send_conn: Connection) -> Non if not drop_old: case_res.metrics.load_duration = cached_load_duration if cached_load_duration else 0.0 except (LoadTimeoutError, PerformanceTimeoutError) as e: - log.warning( - f"[{idx+1}/{running_task.num_cases()}] case {runner.display()} failed to run, reason={e}", - ) + log.warning(f"[{idx+1}/{num_cases}] case {runner.display()} failed to run, reason={e}") case_res.label = ResultLabel.OUTOFRANGE continue except Exception as e: - log.warning( - f"[{idx+1}/{running_task.num_cases()}] case {runner.display()} failed to run, reason={e}", - ) + log.warning(f"[{idx+1}/{num_cases}] case {runner.display()} failed to run, reason={e}") traceback.print_exc() case_res.label = ResultLabel.FAILED continue @@ -217,9 +210,7 @@ def _async_task_v2(self, running_task: TaskRunner, send_conn: Connection) -> Non send_conn.send((SIGNAL.SUCCESS, None)) send_conn.close() - log.info( - f"Success to finish task: label={running_task.task_label}, run_id={running_task.run_id}", - ) + log.info(f"Success to finish task: label={running_task.task_label}, run_id={running_task.run_id}") except Exception as e: err_msg = ( @@ -250,7 +241,7 @@ def _clear_running_task(self): def _run_async(self, conn: Connection) -> bool: log.info( f"task submitted: id={self.running_task.run_id}, {self.running_task.task_label}, " - f"case number: {len(self.running_task.case_runners)}", + f"case number: {len(self.running_task.case_runners)}" ) global global_result_future executor = concurrent.futures.ProcessPoolExecutor( From 6dfcadf6436594afa767b5b59fdece0120609099 Mon Sep 17 00:00:00 2001 From: yangxuan Date: Mon, 13 Jan 2025 19:45:51 +0800 Subject: [PATCH 134/327] enhance: Unify optimize and remove ready_to_load PyMilvus used to be the only client that uses ready_to_load. Not it'll load the collection when creating it, so this PR removes `ready_to_load` from the client.API Also this PR enhance optimize and remove the optimize_with_size Signed-off-by: yangxuan --- .../aliyun_opensearch/aliyun_opensearch.py | 8 +------ .../backend/clients/alloydb/alloydb.py | 5 +--- vectordb_bench/backend/clients/api.py | 23 +++++++------------ .../clients/aws_opensearch/aws_opensearch.py | 11 ++++----- .../backend/clients/chroma/chroma.py | 5 +--- .../clients/elastic_cloud/elastic_cloud.py | 5 +--- .../backend/clients/memorydb/memorydb.py | 7 ++---- .../backend/clients/milvus/milvus.py | 21 +---------------- .../backend/clients/pgdiskann/pgdiskann.py | 5 +--- .../backend/clients/pgvecto_rs/pgvecto_rs.py | 5 +--- .../backend/clients/pgvector/pgvector.py | 5 +--- .../clients/pgvectorscale/pgvectorscale.py | 5 +--- .../backend/clients/pinecone/pinecone.py | 5 +--- .../clients/qdrant_cloud/qdrant_cloud.py | 5 +--- vectordb_bench/backend/clients/redis/redis.py | 5 +--- vectordb_bench/backend/clients/test/test.py | 5 +--- .../clients/weaviate_cloud/weaviate_cloud.py | 5 +--- .../backend/runner/read_write_runner.py | 2 +- .../backend/runner/serial_runner.py | 4 +--- vectordb_bench/backend/task_runner.py | 6 ++--- 20 files changed, 33 insertions(+), 109 deletions(-) diff --git a/vectordb_bench/backend/clients/aliyun_opensearch/aliyun_opensearch.py b/vectordb_bench/backend/clients/aliyun_opensearch/aliyun_opensearch.py index 00227cfff..324871934 100644 --- a/vectordb_bench/backend/clients/aliyun_opensearch/aliyun_opensearch.py +++ b/vectordb_bench/backend/clients/aliyun_opensearch/aliyun_opensearch.py @@ -325,10 +325,7 @@ def need_normalize_cosine(self) -> bool: return False - def optimize(self): - pass - - def optimize_with_size(self, data_size: int): + def optimize(self, data_size: int): log.info(f"optimize count: {data_size}") retry_times = 0 while True: @@ -340,6 +337,3 @@ def optimize_with_size(self, data_size: int): if total_count == data_size: log.info("optimize table finish.") return - - def ready_to_load(self): - """ready_to_load will be called before load in load cases.""" diff --git a/vectordb_bench/backend/clients/alloydb/alloydb.py b/vectordb_bench/backend/clients/alloydb/alloydb.py index c81f77675..b9808ce54 100644 --- a/vectordb_bench/backend/clients/alloydb/alloydb.py +++ b/vectordb_bench/backend/clients/alloydb/alloydb.py @@ -149,10 +149,7 @@ def _drop_table(self): ) self.conn.commit() - def ready_to_load(self): - pass - - def optimize(self): + def optimize(self, data_size: int | None = None): self._post_insert() def _post_insert(self): diff --git a/vectordb_bench/backend/clients/api.py b/vectordb_bench/backend/clients/api.py index aa93abc12..a86849e96 100644 --- a/vectordb_bench/backend/clients/api.py +++ b/vectordb_bench/backend/clients/api.py @@ -137,6 +137,13 @@ def __init__( @contextmanager def init(self) -> None: """create and destory connections to database. + Why contextmanager: + + In multiprocessing search tasks, vectordbbench might init + totally hundreds of thousands of connections with DB server. + + Too many connections may drain local FDs or server connection resources. + If the DB client doesn't have `close()` method, just set the object to None. Examples: >>> with self.init(): @@ -187,9 +194,8 @@ def search_embedding( """ raise NotImplementedError - # TODO: remove @abstractmethod - def optimize(self): + def optimize(self, data_size: int | None = None): """optimize will be called between insertion and search in performance cases. Should be blocked until the vectorDB is ready to be tested on @@ -199,16 +205,3 @@ def optimize(self): Optimize's execution time is limited, the limited time is based on cases. """ raise NotImplementedError - - def optimize_with_size(self, data_size: int): - self.optimize() - - # TODO: remove - @abstractmethod - def ready_to_load(self): - """ready_to_load will be called before load in load cases. - - Should be blocked until the vectorDB is ready to be tested on - heavy load cases. - """ - raise NotImplementedError diff --git a/vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py b/vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py index 487ec67cc..234014f19 100644 --- a/vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py +++ b/vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py @@ -145,15 +145,15 @@ def search_embedding( docvalue_fields=[self.id_col_name], stored_fields="_none_", ) - log.info(f'Search took: {resp["took"]}') - log.info(f'Search shards: {resp["_shards"]}') - log.info(f'Search hits total: {resp["hits"]["total"]}') + log.info(f"Search took: {resp['took']}") + log.info(f"Search shards: {resp['_shards']}") + log.info(f"Search hits total: {resp['hits']['total']}") return [int(h["fields"][self.id_col_name][0]) for h in resp["hits"]["hits"]] except Exception as e: log.warning(f"Failed to search: {self.index_name} error: {e!s}") raise e from None - def optimize(self): + def optimize(self, data_size: int | None = None): """optimize will be called between insertion and search in performance cases.""" # Call refresh first to ensure that all segments are created self._refresh_index() @@ -194,6 +194,3 @@ def _load_graphs_to_memory(self): log.info("Calling warmup API to load graphs into memory") warmup_endpoint = f"/_plugins/_knn/warmup/{self.index_name}" self.client.transport.perform_request("GET", warmup_endpoint) - - def ready_to_load(self): - """ready_to_load will be called before load in load cases.""" diff --git a/vectordb_bench/backend/clients/chroma/chroma.py b/vectordb_bench/backend/clients/chroma/chroma.py index a148fa141..76c810263 100644 --- a/vectordb_bench/backend/clients/chroma/chroma.py +++ b/vectordb_bench/backend/clients/chroma/chroma.py @@ -57,10 +57,7 @@ def init(self) -> None: def ready_to_search(self) -> bool: pass - def ready_to_load(self) -> bool: - pass - - def optimize(self) -> None: + def optimize(self, data_size: int | None = None): pass def insert_embeddings( diff --git a/vectordb_bench/backend/clients/elastic_cloud/elastic_cloud.py b/vectordb_bench/backend/clients/elastic_cloud/elastic_cloud.py index a3183bcb7..ea038c587 100644 --- a/vectordb_bench/backend/clients/elastic_cloud/elastic_cloud.py +++ b/vectordb_bench/backend/clients/elastic_cloud/elastic_cloud.py @@ -143,7 +143,7 @@ def search_embedding( log.warning(f"Failed to search: {self.indice} error: {e!s}") raise e from None - def optimize(self): + def optimize(self, data_size: int | None = None): """optimize will be called between insertion and search in performance cases.""" assert self.client is not None, "should self.init() first" self.client.indices.refresh(index=self.indice) @@ -158,6 +158,3 @@ def optimize(self): task_status = self.client.tasks.get(task_id=force_merge_task_id) if task_status["completed"]: return - - def ready_to_load(self): - """ready_to_load will be called before load in load cases.""" diff --git a/vectordb_bench/backend/clients/memorydb/memorydb.py b/vectordb_bench/backend/clients/memorydb/memorydb.py index d05e30be1..9d077f5df 100644 --- a/vectordb_bench/backend/clients/memorydb/memorydb.py +++ b/vectordb_bench/backend/clients/memorydb/memorydb.py @@ -157,17 +157,14 @@ def init(self) -> Generator[None, None, None]: self.conn = self.get_client() search_param = self.case_config.search_param() if search_param["ef_runtime"]: - self.ef_runtime_str = f'EF_RUNTIME {search_param["ef_runtime"]}' + self.ef_runtime_str = f"EF_RUNTIME {search_param['ef_runtime']}" else: self.ef_runtime_str = "" yield self.conn.close() self.conn = None - def ready_to_load(self) -> bool: - pass - - def optimize(self) -> None: + def optimize(self, data_size: int | None = None): self._post_insert() def insert_embeddings( diff --git a/vectordb_bench/backend/clients/milvus/milvus.py b/vectordb_bench/backend/clients/milvus/milvus.py index 45fe7269b..4015eb1f3 100644 --- a/vectordb_bench/backend/clients/milvus/milvus.py +++ b/vectordb_bench/backend/clients/milvus/milvus.py @@ -138,26 +138,7 @@ def wait_index(): log.warning(f"{self.name} optimize error: {e}") raise e from None - def ready_to_load(self): - assert self.col, "Please call self.init() before" - self._pre_load(self.col) - - def _pre_load(self, coll: Collection): - try: - if not coll.has_index(index_name=self._index_name): - log.info(f"{self.name} create index") - coll.create_index( - self._vector_field, - self.case_config.index_param(), - index_name=self._index_name, - ) - coll.load() - log.info(f"{self.name} load") - except Exception as e: - log.warning(f"{self.name} pre load error: {e}") - raise e from None - - def optimize(self): + def optimize(self, data_size: int | None = None): assert self.col, "Please call self.init() before" self._optimize() diff --git a/vectordb_bench/backend/clients/pgdiskann/pgdiskann.py b/vectordb_bench/backend/clients/pgdiskann/pgdiskann.py index c21972902..8bede0f01 100644 --- a/vectordb_bench/backend/clients/pgdiskann/pgdiskann.py +++ b/vectordb_bench/backend/clients/pgdiskann/pgdiskann.py @@ -143,10 +143,7 @@ def _drop_table(self): ) self.conn.commit() - def ready_to_load(self): - pass - - def optimize(self): + def optimize(self, data_size: int | None = None): self._post_insert() def _post_insert(self): diff --git a/vectordb_bench/backend/clients/pgvecto_rs/pgvecto_rs.py b/vectordb_bench/backend/clients/pgvecto_rs/pgvecto_rs.py index 64e95a1be..3006b861a 100644 --- a/vectordb_bench/backend/clients/pgvecto_rs/pgvecto_rs.py +++ b/vectordb_bench/backend/clients/pgvecto_rs/pgvecto_rs.py @@ -153,10 +153,7 @@ def _drop_table(self): ) self.conn.commit() - def ready_to_load(self): - pass - - def optimize(self): + def optimize(self, data_size: int | None = None): self._post_insert() def _post_insert(self): diff --git a/vectordb_bench/backend/clients/pgvector/pgvector.py b/vectordb_bench/backend/clients/pgvector/pgvector.py index bd024175c..4164461fb 100644 --- a/vectordb_bench/backend/clients/pgvector/pgvector.py +++ b/vectordb_bench/backend/clients/pgvector/pgvector.py @@ -228,10 +228,7 @@ def _drop_table(self): ) self.conn.commit() - def ready_to_load(self): - pass - - def optimize(self): + def optimize(self, data_size: int | None = None): self._post_insert() def _post_insert(self): diff --git a/vectordb_bench/backend/clients/pgvectorscale/pgvectorscale.py b/vectordb_bench/backend/clients/pgvectorscale/pgvectorscale.py index ca7d809b4..3985c0716 100644 --- a/vectordb_bench/backend/clients/pgvectorscale/pgvectorscale.py +++ b/vectordb_bench/backend/clients/pgvectorscale/pgvectorscale.py @@ -143,10 +143,7 @@ def _drop_table(self): ) self.conn.commit() - def ready_to_load(self): - pass - - def optimize(self): + def optimize(self, data_size: int | None = None): self._post_insert() def _post_insert(self): diff --git a/vectordb_bench/backend/clients/pinecone/pinecone.py b/vectordb_bench/backend/clients/pinecone/pinecone.py index c59ee8760..1a681b33f 100644 --- a/vectordb_bench/backend/clients/pinecone/pinecone.py +++ b/vectordb_bench/backend/clients/pinecone/pinecone.py @@ -59,10 +59,7 @@ def init(self): self.index = pc.Index(self.index_name) yield - def ready_to_load(self): - pass - - def optimize(self): + def optimize(self, data_size: int | None = None): pass def insert_embeddings( diff --git a/vectordb_bench/backend/clients/qdrant_cloud/qdrant_cloud.py b/vectordb_bench/backend/clients/qdrant_cloud/qdrant_cloud.py index a0d146a73..5de72798b 100644 --- a/vectordb_bench/backend/clients/qdrant_cloud/qdrant_cloud.py +++ b/vectordb_bench/backend/clients/qdrant_cloud/qdrant_cloud.py @@ -62,10 +62,7 @@ def init(self) -> None: self.qdrant_client = None del self.qdrant_client - def ready_to_load(self): - pass - - def optimize(self): + def optimize(self, data_size: int | None = None): assert self.qdrant_client, "Please call self.init() before" # wait for vectors to be fully indexed try: diff --git a/vectordb_bench/backend/clients/redis/redis.py b/vectordb_bench/backend/clients/redis/redis.py index 139850d2f..ef0aad9aa 100644 --- a/vectordb_bench/backend/clients/redis/redis.py +++ b/vectordb_bench/backend/clients/redis/redis.py @@ -95,10 +95,7 @@ def init(self) -> None: def ready_to_search(self) -> bool: """Check if the database is ready to search.""" - def ready_to_load(self) -> bool: - pass - - def optimize(self) -> None: + def optimize(self, data_size: int | None = None): pass def insert_embeddings( diff --git a/vectordb_bench/backend/clients/test/test.py b/vectordb_bench/backend/clients/test/test.py index ee5a523f3..d2bcb74b5 100644 --- a/vectordb_bench/backend/clients/test/test.py +++ b/vectordb_bench/backend/clients/test/test.py @@ -33,10 +33,7 @@ def init(self) -> Generator[None, None, None]: yield - def ready_to_load(self) -> bool: - return True - - def optimize(self) -> None: + def optimize(self, data_size: int | None = None): pass def insert_embeddings( diff --git a/vectordb_bench/backend/clients/weaviate_cloud/weaviate_cloud.py b/vectordb_bench/backend/clients/weaviate_cloud/weaviate_cloud.py index b42f70af1..aa4368bb7 100644 --- a/vectordb_bench/backend/clients/weaviate_cloud/weaviate_cloud.py +++ b/vectordb_bench/backend/clients/weaviate_cloud/weaviate_cloud.py @@ -67,10 +67,7 @@ def init(self) -> None: self.client = None del self.client - def ready_to_load(self): - """Should call insert first, do nothing""" - - def optimize(self): + def optimize(self, data_size: int | None = None): assert self.client.schema.exists(self.collection_name) self.client.schema.update_config( self.collection_name, diff --git a/vectordb_bench/backend/runner/read_write_runner.py b/vectordb_bench/backend/runner/read_write_runner.py index d7584459a..eaba51f5f 100644 --- a/vectordb_bench/backend/runner/read_write_runner.py +++ b/vectordb_bench/backend/runner/read_write_runner.py @@ -80,7 +80,7 @@ def run_optimize(self): """Optimize needs to run in differenct process for pymilvus schema recursion problem""" with self.db.init(): log.info("Search after write - Optimize start") - self.db.optimize() + self.db.optimize(data_size=self.data_volume) log.info("Search after write - Optimize finished") def run_search(self): diff --git a/vectordb_bench/backend/runner/serial_runner.py b/vectordb_bench/backend/runner/serial_runner.py index 08d42e14c..365641132 100644 --- a/vectordb_bench/backend/runner/serial_runner.py +++ b/vectordb_bench/backend/runner/serial_runner.py @@ -68,7 +68,7 @@ def task(self) -> int: log.info( f"({mp.current_process().name:16}) Finish loading all dataset into VectorDB, " - f"dur={time.perf_counter()-start}" + f"dur={time.perf_counter() - start}" ) return count @@ -156,8 +156,6 @@ def run_endlessness(self) -> int: start_time = time.perf_counter() max_load_count, times = 0, 0 try: - with self.db.init(): - self.db.ready_to_load() while time.perf_counter() - start_time < self.timeout: count = self.endless_insert_data( all_embeddings, diff --git a/vectordb_bench/backend/task_runner.py b/vectordb_bench/backend/task_runner.py index e8be9f07d..2a583b4f5 100644 --- a/vectordb_bench/backend/task_runner.py +++ b/vectordb_bench/backend/task_runner.py @@ -234,13 +234,13 @@ def _conc_search(self): self.stop() @utils.time_it - def _task(self) -> None: + def _optimize_task(self) -> None: with self.db.init(): - self.db.optimize_with_size(data_size=self.ca.dataset.data.size) + self.db.optimize(data_size=self.ca.dataset.data.size) def _optimize(self) -> float: with concurrent.futures.ProcessPoolExecutor(max_workers=1) as executor: - future = executor.submit(self._task) + future = executor.submit(self._optimize_task) try: return future.result(timeout=self.ca.optimize_timeout)[1] except TimeoutError as e: From b1c43fd81e2180adf755526d7173115e3d133f14 Mon Sep 17 00:00:00 2001 From: zhuwenxing Date: Tue, 14 Jan 2025 10:34:43 +0800 Subject: [PATCH 135/327] add mongodb client Signed-off-by: zhuwenxing --- pyproject.toml | 2 +- vectordb_bench/backend/clients/__init__.py | 16 ++ .../backend/clients/mongodb/config.py | 44 ++++ .../backend/clients/mongodb/mongodb.py | 201 ++++++++++++++++++ 4 files changed, 262 insertions(+), 1 deletion(-) create mode 100644 vectordb_bench/backend/clients/mongodb/config.py create mode 100644 vectordb_bench/backend/clients/mongodb/mongodb.py diff --git a/pyproject.toml b/pyproject.toml index 6259bcea6..aafe70750 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,6 +85,7 @@ memorydb = [ "memorydb" ] chromadb = [ "chromadb" ] opensearch = [ "opensearch-py" ] aliyun_opensearch = [ "alibabacloud_ha3engine_vector", "alibabacloud_searchengine20211025"] +mongodb = [ "pymongo" ] [project.urls] "repository" = "https://github.com/zilliztech/VectorDBBench" @@ -207,4 +208,3 @@ builtins-ignorelist = [ # "dict", # TODO # "filter", ] - diff --git a/vectordb_bench/backend/clients/__init__.py b/vectordb_bench/backend/clients/__init__.py index e796aa069..f2f480dcf 100644 --- a/vectordb_bench/backend/clients/__init__.py +++ b/vectordb_bench/backend/clients/__init__.py @@ -40,6 +40,7 @@ class DB(Enum): AliyunElasticsearch = "AliyunElasticsearch" Test = "test" AliyunOpenSearch = "AliyunOpenSearch" + MongoDB = "MongoDB" @property def init_cls(self) -> type[VectorDB]: # noqa: PLR0911, PLR0912, C901 @@ -129,6 +130,11 @@ def init_cls(self) -> type[VectorDB]: # noqa: PLR0911, PLR0912, C901 return AliyunOpenSearch + if self == DB.MongoDB: + from .mongodb.mongodb import MongoDB + + return MongoDB + if self == DB.Test: from .test.test import Test @@ -225,6 +231,11 @@ def config_cls(self) -> type[DBConfig]: # noqa: PLR0911, PLR0912, C901 return AliyunOpenSearchConfig + if self == DB.MongoDB: + from .mongodb.config import MongoDBConfig + + return MongoDBConfig + if self == DB.Test: from .test.config import TestConfig @@ -302,6 +313,11 @@ def case_config_cls( # noqa: PLR0911 return AliyunOpenSearchIndexConfig + if self == DB.MongoDB: + from .mongodb.config import MongoDBIndexConfig + + return MongoDBIndexConfig + # DB.Pinecone, DB.Chroma, DB.Redis return EmptyDBCaseConfig diff --git a/vectordb_bench/backend/clients/mongodb/config.py b/vectordb_bench/backend/clients/mongodb/config.py new file mode 100644 index 000000000..cc09471a4 --- /dev/null +++ b/vectordb_bench/backend/clients/mongodb/config.py @@ -0,0 +1,44 @@ +from pydantic import BaseModel, SecretStr + +from ..api import DBCaseConfig, DBConfig, IndexType, MetricType + + +class MongoDBConfig(DBConfig, BaseModel): + connection_string: SecretStr = "mongodb+srv://:@.heatl.mongodb.net" + database: str = "vdb_bench" + + def to_dict(self) -> dict: + return { + "connection_string": self.connection_string.get_secret_value(), + "database": self.database, + } + + +class MongoDBIndexConfig(BaseModel, DBCaseConfig): + index: IndexType = IndexType.HNSW # MongoDB uses HNSW for vector search + metric_type: MetricType | None = None + num_candidates: int | None = 1500 # Default numCandidates for vector search + exact_search: bool = False # Whether to use exact (ENN) search + + def parse_metric(self) -> str: + if self.metric_type == MetricType.L2: + return "euclidean" + if self.metric_type == MetricType.IP: + return "dotProduct" + return "cosine" # Default to cosine similarity + + def index_param(self) -> dict: + return { + "type": "vectorSearch", + "fields": [ + { + "type": "vector", + "similarity": self.parse_metric(), + "numDimensions": None, # Will be set in MongoDB class + "path": "vector", # Vector field name + } + ], + } + + def search_param(self) -> dict: + return {"numCandidates": self.num_candidates if not self.exact_search else None, "exact": self.exact_search} diff --git a/vectordb_bench/backend/clients/mongodb/mongodb.py b/vectordb_bench/backend/clients/mongodb/mongodb.py new file mode 100644 index 000000000..dddcc9a4c --- /dev/null +++ b/vectordb_bench/backend/clients/mongodb/mongodb.py @@ -0,0 +1,201 @@ +import logging +import time +from contextlib import contextmanager + +from pymongo import MongoClient +from pymongo.operations import SearchIndexModel + +from ..api import VectorDB +from .config import MongoDBIndexConfig + +log = logging.getLogger(__name__) + + +class MongoDBError(Exception): + """Custom exception class for MongoDB client errors.""" + + +class MongoDB(VectorDB): + def __init__( + self, + dim: int, + db_config: dict, + db_case_config: MongoDBIndexConfig, + collection_name: str = "vdb_bench_collection", + id_field: str = "id", + vector_field: str = "vector", + drop_old: bool = False, + **kwargs, + ): + self.dim = dim + self.db_config = db_config + self.case_config = db_case_config + self.collection_name = collection_name + self.id_field = id_field + self.vector_field = vector_field + self.drop_old = drop_old + + # Update index dimensions + index_params = self.case_config.index_param() + log.info(f"index params: {index_params}") + index_params["fields"][0]["numDimensions"] = dim + self.index_params = index_params + + # Initialize - they'll also be set in init() + uri = self.db_config["connection_string"] + self.client = MongoClient(uri) + self.db = self.client[self.db_config["database"]] + self.collection = self.db[self.collection_name] + if self.drop_old and self.collection_name in self.db.list_collection_names(): + log.info(f"MongoDB client dropping old collection: {self.collection_name}") + self.db.drop_collection(self.collection_name) + self.client = None + self.db = None + self.collection = None + + @contextmanager + def init(self): + """Initialize MongoDB client and cleanup when done""" + try: + uri = self.db_config["connection_string"] + self.client = MongoClient(uri) + self.db = self.client[self.db_config["database"]] + self.collection = self.db[self.collection_name] + + yield + finally: + if self.client is not None: + self.client.close() + self.client = None + self.db = None + self.collection = None + + def _create_index(self) -> None: + """Create vector search index""" + index_name = "vector_index" + index_params = self.index_params + log.info(f"index params {index_params}") + # drop index if already exists + if self.collection.list_indexes(): + all_indexes = self.collection.list_search_indexes() + if any(idx.get("name") == index_name for idx in all_indexes): + log.info(f"Drop index: {index_name}") + try: + self.collection.drop_search_index(index_name) + while True: + indices = list(self.collection.list_search_indexes()) + indices = [idx for idx in indices if idx["name"] == index_name] + log.debug(f"index status {indices}") + if len(indices) == 0: + break + log.info(f"index deleting {indices}") + except Exception: + log.exception("Error dropping index") + try: + # Create vector search index + search_index = SearchIndexModel(definition=index_params, name=index_name, type="vectorSearch") + + self.collection.create_search_index(search_index) + log.info(f"Created vector search index: {index_name}") + self._wait_for_index_ready(index_name) + + # Create regular index on id field for faster lookups + self.collection.create_index(self.id_field) + log.info(f"Created index on {self.id_field} field") + + except Exception: + log.exception("Error creating index") + raise + + def _wait_for_index_ready(self, index_name: str, check_interval: int = 5) -> None: + """Wait for index to be ready""" + while True: + indices = list(self.collection.list_search_indexes()) + log.debug(f"index status {indices}") + if indices and any(idx.get("name") == index_name and idx.get("queryable") for idx in indices): + break + for idx in indices: + if idx.get("name") == index_name and idx.get("status") == "FAILED": + error_msg = f"Index {index_name} failed to build" + raise MongoDBError(error_msg) + + time.sleep(check_interval) + log.info(f"Index {index_name} is ready") + + def need_normalize_cosine(self) -> bool: + return False + + def insert_embeddings( + self, + embeddings: list[list[float]], + metadata: list[int], + **kwargs, + ) -> (int, Exception | None): + """Insert embeddings into MongoDB""" + + # Prepare documents in bulk + documents = [ + { + self.id_field: id_, + self.vector_field: embedding, + } + for id_, embedding in zip(metadata, embeddings, strict=False) + ] + + # Use ordered=False for better insert performance + try: + self.collection.insert_many(documents, ordered=False) + except Exception as e: + return 0, e + return len(documents), None + + def search_embedding( + self, + query: list[float], + k: int = 100, + filters: dict | None = None, + **kwargs, + ) -> list[int]: + """Search for similar vectors""" + search_params = self.case_config.search_param() + + vector_search = {"queryVector": query, "index": "vector_index", "path": self.vector_field, "limit": k} + + # Add exact search parameter if specified + if search_params["exact"]: + vector_search["exact"] = True + else: + # Set numCandidates based on k value and data size + # For 50K dataset, use higher multiplier for better recall + num_candidates = min(10000, max(k * 20, search_params["numCandidates"] or 0)) + vector_search["numCandidates"] = num_candidates + + # Add filter if specified + if filters: + log.info(f"Applying filter: {filters}") + vector_search["filter"] = { + "id": {"gt": filters["id"]}, + } + + pipeline = [ + {"$vectorSearch": vector_search}, + { + "$project": { + "_id": 0, + self.id_field: 1, + "score": {"$meta": "vectorSearchScore"}, # Include similarity score + } + }, + ] + + results = list(self.collection.aggregate(pipeline)) + return [doc[self.id_field] for doc in results] + + def optimize(self, data_size: int | None = None) -> None: + """MongoDB vector search indexes are self-optimizing""" + log.info("optimize for search") + self._create_index() + self._wait_for_index_ready("vector_index") + + def ready_to_load(self) -> None: + """MongoDB is always ready to load""" From ac02b14b3feaa7c667b2b208720b646c93877280 Mon Sep 17 00:00:00 2001 From: zhuwenxing Date: Tue, 14 Jan 2025 11:54:31 +0800 Subject: [PATCH 136/327] add mongodb client in readme Signed-off-by: zhuwenxing --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 737fc6064..1bf95dc39 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,7 @@ All the database client supported | chromadb | `pip install vectordb-bench[chromadb]` | | awsopensearch | `pip install vectordb-bench[opensearch]` | | aliyun_opensearch | `pip install vectordb-bench[aliyun_opensearch]` | +| mongodb | `pip install vectordb-bench[mongodb]` | ### Run From 94f4c6ed500da7f13a75b6152a1fcede74de489d Mon Sep 17 00:00:00 2001 From: "min.tian" Date: Sun, 19 Jan 2025 17:05:50 +0800 Subject: [PATCH 137/327] add some risk warnings for custom dataset - limit the number of test query vectors. Signed-off-by: min.tian --- README.md | 7 +++++++ .../frontend/components/custom/displaypPrams.py | 13 ++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1bf95dc39..56ae88753 100644 --- a/README.md +++ b/README.md @@ -319,6 +319,13 @@ We have strict requirements for the data set format, please follow them. - `Folder Path` - The path to the folder containing all the files. Please ensure that all files in the folder are in the `Parquet` format. - Vectors data files: The file must be named `train.parquet` and should have two columns: `id` as an incrementing `int` and `emb` as an array of `float32`. - Query test vectors: The file must be named `test.parquet` and should have two columns: `id` as an incrementing `int` and `emb` as an array of `float32`. + - We recommend limiting the number of test query vectors, like 1,000. + When conducting concurrent query tests, Vdbbench creates a large number of processes. + To minimize additional communication overhead during testing, + we prepare a complete set of test queries for each process, allowing them to run independently. + However, this means that as the number of concurrent processes increases, + the number of copied query vectors also increases significantly, + which can place substantial pressure on memory resources. - Ground truth file: The file must be named `neighbors.parquet` and should have two columns: `id` corresponding to query vectors and `neighbors_id` as an array of `int`. - `Train File Count` - If the vector file is too large, you can consider splitting it into multiple files. The naming format for the split files should be `train-[index]-of-[file_count].parquet`. For example, `train-01-of-10.parquet` represents the second file (0-indexed) among 10 split files. diff --git a/vectordb_bench/frontend/components/custom/displaypPrams.py b/vectordb_bench/frontend/components/custom/displaypPrams.py index b677e5909..712b57b00 100644 --- a/vectordb_bench/frontend/components/custom/displaypPrams.py +++ b/vectordb_bench/frontend/components/custom/displaypPrams.py @@ -3,7 +3,7 @@ def displayParams(st): """ - `Folder Path` - The path to the folder containing all the files. Please ensure that all files in the folder are in the `Parquet` format. - Vectors data files: The file must be named `train.parquet` and should have two columns: `id` as an incrementing `int` and `emb` as an array of `float32`. - - Query test vectors: The file must be named `test.parquet` and should have two columns: `id` as an incrementing `int` and `emb` as an array of `float32`. + - Query test vectors: The file must be named `test.parquet` and should have two columns: `id` as an incrementing `int` and `emb` as an array of `float32`. - Ground truth file: The file must be named `neighbors.parquet` and should have two columns: `id` corresponding to query vectors and `neighbors_id` as an array of `int`. - `Train File Count` - If the vector file is too large, you can consider splitting it into multiple files. The naming format for the split files should be `train-[index]-of-[file_count].parquet`. For example, `train-01-of-10.parquet` represents the second file (0-indexed) among 10 split files. @@ -11,3 +11,14 @@ def displayParams(st): - `Use Shuffled Data` - If you check this option, the vector data files need to be modified. VectorDBBench will load the data labeled with `shuffle`. For example, use `shuffle_train.parquet` instead of `train.parquet` and `shuffle_train-04-of-10.parquet` instead of `train-04-of-10.parquet`. The `id` column in the shuffled data can be in any order. """ ) + st.caption( + """We recommend limiting the number of test query vectors, like 1,000.""", + help=""" +When conducting concurrent query tests, Vdbbench creates a large number of processes. +To minimize additional communication overhead during testing, +we prepare a complete set of test queries for each process, allowing them to run independently.\n +However, this means that as the number of concurrent processes increases, +the number of copied query vectors also increases significantly, +which can place substantial pressure on memory resources. +""", + ) From 7a00ca0771dd78cd1ff0eada992a9d41d3fac0cd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Jan 2025 06:28:13 +0000 Subject: [PATCH 138/327] Bump grpcio from 1.53.0 to 1.53.2 in /install Bumps [grpcio](https://github.com/grpc/grpc) from 1.53.0 to 1.53.2. - [Release notes](https://github.com/grpc/grpc/releases) - [Changelog](https://github.com/grpc/grpc/blob/master/doc/grpc_release_schedule.md) - [Commits](https://github.com/grpc/grpc/compare/v1.53.0...v1.53.2) --- updated-dependencies: - dependency-name: grpcio dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- install/requirements_py3.11.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/requirements_py3.11.txt b/install/requirements_py3.11.txt index c3a3bbbda..18138382f 100644 --- a/install/requirements_py3.11.txt +++ b/install/requirements_py3.11.txt @@ -1,4 +1,4 @@ -grpcio==1.53.0 +grpcio==1.53.2 grpcio-tools==1.53.0 qdrant-client pinecone-client From 7811d73138ead217bd96bc336d88399a36791f44 Mon Sep 17 00:00:00 2001 From: zhuwenxing Date: Tue, 14 Jan 2025 17:23:35 +0800 Subject: [PATCH 139/327] add mongodb config Signed-off-by: zhuwenxing --- .gitignore | 4 ++- install.py | 3 +- .../backend/clients/mongodb/config.py | 17 +++++++--- .../backend/clients/mongodb/mongodb.py | 9 +++--- .../frontend/config/dbCaseConfigs.py | 32 +++++++++++++++++++ vectordb_bench/log_util.py | 17 ++++++++-- vectordb_bench/models.py | 4 +++ 7 files changed, 73 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index f8602c11a..aa3a72f44 100644 --- a/.gitignore +++ b/.gitignore @@ -8,5 +8,7 @@ __MACOSX .DS_Store build/ venv/ +.venv/ .idea/ -results/ \ No newline at end of file +results/ +logs/ \ No newline at end of file diff --git a/install.py b/install.py index f683a37b2..5807485fd 100644 --- a/install.py +++ b/install.py @@ -1,7 +1,8 @@ -import os import argparse +import os import subprocess + def docker_tag_base(): return 'vdbbench' diff --git a/vectordb_bench/backend/clients/mongodb/config.py b/vectordb_bench/backend/clients/mongodb/config.py index cc09471a4..a2d8ca57a 100644 --- a/vectordb_bench/backend/clients/mongodb/config.py +++ b/vectordb_bench/backend/clients/mongodb/config.py @@ -1,8 +1,16 @@ +from enum import Enum + from pydantic import BaseModel, SecretStr from ..api import DBCaseConfig, DBConfig, IndexType, MetricType +class QuantizationType(Enum): + NONE = "none" + BINARY = "binary" + SCALAR = "scalar" + + class MongoDBConfig(DBConfig, BaseModel): connection_string: SecretStr = "mongodb+srv://:@.heatl.mongodb.net" database: str = "vdb_bench" @@ -16,9 +24,9 @@ def to_dict(self) -> dict: class MongoDBIndexConfig(BaseModel, DBCaseConfig): index: IndexType = IndexType.HNSW # MongoDB uses HNSW for vector search - metric_type: MetricType | None = None - num_candidates: int | None = 1500 # Default numCandidates for vector search - exact_search: bool = False # Whether to use exact (ENN) search + metric_type: MetricType = MetricType.COSINE + num_candidates_ratio: int = 10 # Default numCandidates ratio for vector search + quantization: QuantizationType = QuantizationType.NONE # Quantization type if applicable def parse_metric(self) -> str: if self.metric_type == MetricType.L2: @@ -36,9 +44,10 @@ def index_param(self) -> dict: "similarity": self.parse_metric(), "numDimensions": None, # Will be set in MongoDB class "path": "vector", # Vector field name + "quantization": self.quantization.value, } ], } def search_param(self) -> dict: - return {"numCandidates": self.num_candidates if not self.exact_search else None, "exact": self.exact_search} + return {"num_candidates_ratio": self.num_candidates_ratio} diff --git a/vectordb_bench/backend/clients/mongodb/mongodb.py b/vectordb_bench/backend/clients/mongodb/mongodb.py index dddcc9a4c..0bbfd5d9c 100644 --- a/vectordb_bench/backend/clients/mongodb/mongodb.py +++ b/vectordb_bench/backend/clients/mongodb/mongodb.py @@ -90,7 +90,7 @@ def _create_index(self) -> None: break log.info(f"index deleting {indices}") except Exception: - log.exception("Error dropping index") + log.exception(f"Error dropping index {index_name}") try: # Create vector search index search_index = SearchIndexModel(definition=index_params, name=index_name, type="vectorSearch") @@ -104,7 +104,7 @@ def _create_index(self) -> None: log.info(f"Created index on {self.id_field} field") except Exception: - log.exception("Error creating index") + log.exception(f"Error creating index {index_name}") raise def _wait_for_index_ready(self, index_name: str, check_interval: int = 5) -> None: @@ -167,16 +167,15 @@ def search_embedding( else: # Set numCandidates based on k value and data size # For 50K dataset, use higher multiplier for better recall - num_candidates = min(10000, max(k * 20, search_params["numCandidates"] or 0)) + num_candidates = min(10000, k * search_params["num_candidates_ratio"]) vector_search["numCandidates"] = num_candidates # Add filter if specified if filters: log.info(f"Applying filter: {filters}") vector_search["filter"] = { - "id": {"gt": filters["id"]}, + "id": {"gte": filters["id"]}, } - pipeline = [ {"$vectorSearch": vector_search}, { diff --git a/vectordb_bench/frontend/config/dbCaseConfigs.py b/vectordb_bench/frontend/config/dbCaseConfigs.py index e004f2ba7..13858f879 100644 --- a/vectordb_bench/frontend/config/dbCaseConfigs.py +++ b/vectordb_bench/frontend/config/dbCaseConfigs.py @@ -1041,6 +1041,26 @@ class CaseConfigInput(BaseModel): ) +CaseConfigParamInput_MongoDBQuantizationType = CaseConfigInput( + label=CaseConfigParamType.mongodb_quantization_type, + inputType=InputType.Option, + inputConfig={ + "options": ["none", "scalar", "binary"], + }, +) + + +CaseConfigParamInput_MongoDBNumCandidatesRatio = CaseConfigInput( + label=CaseConfigParamType.mongodb_num_candidates_ratio, + inputType=InputType.Number, + inputConfig={ + "min": 10, + "max": 20, + "value": 10, + }, +) + + MilvusLoadConfig = [ CaseConfigParamInput_IndexType, CaseConfigParamInput_M, @@ -1224,6 +1244,14 @@ class CaseConfigInput(BaseModel): CaseConfigParamInput_NumCandidates_AliES, ] +MongoDBLoadingConfig = [ + CaseConfigParamInput_MongoDBQuantizationType, +] +MongoDBPerformanceConfig = [ + CaseConfigParamInput_MongoDBQuantizationType, + CaseConfigParamInput_MongoDBNumCandidatesRatio, +] + CASE_CONFIG_MAP = { DB.Milvus: { CaseLabel.Load: MilvusLoadConfig, @@ -1272,4 +1300,8 @@ class CaseConfigInput(BaseModel): CaseLabel.Load: AliyunOpensearchLoadingConfig, CaseLabel.Performance: AliyunOpenSearchPerformanceConfig, }, + DB.MongoDB: { + CaseLabel.Load: MongoDBLoadingConfig, + CaseLabel.Performance: MongoDBPerformanceConfig, + }, } diff --git a/vectordb_bench/log_util.py b/vectordb_bench/log_util.py index d75688137..6ca6ccabf 100644 --- a/vectordb_bench/log_util.py +++ b/vectordb_bench/log_util.py @@ -1,8 +1,13 @@ import logging from logging import config +from pathlib import Path def init(log_level: str): + # Create logs directory if it doesn't exist + log_dir = Path("logs") + log_dir.mkdir(exist_ok=True) + log_config = { "version": 1, "disable_existing_loggers": False, @@ -24,15 +29,23 @@ def init(log_level: str): "class": "logging.StreamHandler", "formatter": "default", }, + "file": { + "class": "logging.handlers.RotatingFileHandler", + "formatter": "default", + "filename": "logs/vectordb_bench.log", + "maxBytes": 10485760, # 10MB + "backupCount": 5, + "encoding": "utf8", + }, }, "loggers": { "vectordb_bench": { - "handlers": ["console"], + "handlers": ["console", "file"], "level": log_level, "propagate": False, }, "no_color": { - "handlers": ["no_color_console"], + "handlers": ["no_color_console", "file"], "level": log_level, "propagate": False, }, diff --git a/vectordb_bench/models.py b/vectordb_bench/models.py index 49bb04ae0..bf71ebb89 100644 --- a/vectordb_bench/models.py +++ b/vectordb_bench/models.py @@ -88,6 +88,10 @@ class CaseConfigParamType(Enum): numSearchThreads = "num_search_threads" maxNumPrefetchDatasets = "max_num_prefetch_datasets" + # mongodb params + mongodb_quantization_type = "quantization" + mongodb_num_candidates_ratio = "num_candidates_ratio" + class CustomizedCase(BaseModel): pass From 23597474675871cd27d3e9645447a5906051c1e7 Mon Sep 17 00:00:00 2001 From: Xavierantony1982 Date: Thu, 30 Jan 2025 19:54:56 -0800 Subject: [PATCH 140/327] Opensearch interal configuration parameters (#463) * Added the configuration parameters to create Opensearch dynamically with right replicas, shards and other opensearch related configurations. Added the feature to create OS index with 0 replica and once the data is loaded update the replicas according to the parameter. * Updated the readme for config parameters --------- Co-authored-by: xavrathi --- README.md | 41 +++++++++ .../clients/aws_opensearch/aws_opensearch.py | 57 +++++++++++- .../backend/clients/aws_opensearch/cli.py | 86 ++++++++++++++++++- .../backend/clients/aws_opensearch/config.py | 10 +++ 4 files changed, 189 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 56ae88753..e8be99cec 100644 --- a/README.md +++ b/README.md @@ -153,6 +153,47 @@ Options: with-gt] --help Show this message and exit. ``` + +### Run awsopensearch from command line + +```shell +vectordbbench awsopensearch --db-label awsopensearch \ +--m 16 --ef-construction 256 \ +--host search-vector-db-prod-h4f6m4of6x7yp2rz7gdmots7w4.us-west-2.es.amazonaws.com --port 443 \ +--user vector --password '' \ +--case-type Performance1536D5M --num-insert-workers 10 \ +--skip-load --num-concurrency 75 +``` + +To list the options for awsopensearch, execute `vectordbbench awsopensearch --help` + +```text +$ vectordbbench awsopensearch --help +Usage: vectordbbench awsopensearch [OPTIONS] + +Options: + # Sharding and Replication + --number-of-shards INTEGER Number of primary shards for the index + --number-of-replicas INTEGER Number of replica copies for each primary + shard + # Indexing Performance + --index-thread-qty INTEGER Thread count for native engine indexing + --index-thread-qty-during-force-merge INTEGER + Thread count during force merge operations + --number-of-indexing-clients INTEGER + Number of concurrent indexing clients + # Index Management + --number-of-segments INTEGER Target number of segments after merging + --refresh-interval TEXT How often to make new data available for + search + --force-merge-enabled BOOLEAN Whether to perform force merge operation + --flush-threshold-size TEXT Size threshold for flushing the transaction + log + # Memory Management + --cb-threshold TEXT k-NN Memory circuit breaker threshold + + --help Show this message and exit.``` + #### Using a configuration file. The vectordbbench command can optionally read some or all the options from a yaml formatted configuration file. diff --git a/vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py b/vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py index 234014f19..adb766300 100644 --- a/vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py +++ b/vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py @@ -12,6 +12,7 @@ WAITING_FOR_REFRESH_SEC = 30 WAITING_FOR_FORCE_MERGE_SEC = 30 +SECONDS_WAITING_FOR_REPLICAS_TO_BE_ENABLED_SEC = 30 class AWSOpenSearch(VectorDB): @@ -52,10 +53,27 @@ def case_config_cls(cls, index_type: IndexType | None = None) -> AWSOpenSearchIn return AWSOpenSearchIndexConfig def _create_index(self, client: OpenSearch): + cluster_settings_body = { + "persistent": { + "knn.algo_param.index_thread_qty": self.case_config.index_thread_qty, + "knn.memory.circuit_breaker.limit": self.case_config.cb_threshold, + } + } + client.cluster.put_settings(cluster_settings_body) settings = { "index": { "knn": True, + "number_of_shards": self.case_config.number_of_shards, + "number_of_replicas": 0, + "translog.flush_threshold_size": self.case_config.flush_threshold_size, + # Setting trans log threshold to 5GB + **( + {"knn.algo_param.ef_search": self.case_config.ef_search} + if self.case_config.engine == AWSOS_Engine.nmslib + else {} + ), }, + "refresh_interval": self.case_config.refresh_interval, } mappings = { "properties": { @@ -145,9 +163,9 @@ def search_embedding( docvalue_fields=[self.id_col_name], stored_fields="_none_", ) - log.info(f"Search took: {resp['took']}") - log.info(f"Search shards: {resp['_shards']}") - log.info(f"Search hits total: {resp['hits']['total']}") + log.debug(f"Search took: {resp['took']}") + log.debug(f"Search shards: {resp['_shards']}") + log.debug(f"Search hits total: {resp['hits']['total']}") return [int(h["fields"][self.id_col_name][0]) for h in resp["hits"]["hits"]] except Exception as e: log.warning(f"Failed to search: {self.index_name} error: {e!s}") @@ -157,12 +175,37 @@ def optimize(self, data_size: int | None = None): """optimize will be called between insertion and search in performance cases.""" # Call refresh first to ensure that all segments are created self._refresh_index() - self._do_force_merge() + if self.case_config.force_merge_enabled: + self._do_force_merge() + self._refresh_index() + self._update_replicas() # Call refresh again to ensure that the index is ready after force merge. self._refresh_index() # ensure that all graphs are loaded in memory and ready for search self._load_graphs_to_memory() + def _update_replicas(self): + index_settings = self.client.indices.get_settings(index=self.index_name) + current_number_of_replicas = int(index_settings[self.index_name]["settings"]["index"]["number_of_replicas"]) + log.info( + f"Current Number of replicas are {current_number_of_replicas}" + f" and changing the replicas to {self.case_config.number_of_replicas}" + ) + settings_body = {"index": {"number_of_replicas": self.case_config.number_of_replicas}} + self.client.indices.put_settings(index=self.index_name, body=settings_body) + self._wait_till_green() + + def _wait_till_green(self): + log.info("Wait for index to become green..") + while True: + res = self.client.cat.indices(index=self.index_name, h="health", format="json") + health = res[0]["health"] + if health != "green": + break + log.info(f"The index {self.index_name} has health : {health} and is not green. Retrying") + time.sleep(SECONDS_WAITING_FOR_REPLICAS_TO_BE_ENABLED_SEC) + log.info(f"Index {self.index_name} is green..") + def _refresh_index(self): log.debug(f"Starting refresh for index {self.index_name}") while True: @@ -179,6 +222,12 @@ def _refresh_index(self): log.debug(f"Completed refresh for index {self.index_name}") def _do_force_merge(self): + log.info(f"Updating the Index thread qty to {self.case_config.index_thread_qty_during_force_merge}.") + + cluster_settings_body = { + "persistent": {"knn.algo_param.index_thread_qty": self.case_config.index_thread_qty_during_force_merge} + } + self.client.cluster.put_settings(cluster_settings_body) log.debug(f"Starting force merge for index {self.index_name}") force_merge_endpoint = f"/{self.index_name}/_forcemerge?max_num_segments=1&wait_for_completion=false" force_merge_task_id = self.client.transport.perform_request("POST", force_merge_endpoint)["task"] diff --git a/vectordb_bench/backend/clients/aws_opensearch/cli.py b/vectordb_bench/backend/clients/aws_opensearch/cli.py index bb0c2450d..fa457154d 100644 --- a/vectordb_bench/backend/clients/aws_opensearch/cli.py +++ b/vectordb_bench/backend/clients/aws_opensearch/cli.py @@ -18,6 +18,79 @@ class AWSOpenSearchTypedDict(TypedDict): port: Annotated[int, click.option("--port", type=int, default=443, help="Db Port")] user: Annotated[str, click.option("--user", type=str, default="admin", help="Db User")] password: Annotated[str, click.option("--password", type=str, help="Db password")] + number_of_shards: Annotated[ + int, + click.option("--number-of-shards", type=int, help="Number of primary shards for the index", default=1), + ] + number_of_replicas: Annotated[ + int, + click.option( + "--number-of-replicas", type=int, help="Number of replica copies for each primary shard", default=1 + ), + ] + index_thread_qty: Annotated[ + int, + click.option( + "--index-thread-qty", + type=int, + help="Thread count for native engine indexing", + default=4, + ), + ] + + index_thread_qty_during_force_merge: Annotated[ + int, + click.option( + "--index-thread-qty-during-force-merge", + type=int, + help="Thread count during force merge operations", + default=4, + ), + ] + + number_of_indexing_clients: Annotated[ + int, + click.option( + "--number-of-indexing-clients", + type=int, + help="Number of concurrent indexing clients", + default=1, + ), + ] + + number_of_segments: Annotated[ + int, + click.option("--number-of-segments", type=int, help="Target number of segments after merging", default=1), + ] + + refresh_interval: Annotated[ + int, + click.option( + "--refresh-interval", type=str, help="How often to make new data available for search", default="60s" + ), + ] + + force_merge_enabled: Annotated[ + int, + click.option("--force-merge-enabled", type=bool, help="Whether to perform force merge operation", default=True), + ] + + flush_threshold_size: Annotated[ + int, + click.option( + "--flush-threshold-size", type=str, help="Size threshold for flushing the transaction log", default="5120mb" + ), + ] + + cb_threshold: Annotated[ + int, + click.option( + "--cb-threshold", + type=str, + help="k-NN Memory circuit breaker threshold", + default="50%", + ), + ] class AWSOpenSearchHNSWTypedDict(CommonTypedDict, AWSOpenSearchTypedDict, HNSWFlavor2): ... @@ -36,6 +109,17 @@ def AWSOpenSearch(**parameters: Unpack[AWSOpenSearchHNSWTypedDict]): user=parameters["user"], password=SecretStr(parameters["password"]), ), - db_case_config=AWSOpenSearchIndexConfig(), + db_case_config=AWSOpenSearchIndexConfig( + number_of_shards=parameters["number_of_shards"], + number_of_replicas=parameters["number_of_replicas"], + index_thread_qty=parameters["index_thread_qty"], + number_of_segments=parameters["number_of_segments"], + refresh_interval=parameters["refresh_interval"], + force_merge_enabled=parameters["force_merge_enabled"], + flush_threshold_size=parameters["flush_threshold_size"], + number_of_indexing_clients=parameters["number_of_indexing_clients"], + index_thread_qty_during_force_merge=parameters["index_thread_qty_during_force_merge"], + cb_threshold=parameters["cb_threshold"], + ), **parameters, ) diff --git a/vectordb_bench/backend/clients/aws_opensearch/config.py b/vectordb_bench/backend/clients/aws_opensearch/config.py index e9ccc7277..dd51b266d 100644 --- a/vectordb_bench/backend/clients/aws_opensearch/config.py +++ b/vectordb_bench/backend/clients/aws_opensearch/config.py @@ -39,6 +39,16 @@ class AWSOpenSearchIndexConfig(BaseModel, DBCaseConfig): efConstruction: int = 256 efSearch: int = 256 M: int = 16 + index_thread_qty: int | None = 4 + number_of_shards: int | None = 1 + number_of_replicas: int | None = 0 + number_of_segments: int | None = 1 + refresh_interval: str | None = "60s" + force_merge_enabled: bool | None = True + flush_threshold_size: str | None = "5120mb" + number_of_indexing_clients: int | None = 1 + index_thread_qty_during_force_merge: int + cb_threshold: str | None = "50%" def parse_metric(self) -> str: if self.metric_type == MetricType.IP: From ec439e4c837b3e5f268f1a97995f670af8c78460 Mon Sep 17 00:00:00 2001 From: "siqi.an" Date: Mon, 10 Feb 2025 17:30:35 +0800 Subject: [PATCH 141/327] ui control num of concurrencies Signed-off-by: siqi.an --- .../components/run_test/submitTask.py | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/vectordb_bench/frontend/components/run_test/submitTask.py b/vectordb_bench/frontend/components/run_test/submitTask.py index 426095397..5827efeb4 100644 --- a/vectordb_bench/frontend/components/run_test/submitTask.py +++ b/vectordb_bench/frontend/components/run_test/submitTask.py @@ -1,6 +1,8 @@ from datetime import datetime +from vectordb_bench import config from vectordb_bench.frontend.config import styles from vectordb_bench.interface import benchmark_runner +from vectordb_bench.models import TaskConfig def submitTask(st, tasks, isAllValid): @@ -47,16 +49,31 @@ def advancedSettings(st): k = container[0].number_input("k", min_value=1, value=100, label_visibility="collapsed") container[1].caption("K value for number of nearest neighbors to search") - return index_already_exists, use_aliyun, k + container = st.columns([1, 2]) + defaultconcurrentInput = ",".join(map(str, config.NUM_CONCURRENCY)) + concurrentInput = container[0].text_input( + "Concurrent Input", value=defaultconcurrentInput, label_visibility="collapsed" + ) + container[1].caption("num of concurrencies for search tests to get max-qps") + return index_already_exists, use_aliyun, k, concurrentInput -def controlPanel(st, tasks, taskLabel, isAllValid): - index_already_exists, use_aliyun, k = advancedSettings(st) +def controlPanel(st, tasks: list[TaskConfig], taskLabel, isAllValid): + index_already_exists, use_aliyun, k, concurrentInput = advancedSettings(st) def runHandler(): benchmark_runner.set_drop_old(not index_already_exists) + + try: + concurrentInput_list = [int(item.strip()) for item in concurrentInput.split(",")] + except ValueError: + st.write("please input correct number") + return None + for task in tasks: task.case_config.k = k + task.case_config.concurrency_search_config.num_concurrency = concurrentInput_list + benchmark_runner.set_download_address(use_aliyun) benchmark_runner.run(tasks, taskLabel) From e4d987c2b25c4981c1d7f60714a511adfe26df44 Mon Sep 17 00:00:00 2001 From: Xiaofan <83447078+xiaofan-luan@users.noreply.github.com> Date: Wed, 12 Feb 2025 12:36:48 +0800 Subject: [PATCH 142/327] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index e8be99cec..86c6ab529 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,8 @@ Closely mimicking real-world production environments, we've set up diverse testi Prepare to delve into the world of VectorDBBench, and let it guide you in uncovering your perfect vector database match. +VectorDBBench is sponsered by Zilliz，the leading opensource vectorDB company behind Milvus. Choose smarter with VectorDBBench- start your free test on [zilliz cloud](https://zilliz.com/) today! + **Leaderboard:** https://zilliz.com/benchmark ## Quick Start ### Prerequirement From 9c3446f425246d4b4f61195127273bcd5f4bd17a Mon Sep 17 00:00:00 2001 From: "min.tian" Date: Thu, 13 Feb 2025 10:26:39 +0800 Subject: [PATCH 143/327] environs version should <14.1.0 Signed-off-by: min.tian --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index aafe70750..8bc83f29b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,7 @@ dependencies = [ "psutil", "polars", "plotly", - "environs", + "environs<14.1.0", "pydantic Date: Mon, 24 Feb 2025 14:14:59 +0530 Subject: [PATCH 144/327] Support GPU_BRUTE_FORCE index for Milvus (#476) Signed-off-by: Rachit Chaudhary Co-authored-by: Signed-off-by: Rachit Chaudhary - r0c0axe --- vectordb_bench/backend/clients/api.py | 1 + vectordb_bench/backend/clients/milvus/cli.py | 20 +++++++++++ .../backend/clients/milvus/config.py | 33 +++++++++++++++++++ .../frontend/config/dbCaseConfigs.py | 5 +++ 4 files changed, 59 insertions(+) diff --git a/vectordb_bench/backend/clients/api.py b/vectordb_bench/backend/clients/api.py index a86849e96..e498ab077 100644 --- a/vectordb_bench/backend/clients/api.py +++ b/vectordb_bench/backend/clients/api.py @@ -25,6 +25,7 @@ class IndexType(str, Enum): ES_HNSW = "hnsw" ES_IVFFlat = "ivfflat" GPU_IVF_FLAT = "GPU_IVF_FLAT" + GPU_BRUTE_FORCE = "GPU_BRUTE_FORCE" GPU_IVF_PQ = "GPU_IVF_PQ" GPU_CAGRA = "GPU_CAGRA" SCANN = "scann" diff --git a/vectordb_bench/backend/clients/milvus/cli.py b/vectordb_bench/backend/clients/milvus/cli.py index 51ea82eff..303eec5f9 100644 --- a/vectordb_bench/backend/clients/milvus/cli.py +++ b/vectordb_bench/backend/clients/milvus/cli.py @@ -194,6 +194,26 @@ def MilvusGPUIVFFlat(**parameters: Unpack[MilvusGPUIVFTypedDict]): **parameters, ) +@cli.command() +@click_parameter_decorators_from_typed_dict(MilvusGPUBruteForceTypedDict) +def MilvusGPUBruteForce(**parameters: Unpack[MilvusGPUBruteForceTypedDict]): + from .config import GPUBruteForceConfig, MilvusConfig + + run( + db=DBTYPE, + db_config=MilvusConfig( + db_label=parameters["db_label"], + uri=SecretStr(parameters["uri"]), + user=parameters["user_name"], + password=SecretStr(parameters["password"]), + ), + db_case_config=GPUBruteForceConfig( + metric_type=parameters["metric_type"], + limit=parameters["limit"], # top-k for search + ), + **parameters, + ) + class MilvusGPUIVFPQTypedDict( CommonTypedDict, diff --git a/vectordb_bench/backend/clients/milvus/config.py b/vectordb_bench/backend/clients/milvus/config.py index 7d0df803a..1ff3bea5f 100644 --- a/vectordb_bench/backend/clients/milvus/config.py +++ b/vectordb_bench/backend/clients/milvus/config.py @@ -40,6 +40,7 @@ def is_gpu_index(self) -> bool: IndexType.GPU_CAGRA, IndexType.GPU_IVF_FLAT, IndexType.GPU_IVF_PQ, + IndexType.GPU_BRUTE_FORCE, ] def parse_metric(self) -> str: @@ -184,6 +185,37 @@ def search_param(self) -> dict: } +class GPUBruteForceConfig(MilvusIndexConfig, DBCaseConfig): + limit: int = 10 # Default top-k for search + metric_type: str # Metric type (e.g., 'L2', 'IP', etc.) + index: IndexType = IndexType.GPU_BRUTE_FORCE # Index type set to GPU_BRUTE_FORCE + + def index_param(self) -> dict: + """ + Returns the parameters for creating the GPU_BRUTE_FORCE index. + No additional parameters required for index building. + """ + return { + "metric_type": self.parse_metric(), # Metric type for distance calculation (L2, IP, etc.) + "index_type": self.index.value, # GPU_BRUTE_FORCE index type + "params": {}, # No additional parameters for GPU_BRUTE_FORCE + } + + def search_param(self) -> dict: + """ + Returns the parameters for performing a search on the GPU_BRUTE_FORCE index. + Only metric_type and top-k (limit) are needed for search. + """ + return { + "metric_type": self.parse_metric(), # Metric type for search + "params": { + "nprobe": 1, # For GPU_BRUTE_FORCE, set nprobe to 1 (brute force search) + "limit": self.limit, # Top-k for search + }, + } + + + class GPUIVFPQConfig(MilvusIndexConfig, DBCaseConfig): nlist: int = 1024 m: int = 0 @@ -261,4 +293,5 @@ def search_param(self) -> dict: IndexType.GPU_IVF_FLAT: GPUIVFFlatConfig, IndexType.GPU_IVF_PQ: GPUIVFPQConfig, IndexType.GPU_CAGRA: GPUCAGRAConfig, + IndexType.GPU_BRUTE_FORCE: GPUBruteForceConfig, } diff --git a/vectordb_bench/frontend/config/dbCaseConfigs.py b/vectordb_bench/frontend/config/dbCaseConfigs.py index 13858f879..a1c67e89a 100644 --- a/vectordb_bench/frontend/config/dbCaseConfigs.py +++ b/vectordb_bench/frontend/config/dbCaseConfigs.py @@ -173,6 +173,7 @@ class CaseConfigInput(BaseModel): IndexType.GPU_IVF_FLAT.value, IndexType.GPU_IVF_PQ.value, IndexType.GPU_CAGRA.value, + IndexType.GPU_BRUTE_FORCE.value, ], }, ) @@ -562,6 +563,7 @@ class CaseConfigInput(BaseModel): IndexType.IVFSQ8.value, IndexType.GPU_IVF_FLAT.value, IndexType.GPU_IVF_PQ.value, + IndexType.GPU_BRUTE_FORCE.value, ], ) @@ -579,6 +581,7 @@ class CaseConfigInput(BaseModel): IndexType.IVFSQ8.value, IndexType.GPU_IVF_FLAT.value, IndexType.GPU_IVF_PQ.value, + IndexType.GPU_BRUTE_FORCE.value, ], ) @@ -703,6 +706,7 @@ class CaseConfigInput(BaseModel): IndexType.GPU_CAGRA.value, IndexType.GPU_IVF_PQ.value, IndexType.GPU_IVF_FLAT.value, + IndexType.GPU_BRUTE_FORCE.value, ], ) @@ -720,6 +724,7 @@ class CaseConfigInput(BaseModel): IndexType.GPU_CAGRA.value, IndexType.GPU_IVF_PQ.value, IndexType.GPU_IVF_FLAT.value, + IndexType.GPU_BRUTE_FORCE.value, ], ) From 040041b594faac74bf150d55d48c77f9ddd4cf97 Mon Sep 17 00:00:00 2001 From: Luca Giacchino Date: Tue, 5 Nov 2024 15:44:32 -0800 Subject: [PATCH 145/327] Add table quantization type --- README.md | 6 +- .../backend/clients/pgvector/cli.py | 14 +++- .../backend/clients/pgvector/config.py | 27 +++++-- .../backend/clients/pgvector/pgvector.py | 81 ++++++++++++++----- .../frontend/config/dbCaseConfigs.py | 15 ++++ vectordb_bench/models.py | 1 + 6 files changed, 118 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 86c6ab529..80db92ec5 100644 --- a/README.md +++ b/README.md @@ -131,7 +131,11 @@ Options: --ef-construction INTEGER hnsw ef-construction --ef-search INTEGER hnsw ef-search --quantization-type [none|bit|halfvec] - quantization type for vectors + quantization type for vectors (in index) + --table-quantization-type [none|bit|halfvec] + quantization type for vectors (in table). If + equal to bit, the parameter + quantization_type will be set to bit too. --custom-case-name TEXT Custom case name i.e. PerformanceCase1536D50K --custom-case-description TEXT Custom name description --custom-case-load-timeout INTEGER diff --git a/vectordb_bench/backend/clients/pgvector/cli.py b/vectordb_bench/backend/clients/pgvector/cli.py index 55a462055..1780af991 100644 --- a/vectordb_bench/backend/clients/pgvector/cli.py +++ b/vectordb_bench/backend/clients/pgvector/cli.py @@ -82,7 +82,17 @@ class PgVectorTypedDict(CommonTypedDict): click.option( "--quantization-type", type=click.Choice(["none", "bit", "halfvec"]), - help="quantization type for vectors", + help="quantization type for vectors (in index)", + required=False, + ), + ] + table_quantization_type: Annotated[ + str | None, + click.option( + "--table-quantization-type", + type=click.Choice(["none", "bit", "halfvec"]), + help="quantization type for vectors (in table). " + "If equal to bit, the parameter quantization_type will be set to bit too.", required=False, ), ] @@ -146,6 +156,7 @@ def PgVectorIVFFlat( lists=parameters["lists"], probes=parameters["probes"], quantization_type=parameters["quantization_type"], + table_quantization_type=parameters["table_quantization_type"], reranking=parameters["reranking"], reranking_metric=parameters["reranking_metric"], quantized_fetch_limit=parameters["quantized_fetch_limit"], @@ -182,6 +193,7 @@ def PgVectorHNSW( maintenance_work_mem=parameters["maintenance_work_mem"], max_parallel_workers=parameters["max_parallel_workers"], quantization_type=parameters["quantization_type"], + table_quantization_type=parameters["table_quantization_type"], reranking=parameters["reranking"], reranking_metric=parameters["reranking_metric"], quantized_fetch_limit=parameters["quantized_fetch_limit"], diff --git a/vectordb_bench/backend/clients/pgvector/config.py b/vectordb_bench/backend/clients/pgvector/config.py index c386d75ef..abfddc0cf 100644 --- a/vectordb_bench/backend/clients/pgvector/config.py +++ b/vectordb_bench/backend/clients/pgvector/config.py @@ -80,7 +80,12 @@ def parse_metric(self) -> str: if d.get(self.quantization_type) is None: return d.get("_fallback").get(self.metric_type) - return d.get(self.quantization_type).get(self.metric_type) + metric = d.get(self.quantization_type).get(self.metric_type) + # If using binary quantization for the index, use a bit metric + # no matter what metric was selected for vector or halfvec data + if self.quantization_type == "bit" and metric is None: + return "bit_hamming_ops" + return metric def parse_metric_fun_op(self) -> LiteralString: if self.quantization_type == "bit": @@ -168,14 +173,19 @@ class PgVectorIVFFlatConfig(PgVectorIndexConfig): maintenance_work_mem: str | None = None max_parallel_workers: int | None = None quantization_type: str | None = None + table_quantization_type: str | None reranking: bool | None = None quantized_fetch_limit: int | None = None reranking_metric: str | None = None def index_param(self) -> PgVectorIndexParam: index_parameters = {"lists": self.lists} - if self.quantization_type == "none": - self.quantization_type = None + if self.quantization_type == "none" or self.quantization_type is None: + self.quantization_type = "vector" + if self.table_quantization_type == "none" or self.table_quantization_type is None: + self.table_quantization_type = "vector" + if self.table_quantization_type == "bit": + self.quantization_type = "bit" return { "metric": self.parse_metric(), "index_type": self.index.value, @@ -183,6 +193,7 @@ def index_param(self) -> PgVectorIndexParam: "maintenance_work_mem": self.maintenance_work_mem, "max_parallel_workers": self.max_parallel_workers, "quantization_type": self.quantization_type, + "table_quantization_type": self.table_quantization_type, } def search_param(self) -> PgVectorSearchParam: @@ -212,14 +223,19 @@ class PgVectorHNSWConfig(PgVectorIndexConfig): maintenance_work_mem: str | None = None max_parallel_workers: int | None = None quantization_type: str | None = None + table_quantization_type: str | None reranking: bool | None = None quantized_fetch_limit: int | None = None reranking_metric: str | None = None def index_param(self) -> PgVectorIndexParam: index_parameters = {"m": self.m, "ef_construction": self.ef_construction} - if self.quantization_type == "none": - self.quantization_type = None + if self.quantization_type == "none" or self.quantization_type is None: + self.quantization_type = "vector" + if self.table_quantization_type == "none" or self.table_quantization_type is None: + self.table_quantization_type = "vector" + if self.table_quantization_type == "bit": + self.quantization_type = "bit" return { "metric": self.parse_metric(), "index_type": self.index.value, @@ -227,6 +243,7 @@ def index_param(self) -> PgVectorIndexParam: "maintenance_work_mem": self.maintenance_work_mem, "max_parallel_workers": self.max_parallel_workers, "quantization_type": self.quantization_type, + "table_quantization_type": self.table_quantization_type, } def search_param(self) -> PgVectorSearchParam: diff --git a/vectordb_bench/backend/clients/pgvector/pgvector.py b/vectordb_bench/backend/clients/pgvector/pgvector.py index 4164461fb..7d06a2ba5 100644 --- a/vectordb_bench/backend/clients/pgvector/pgvector.py +++ b/vectordb_bench/backend/clients/pgvector/pgvector.py @@ -94,7 +94,7 @@ def _generate_search_query(self, filtered: bool = False) -> sql.Composed: reranking = self.case_config.search_param()["reranking"] column_name = ( sql.SQL("binary_quantize({0})").format(sql.Identifier("embedding")) - if index_param["quantization_type"] == "bit" + if index_param["quantization_type"] == "bit" and index_param["table_quantization_type"] != "bit" else sql.SQL("embedding") ) search_vector = ( @@ -104,7 +104,8 @@ def _generate_search_query(self, filtered: bool = False) -> sql.Composed: ) # The following sections assume that the quantization_type value matches the quantization function name - if index_param["quantization_type"] is not None: + if index_param["quantization_type"] != index_param["table_quantization_type"]: + # Reranking makes sense only if table quantization is not "bit" if index_param["quantization_type"] == "bit" and reranking: # Embeddings needs to be passed to binary_quantize function if quantization_type is bit search_query = sql.Composed( @@ -113,7 +114,7 @@ def _generate_search_query(self, filtered: bool = False) -> sql.Composed: """ SELECT i.id FROM ( - SELECT id, embedding {reranking_metric_fun_op} %s::vector AS distance + SELECT id, embedding {reranking_metric_fun_op} %s::{table_quantization_type} AS distance FROM public.{table_name} {where_clause} ORDER BY {column_name}::{quantization_type}({dim}) """, @@ -123,6 +124,8 @@ def _generate_search_query(self, filtered: bool = False) -> sql.Composed: reranking_metric_fun_op=sql.SQL( self.case_config.search_param()["reranking_metric_fun_op"], ), + search_vector=search_vector, + table_quantization_type=sql.SQL(index_param["table_quantization_type"]), quantization_type=sql.SQL(index_param["quantization_type"]), dim=sql.Literal(self.dim), where_clause=sql.SQL("WHERE id >= %s") if filtered else sql.SQL(""), @@ -130,7 +133,7 @@ def _generate_search_query(self, filtered: bool = False) -> sql.Composed: sql.SQL(self.case_config.search_param()["metric_fun_op"]), sql.SQL( """ - {search_vector} + {search_vector}::{quantization_type}({dim}) LIMIT {quantized_fetch_limit} ) i ORDER BY i.distance @@ -138,6 +141,8 @@ def _generate_search_query(self, filtered: bool = False) -> sql.Composed: """, ).format( search_vector=search_vector, + quantization_type=sql.SQL(index_param["quantization_type"]), + dim=sql.Literal(self.dim), quantized_fetch_limit=sql.Literal( self.case_config.search_param()["quantized_fetch_limit"], ), @@ -160,10 +165,12 @@ def _generate_search_query(self, filtered: bool = False) -> sql.Composed: where_clause=sql.SQL("WHERE id >= %s") if filtered else sql.SQL(""), ), sql.SQL(self.case_config.search_param()["metric_fun_op"]), - sql.SQL(" {search_vector} LIMIT %s::int").format( + sql.SQL(" {search_vector}::{quantization_type}({dim}) LIMIT %s::int").format( search_vector=search_vector, + quantization_type=sql.SQL(index_param["quantization_type"]), + dim=sql.Literal(self.dim), ), - ], + ] ) else: search_query = sql.Composed( @@ -175,8 +182,12 @@ def _generate_search_query(self, filtered: bool = False) -> sql.Composed: where_clause=sql.SQL("WHERE id >= %s") if filtered else sql.SQL(""), ), sql.SQL(self.case_config.search_param()["metric_fun_op"]), - sql.SQL(" %s::vector LIMIT %s::int"), - ], + sql.SQL(" {search_vector}::{quantization_type}({dim}) LIMIT %s::int").format( + search_vector=search_vector, + quantization_type=sql.SQL(index_param["quantization_type"]), + dim=sql.Literal(self.dim), + ), + ] ) return search_query @@ -323,7 +334,7 @@ def _create_index(self): ) with_clause = sql.SQL("WITH ({});").format(sql.SQL(", ").join(options)) if any(options) else sql.Composed(()) - if index_param["quantization_type"] is not None: + if index_param["quantization_type"] != index_param["table_quantization_type"]: index_create_sql = sql.SQL( """ CREATE INDEX IF NOT EXISTS {index_name} ON public.{table_name} @@ -365,14 +376,23 @@ def _create_table(self, dim: int): assert self.conn is not None, "Connection is not initialized" assert self.cursor is not None, "Cursor is not initialized" + index_param = self.case_config.index_param() + try: log.info(f"{self.name} client create table : {self.table_name}") # create table self.cursor.execute( sql.SQL( - "CREATE TABLE IF NOT EXISTS public.{table_name} (id BIGINT PRIMARY KEY, embedding vector({dim}));", - ).format(table_name=sql.Identifier(self.table_name), dim=dim), + """ + CREATE TABLE IF NOT EXISTS public.{table_name} + (id BIGINT PRIMARY KEY, embedding {table_quantization_type}({dim})); + """ + ).format( + table_name=sql.Identifier(self.table_name), + table_quantization_type=sql.SQL(index_param["table_quantization_type"]), + dim=dim, + ) ) self.cursor.execute( sql.SQL( @@ -393,18 +413,41 @@ def insert_embeddings( assert self.conn is not None, "Connection is not initialized" assert self.cursor is not None, "Cursor is not initialized" + index_param = self.case_config.index_param() + try: metadata_arr = np.array(metadata) embeddings_arr = np.array(embeddings) - with self.cursor.copy( - sql.SQL("COPY public.{table_name} FROM STDIN (FORMAT BINARY)").format( - table_name=sql.Identifier(self.table_name), - ), - ) as copy: - copy.set_types(["bigint", "vector"]) - for i, row in enumerate(metadata_arr): - copy.write_row((row, embeddings_arr[i])) + if index_param["table_quantization_type"] == "bit": + with self.cursor.copy( + sql.SQL("COPY public.{table_name} FROM STDIN (FORMAT TEXT)").format( + table_name=sql.Identifier(self.table_name) + ) + ) as copy: + # Same logic as pgvector binary_quantize + for i, row in enumerate(metadata_arr): + embeddings_bit = "" + for embedding in embeddings_arr[i]: + if embedding > 0: + embeddings_bit += "1" + else: + embeddings_bit += "0" + copy.write_row((str(row), embeddings_bit)) + else: + with self.cursor.copy( + sql.SQL("COPY public.{table_name} FROM STDIN (FORMAT BINARY)").format( + table_name=sql.Identifier(self.table_name) + ) + ) as copy: + if index_param["table_quantization_type"] == "halfvec": + copy.set_types(["bigint", "halfvec"]) + for i, row in enumerate(metadata_arr): + copy.write_row((row, np.float16(embeddings_arr[i]))) + else: + copy.set_types(["bigint", "vector"]) + for i, row in enumerate(metadata_arr): + copy.write_row((row, embeddings_arr[i])) self.conn.commit() if kwargs.get("last_batch"): diff --git a/vectordb_bench/frontend/config/dbCaseConfigs.py b/vectordb_bench/frontend/config/dbCaseConfigs.py index a1c67e89a..c3918e7d6 100644 --- a/vectordb_bench/frontend/config/dbCaseConfigs.py +++ b/vectordb_bench/frontend/config/dbCaseConfigs.py @@ -823,6 +823,19 @@ class CaseConfigInput(BaseModel): ], ) +CaseConfigParamInput_TableQuantizationType_PgVector = CaseConfigInput( + label=CaseConfigParamType.tableQuantizationType, + inputType=InputType.Option, + inputConfig={ + "options": ["none", "bit", "halfvec"], + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) + in [ + IndexType.HNSW.value, + IndexType.IVFFlat.value, + ], +) + CaseConfigParamInput_max_parallel_workers_PgVectorRS = CaseConfigInput( label=CaseConfigParamType.max_parallel_workers, displayLabel="Max parallel workers", @@ -1138,6 +1151,7 @@ class CaseConfigInput(BaseModel): CaseConfigParamInput_m, CaseConfigParamInput_EFConstruction_PgVector, CaseConfigParamInput_QuantizationType_PgVector, + CaseConfigParamInput_TableQuantizationType_PgVector, CaseConfigParamInput_maintenance_work_mem_PgVector, CaseConfigParamInput_max_parallel_workers_PgVector, ] @@ -1149,6 +1163,7 @@ class CaseConfigInput(BaseModel): CaseConfigParamInput_Lists_PgVector, CaseConfigParamInput_Probes_PgVector, CaseConfigParamInput_QuantizationType_PgVector, + CaseConfigParamInput_TableQuantizationType_PgVector, CaseConfigParamInput_maintenance_work_mem_PgVector, CaseConfigParamInput_max_parallel_workers_PgVector, CaseConfigParamInput_reranking_PgVector, diff --git a/vectordb_bench/models.py b/vectordb_bench/models.py index bf71ebb89..fa80da2d5 100644 --- a/vectordb_bench/models.py +++ b/vectordb_bench/models.py @@ -49,6 +49,7 @@ class CaseConfigParamType(Enum): probes = "probes" quantizationType = "quantization_type" quantizationRatio = "quantization_ratio" + tableQuantizationType = "table_quantization_type" reranking = "reranking" rerankingMetric = "reranking_metric" quantizedFetchLimit = "quantized_fetch_limit" From 38a9a322274c64d333ad921084ef207b6b486797 Mon Sep 17 00:00:00 2001 From: Hugo Wen <46255328+HugoWenTD@users.noreply.github.com> Date: Mon, 10 Mar 2025 18:51:47 -0700 Subject: [PATCH 146/327] Support MariaDB database (#375) MariaDB introduced vector support in version 11.7, enabling MariaDB Server to function as a relational vector database. https://mariadb.com/kb/en/vectors/ Now add support for MariaDB server, verified against MariaDB server of version 11.7.1: - Support MariaDB vector search with HNSW algorithm, support filter search. - Support index and search parameters: - storage_engine: InnoDB or MyISAM - M: M parameter in MHNSW vector indexing - ef_search: minimal number of result candidates to look for in the vector index for ORDER BY ... LIMIT N queries. - max_cache_size: Upper limit for one MHNSW vector index cache - Support CLI of `vectordbbench mariadbhnsw`. --- pyproject.toml | 2 + vectordb_bench/backend/clients/__init__.py | 15 ++ vectordb_bench/backend/clients/mariadb/cli.py | 107 +++++++++ .../backend/clients/mariadb/config.py | 71 ++++++ .../backend/clients/mariadb/mariadb.py | 214 ++++++++++++++++++ vectordb_bench/cli/vectordbbench.py | 2 + .../frontend/config/dbCaseConfigs.py | 76 +++++++ vectordb_bench/models.py | 2 + 8 files changed, 489 insertions(+) create mode 100644 vectordb_bench/backend/clients/mariadb/cli.py create mode 100644 vectordb_bench/backend/clients/mariadb/config.py create mode 100644 vectordb_bench/backend/clients/mariadb/mariadb.py diff --git a/pyproject.toml b/pyproject.toml index 8bc83f29b..ad80bacfe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,6 +68,7 @@ all = [ "memorydb", "alibabacloud_ha3engine_vector", "alibabacloud_searchengine20211025", + "mariadb", ] qdrant = [ "qdrant-client" ] @@ -86,6 +87,7 @@ chromadb = [ "chromadb" ] opensearch = [ "opensearch-py" ] aliyun_opensearch = [ "alibabacloud_ha3engine_vector", "alibabacloud_searchengine20211025"] mongodb = [ "pymongo" ] +mariadb = [ "mariadb" ] [project.urls] "repository" = "https://github.com/zilliztech/VectorDBBench" diff --git a/vectordb_bench/backend/clients/__init__.py b/vectordb_bench/backend/clients/__init__.py index f2f480dcf..742da5213 100644 --- a/vectordb_bench/backend/clients/__init__.py +++ b/vectordb_bench/backend/clients/__init__.py @@ -38,6 +38,7 @@ class DB(Enum): Chroma = "Chroma" AWSOpenSearch = "OpenSearch" AliyunElasticsearch = "AliyunElasticsearch" + MariaDB = "MariaDB" Test = "test" AliyunOpenSearch = "AliyunOpenSearch" MongoDB = "MongoDB" @@ -135,6 +136,11 @@ def init_cls(self) -> type[VectorDB]: # noqa: PLR0911, PLR0912, C901 return MongoDB + if self == DB.MariaDB: + from .mariadb.mariadb import MariaDB + + return MariaDB + if self == DB.Test: from .test.test import Test @@ -236,6 +242,10 @@ def config_cls(self) -> type[DBConfig]: # noqa: PLR0911, PLR0912, C901 return MongoDBConfig + if self == DB.MariaDB: + from .mariadb.config import MariaDBConfig + return MariaDBConfig + if self == DB.Test: from .test.config import TestConfig @@ -318,6 +328,11 @@ def case_config_cls( # noqa: PLR0911 return MongoDBIndexConfig + if self == DB.MariaDB: + from .mariadb.config import _mariadb_case_config + + return _mariadb_case_config.get(index_type) + # DB.Pinecone, DB.Chroma, DB.Redis return EmptyDBCaseConfig diff --git a/vectordb_bench/backend/clients/mariadb/cli.py b/vectordb_bench/backend/clients/mariadb/cli.py new file mode 100644 index 000000000..c5439f37d --- /dev/null +++ b/vectordb_bench/backend/clients/mariadb/cli.py @@ -0,0 +1,107 @@ +from typing import Annotated, Optional, Unpack + +import click +import os +from pydantic import SecretStr + +from ....cli.cli import ( + CommonTypedDict, + HNSWFlavor1, + cli, + click_parameter_decorators_from_typed_dict, + run, +) +from vectordb_bench.backend.clients import DB + + +class MariaDBTypedDict(CommonTypedDict): + user_name: Annotated[ + str, click.option("--username", + type=str, + help="Username", + required=True, + ), + ] + password: Annotated[ + str, click.option("--password", + type=str, + help="Password", + required=True, + ), + ] + + host: Annotated[ + str, click.option("--host", + type=str, + help="Db host", + default="127.0.0.1", + ), + ] + + port: Annotated[ + int, click.option("--port", + type=int, + default=3306, + help="Db Port", + ), + ] + + storage_engine: Annotated[ + int, click.option("--storage-engine", + type=click.Choice(["InnoDB", "MyISAM"]), + help="DB storage engine", + required=True, + ), + ] + +class MariaDBHNSWTypedDict(MariaDBTypedDict): + ... + m: Annotated[ + Optional[int], click.option("--m", + type=int, + help="M parameter in MHNSW vector indexing", + required=False, + ), + ] + + ef_search: Annotated[ + Optional[int], click.option("--ef-search", + type=int, + help="MariaDB system variable mhnsw_min_limit", + required=False, + ), + ] + + max_cache_size: Annotated[ + Optional[int], click.option("--max-cache-size", + type=int, + help="MariaDB system variable mhnsw_max_cache_size", + required=False, + ), + ] + + +@cli.command() +@click_parameter_decorators_from_typed_dict(MariaDBHNSWTypedDict) +def MariaDBHNSW( + **parameters: Unpack[MariaDBHNSWTypedDict], +): + from .config import MariaDBConfig, MariaDBHNSWConfig + + run( + db=DB.MariaDB, + db_config=MariaDBConfig( + db_label=parameters["db_label"], + user_name=parameters["username"], + password=SecretStr(parameters["password"]), + host=parameters["host"], + port=parameters["port"], + ), + db_case_config=MariaDBHNSWConfig( + M=parameters["m"], + ef_search=parameters["ef_search"], + storage_engine=parameters["storage_engine"], + max_cache_size=parameters["max_cache_size"], + ), + **parameters, + ) diff --git a/vectordb_bench/backend/clients/mariadb/config.py b/vectordb_bench/backend/clients/mariadb/config.py new file mode 100644 index 000000000..c7b2cd5fe --- /dev/null +++ b/vectordb_bench/backend/clients/mariadb/config.py @@ -0,0 +1,71 @@ +from pydantic import SecretStr, BaseModel +from typing import TypedDict +from ..api import DBConfig, DBCaseConfig, MetricType, IndexType + +class MariaDBConfigDict(TypedDict): + """These keys will be directly used as kwargs in mariadb connection string, + so the names must match exactly mariadb API""" + + user: str + password: str + host: str + port: int + + +class MariaDBConfig(DBConfig): + user_name: str = "root" + password: SecretStr + host: str = "127.0.0.1" + port: int = 3306 + + def to_dict(self) -> MariaDBConfigDict: + pwd_str = self.password.get_secret_value() + return { + "host": self.host, + "port": self.port, + "user": self.user_name, + "password": pwd_str, + } + + +class MariaDBIndexConfig(BaseModel): + """Base config for MariaDB""" + + metric_type: MetricType | None = None + + def parse_metric(self) -> str: + if self.metric_type == MetricType.L2: + return "euclidean" + elif self.metric_type == MetricType.COSINE: + return "cosine" + else: + raise ValueError(f"Metric type {self.metric_type} is not supported!") + +class MariaDBHNSWConfig(MariaDBIndexConfig, DBCaseConfig): + M: int | None + ef_search: int | None + index: IndexType = IndexType.HNSW + storage_engine: str = "InnoDB" + max_cache_size: int | None + + def index_param(self) -> dict: + return { + "storage_engine": self.storage_engine, + "metric_type": self.parse_metric(), + "index_type": self.index.value, + "M": self.M, + "max_cache_size": self.max_cache_size, + } + + def search_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "ef_search": self.ef_search, + } + + +_mariadb_case_config = { + IndexType.HNSW: MariaDBHNSWConfig, +} + + diff --git a/vectordb_bench/backend/clients/mariadb/mariadb.py b/vectordb_bench/backend/clients/mariadb/mariadb.py new file mode 100644 index 000000000..42b621d9c --- /dev/null +++ b/vectordb_bench/backend/clients/mariadb/mariadb.py @@ -0,0 +1,214 @@ +from ..api import VectorDB + +import logging +from contextlib import contextmanager +from typing import Any, Optional, Tuple +from ..api import VectorDB +from .config import MariaDBConfigDict, MariaDBIndexConfig +import numpy as np + +import mariadb + +log = logging.getLogger(__name__) + +class MariaDB(VectorDB): + def __init__( + self, + dim: int, + db_config: MariaDBConfigDict, + db_case_config: MariaDBIndexConfig, + collection_name: str = "vec_collection", + drop_old: bool = False, + **kwargs, + ): + + self.name = "MariaDB" + self.db_config = db_config + self.case_config = db_case_config + self.db_name = "vectordbbench" + self.table_name = collection_name + self.dim = dim + + # construct basic units + self.conn, self.cursor = self._create_connection(**self.db_config) + + if drop_old: + self._drop_db() + self._create_db_table(dim) + + self.cursor.close() + self.conn.close() + self.cursor = None + self.conn = None + + + @staticmethod + def _create_connection(**kwargs) -> Tuple[mariadb.Connection, mariadb.Cursor]: + conn = mariadb.connect(**kwargs) + cursor = conn.cursor() + + assert conn is not None, "Connection is not initialized" + assert cursor is not None, "Cursor is not initialized" + + return conn, cursor + + + def _drop_db(self): + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + log.info(f"{self.name} client drop db : {self.db_name}") + + # flush tables before dropping database to avoid some locking issue + self.cursor.execute("FLUSH TABLES") + self.cursor.execute(f"DROP DATABASE IF EXISTS {self.db_name}") + self.cursor.execute("COMMIT") + self.cursor.execute("FLUSH TABLES") + + def _create_db_table(self, dim: int): + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + index_param = self.case_config.index_param() + + try: + log.info(f"{self.name} client create database : {self.db_name}") + self.cursor.execute(f"CREATE DATABASE {self.db_name}") + + log.info(f"{self.name} client create table : {self.table_name}") + self.cursor.execute(f"USE {self.db_name}") + + self.cursor.execute(f""" + CREATE TABLE {self.table_name} ( + id INT PRIMARY KEY, + v VECTOR({self.dim}) NOT NULL + ) ENGINE={index_param["storage_engine"]} + """) + self.cursor.execute("COMMIT") + + except Exception as e: + log.warning( + f"Failed to create table: {self.table_name} error: {e}" + ) + raise e from None + + + @contextmanager + def init(self) -> None: + """ create and destory connections to database. + + Examples: + >>> with self.init(): + >>> self.insert_embeddings() + """ + self.conn, self.cursor = self._create_connection(**self.db_config) + + index_param = self.case_config.index_param() + search_param = self.case_config.search_param() + + # maximize allowed package size + self.cursor.execute("SET GLOBAL max_allowed_packet = 1073741824") + + if index_param["index_type"] == "HNSW": + if index_param["max_cache_size"] != None: + self.cursor.execute(f"SET GLOBAL mhnsw_max_cache_size = {index_param["max_cache_size"]}") + if search_param["ef_search"] != None: + self.cursor.execute(f"SET mhnsw_ef_search = {search_param["ef_search"]}") + self.cursor.execute("COMMIT") + + self.insert_sql = f"INSERT INTO {self.db_name}.{self.table_name} (id, v) VALUES (%s, %s)" + self.select_sql = f"SELECT id FROM {self.db_name}.{self.table_name} ORDER by vec_distance_{search_param["metric_type"]}(v, %s) LIMIT %d" + self.select_sql_with_filter = f"SELECT id FROM {self.db_name}.{self.table_name} WHERE id >= %d ORDER by vec_distance_{search_param["metric_type"]}(v, %s) LIMIT %d" + + try: + yield + finally: + self.cursor.close() + self.conn.close() + self.cursor = None + self.conn = None + + + def ready_to_load(self) -> bool: + pass + + def optimize(self) -> None: + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + index_param = self.case_config.index_param() + + try: + index_options = f"DISTANCE={index_param['metric_type']}" + if index_param["index_type"] == "HNSW" and index_param["M"] != None: + index_options += f" M={index_param['M']}" + + self.cursor.execute(f""" + ALTER TABLE {self.db_name}.{self.table_name} + ADD VECTOR KEY v(v) {index_options} + """) + self.cursor.execute("COMMIT") + + except Exception as e: + log.warning( + f"Failed to create index: {self.table_name} error: {e}" + ) + raise e from None + + pass + + @staticmethod + def vector_to_hex(v): + return np.array(v, 'float32').tobytes() + + def insert_embeddings( + self, + embeddings: list[list[float]], + metadata: list[int], + **kwargs: Any, + ) -> Tuple[int, Optional[Exception]]: + """Insert embeddings into the database. + Should call self.init() first. + """ + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + try: + metadata_arr = np.array(metadata) + embeddings_arr = np.array(embeddings) + + batch_data = [] + for i, row in enumerate(metadata_arr): + batch_data.append((int(row), self.vector_to_hex(embeddings_arr[i]))); + + self.cursor.executemany(self.insert_sql, batch_data) + self.cursor.execute("COMMIT") + self.cursor.execute("FLUSH TABLES") + + return len(metadata), None + except Exception as e: + log.warning( + f"Failed to insert data into Vector table ({self.table_name}), error: {e}" + ) + return 0, e + + + def search_embedding( + self, + query: list[float], + k: int = 100, + filters: dict | None = None, + timeout: int | None = None, + **kwargs: Any, + ) -> (list[int]): + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + search_param = self.case_config.search_param() + + if filters: + self.cursor.execute(self.select_sql_with_filter, (filters.get('id'), self.vector_to_hex(query), k)) + else: + self.cursor.execute(self.select_sql, (self.vector_to_hex(query), k)) + + return [id for id, in self.cursor.fetchall()] + diff --git a/vectordb_bench/cli/vectordbbench.py b/vectordb_bench/cli/vectordbbench.py index 5e3798691..7934b3871 100644 --- a/vectordb_bench/cli/vectordbbench.py +++ b/vectordb_bench/cli/vectordbbench.py @@ -1,5 +1,6 @@ from ..backend.clients.alloydb.cli import AlloyDBScaNN from ..backend.clients.aws_opensearch.cli import AWSOpenSearch +from ..backend.clients.mariadb.cli import MariaDBHNSW from ..backend.clients.memorydb.cli import MemoryDB from ..backend.clients.milvus.cli import MilvusAutoIndex from ..backend.clients.pgdiskann.cli import PgDiskAnn @@ -25,6 +26,7 @@ cli.add_command(PgVectorScaleDiskAnn) cli.add_command(PgDiskAnn) cli.add_command(AlloyDBScaNN) +cli.add_command(MariaDBHNSW) if __name__ == "__main__": diff --git a/vectordb_bench/frontend/config/dbCaseConfigs.py b/vectordb_bench/frontend/config/dbCaseConfigs.py index c3918e7d6..0ab3a932b 100644 --- a/vectordb_bench/frontend/config/dbCaseConfigs.py +++ b/vectordb_bench/frontend/config/dbCaseConfigs.py @@ -1058,6 +1058,64 @@ class CaseConfigInput(BaseModel): }, ) +CaseConfigParamInput_IndexType_MariaDB = CaseConfigInput( + label=CaseConfigParamType.IndexType, + inputHelp="Select Index Type", + inputType=InputType.Option, + inputConfig={ + "options": [ + IndexType.HNSW.value, + ], + }, +) + +CaseConfigParamInput_StorageEngine_MariaDB = CaseConfigInput( + label=CaseConfigParamType.storage_engine, + inputHelp="Select Storage Engine", + inputType=InputType.Option, + inputConfig={ + "options": ["InnoDB", "MyISAM"], + }, +) + +CaseConfigParamInput_M_MariaDB = CaseConfigInput( + label=CaseConfigParamType.M, + inputHelp="M parameter in MHNSW vector indexing", + inputType=InputType.Number, + inputConfig={ + "min": 3, + "max": 200, + "value": 6, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) + == IndexType.HNSW.value, +) + +CaseConfigParamInput_EFSearch_MariaDB = CaseConfigInput( + label=CaseConfigParamType.ef_search, + inputHelp="mhnsw_ef_search", + inputType=InputType.Number, + inputConfig={ + "min": 1, + "max": 10000, + "value": 20, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) + == IndexType.HNSW.value, +) + +CaseConfigParamInput_CacheSize_MariaDB = CaseConfigInput( + label=CaseConfigParamType.max_cache_size, + inputHelp="mhnsw_max_cache_size", + inputType=InputType.Number, + inputConfig={ + "min": 1048576, + "max": (1 << 53) - 1, + "value": 16 * 1024 ** 3, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) + == IndexType.HNSW.value, +) CaseConfigParamInput_MongoDBQuantizationType = CaseConfigInput( label=CaseConfigParamType.mongodb_quantization_type, @@ -1272,6 +1330,20 @@ class CaseConfigInput(BaseModel): CaseConfigParamInput_MongoDBNumCandidatesRatio, ] +MariaDBLoadingConfig = [ + CaseConfigParamInput_IndexType_MariaDB, + CaseConfigParamInput_StorageEngine_MariaDB, + CaseConfigParamInput_M_MariaDB, + CaseConfigParamInput_CacheSize_MariaDB, +] +MariaDBPerformanceConfig = [ + CaseConfigParamInput_IndexType_MariaDB, + CaseConfigParamInput_StorageEngine_MariaDB, + CaseConfigParamInput_M_MariaDB, + CaseConfigParamInput_CacheSize_MariaDB, + CaseConfigParamInput_EFSearch_MariaDB, +] + CASE_CONFIG_MAP = { DB.Milvus: { CaseLabel.Load: MilvusLoadConfig, @@ -1324,4 +1396,8 @@ class CaseConfigInput(BaseModel): CaseLabel.Load: MongoDBLoadingConfig, CaseLabel.Performance: MongoDBPerformanceConfig, }, + DB.MariaDB: { + CaseLabel.Load: MariaDBLoadingConfig, + CaseLabel.Performance: MariaDBPerformanceConfig, + }, } diff --git a/vectordb_bench/models.py b/vectordb_bench/models.py index fa80da2d5..e206919ac 100644 --- a/vectordb_bench/models.py +++ b/vectordb_bench/models.py @@ -88,6 +88,8 @@ class CaseConfigParamType(Enum): preReorderingNumNeigbors = "pre_reordering_num_neighbors" numSearchThreads = "num_search_threads" maxNumPrefetchDatasets = "max_num_prefetch_datasets" + storage_engine = "storage_engine" + max_cache_size = "max_cache_size" # mongodb params mongodb_quantization_type = "quantization" From f7d9210ec188b32a3123742550f23c1fc3a7dfe7 Mon Sep 17 00:00:00 2001 From: Wenxuan Date: Thu, 13 Mar 2025 10:21:05 +0800 Subject: [PATCH 147/327] Add TiDB backend (#484) * Add TiDB backend Signed-off-by: Wish * Fix Signed-off-by: Wish * Fix Signed-off-by: Wish * Improve error handling Signed-off-by: Wish --------- Signed-off-by: Wish --- README.md | 1 + pyproject.toml | 4 +- vectordb_bench/backend/clients/__init__.py | 17 ++ vectordb_bench/backend/clients/tidb/cli.py | 98 ++++++++ vectordb_bench/backend/clients/tidb/config.py | 49 ++++ vectordb_bench/backend/clients/tidb/tidb.py | 234 ++++++++++++++++++ vectordb_bench/cli/vectordbbench.py | 2 + vectordb_bench/frontend/config/styles.py | 2 + 8 files changed, 406 insertions(+), 1 deletion(-) create mode 100644 vectordb_bench/backend/clients/tidb/cli.py create mode 100644 vectordb_bench/backend/clients/tidb/config.py create mode 100644 vectordb_bench/backend/clients/tidb/tidb.py diff --git a/README.md b/README.md index 80db92ec5..7a52c3242 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,7 @@ All the database client supported | awsopensearch | `pip install vectordb-bench[opensearch]` | | aliyun_opensearch | `pip install vectordb-bench[aliyun_opensearch]` | | mongodb | `pip install vectordb-bench[mongodb]` | +| tidb | `pip install vectordb-bench[tidb]` | ### Run diff --git a/pyproject.toml b/pyproject.toml index ad80bacfe..019c2973e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,6 +69,7 @@ all = [ "alibabacloud_ha3engine_vector", "alibabacloud_searchengine20211025", "mariadb", + "PyMySQL", ] qdrant = [ "qdrant-client" ] @@ -87,7 +88,8 @@ chromadb = [ "chromadb" ] opensearch = [ "opensearch-py" ] aliyun_opensearch = [ "alibabacloud_ha3engine_vector", "alibabacloud_searchengine20211025"] mongodb = [ "pymongo" ] -mariadb = [ "mariadb" ] +mariadb = [ "mariadb" ] +tidb = [ "PyMySQL" ] [project.urls] "repository" = "https://github.com/zilliztech/VectorDBBench" diff --git a/vectordb_bench/backend/clients/__init__.py b/vectordb_bench/backend/clients/__init__.py index 742da5213..8d667ec23 100644 --- a/vectordb_bench/backend/clients/__init__.py +++ b/vectordb_bench/backend/clients/__init__.py @@ -42,6 +42,7 @@ class DB(Enum): Test = "test" AliyunOpenSearch = "AliyunOpenSearch" MongoDB = "MongoDB" + TiDB = "TiDB" @property def init_cls(self) -> type[VectorDB]: # noqa: PLR0911, PLR0912, C901 @@ -141,6 +142,11 @@ def init_cls(self) -> type[VectorDB]: # noqa: PLR0911, PLR0912, C901 return MariaDB + if self == DB.TiDB: + from .tidb.tidb import TiDB + + return TiDB + if self == DB.Test: from .test.test import Test @@ -244,8 +250,14 @@ def config_cls(self) -> type[DBConfig]: # noqa: PLR0911, PLR0912, C901 if self == DB.MariaDB: from .mariadb.config import MariaDBConfig + return MariaDBConfig + if self == DB.TiDB: + from .tidb.config import TiDBConfig + + return TiDBConfig + if self == DB.Test: from .test.config import TestConfig @@ -333,6 +345,11 @@ def case_config_cls( # noqa: PLR0911 return _mariadb_case_config.get(index_type) + if self == DB.TiDB: + from .tidb.config import TiDBIndexConfig + + return TiDBIndexConfig + # DB.Pinecone, DB.Chroma, DB.Redis return EmptyDBCaseConfig diff --git a/vectordb_bench/backend/clients/tidb/cli.py b/vectordb_bench/backend/clients/tidb/cli.py new file mode 100644 index 000000000..cdfcbe432 --- /dev/null +++ b/vectordb_bench/backend/clients/tidb/cli.py @@ -0,0 +1,98 @@ +from typing import Annotated, Unpack + +import click +from pydantic import SecretStr + +from vectordb_bench.backend.clients import DB + +from ....cli.cli import CommonTypedDict, cli, click_parameter_decorators_from_typed_dict, run + + +class TiDBTypedDict(CommonTypedDict): + user_name: Annotated[ + str, + click.option( + "--username", + type=str, + help="Username", + default="root", + show_default=True, + required=True, + ), + ] + password: Annotated[ + str, + click.option( + "--password", + type=str, + default="", + show_default=True, + help="Password", + ), + ] + host: Annotated[ + str, + click.option( + "--host", + type=str, + default="127.0.0.1", + show_default=True, + required=True, + help="Db host", + ), + ] + port: Annotated[ + int, + click.option( + "--port", + type=int, + default=4000, + show_default=True, + required=True, + help="Db Port", + ), + ] + db_name: Annotated[ + str, + click.option( + "--db-name", + type=str, + default="test", + show_default=True, + required=True, + help="Db name", + ), + ] + ssl: Annotated[ + bool, + click.option( + "--ssl/--no-ssl", + default=False, + show_default=True, + is_flag=True, + help="Enable or disable SSL, for TiDB Serverless SSL must be enabled", + ), + ] + + +@cli.command() +@click_parameter_decorators_from_typed_dict(TiDBTypedDict) +def TiDB( + **parameters: Unpack[TiDBTypedDict], +): + from .config import TiDBConfig, TiDBIndexConfig + + run( + db=DB.TiDB, + db_config=TiDBConfig( + db_label=parameters["db_label"], + user_name=parameters["username"], + password=SecretStr(parameters["password"]), + host=parameters["host"], + port=parameters["port"], + db_name=parameters["db_name"], + ssl=parameters["ssl"], + ), + db_case_config=TiDBIndexConfig(), + **parameters, + ) diff --git a/vectordb_bench/backend/clients/tidb/config.py b/vectordb_bench/backend/clients/tidb/config.py new file mode 100644 index 000000000..213a18bc5 --- /dev/null +++ b/vectordb_bench/backend/clients/tidb/config.py @@ -0,0 +1,49 @@ +from pydantic import SecretStr, BaseModel, validator +from ..api import DBConfig, DBCaseConfig, MetricType + + +class TiDBConfig(DBConfig): + user_name: str = "root" + password: SecretStr + host: str = "127.0.0.1" + port: int = 4000 + db_name: str = "test" + ssl: bool = False + + @validator("*") + def not_empty_field(cls, v: any, field: any): + return v + + def to_dict(self) -> dict: + pwd_str = self.password.get_secret_value() + return { + "host": self.host, + "port": self.port, + "user": self.user_name, + "password": pwd_str, + "database": self.db_name, + "ssl_verify_cert": self.ssl, + "ssl_verify_identity": self.ssl, + } + + +class TiDBIndexConfig(BaseModel, DBCaseConfig): + metric_type: MetricType | None = None + + def get_metric_fn(self) -> str: + if self.metric_type == MetricType.L2: + return "vec_l2_distance" + elif self.metric_type == MetricType.COSINE: + return "vec_cosine_distance" + else: + raise ValueError(f"Unsupported metric type: {self.metric_type}") + + def index_param(self) -> dict: + return { + "metric_fn": self.get_metric_fn(), + } + + def search_param(self) -> dict: + return { + "metric_fn": self.get_metric_fn(), + } diff --git a/vectordb_bench/backend/clients/tidb/tidb.py b/vectordb_bench/backend/clients/tidb/tidb.py new file mode 100644 index 000000000..d1f26084e --- /dev/null +++ b/vectordb_bench/backend/clients/tidb/tidb.py @@ -0,0 +1,234 @@ +import concurrent.futures +import io +import logging +import time +from contextlib import contextmanager +from typing import Any, Optional, Tuple + +import pymysql + +from ..api import VectorDB +from .config import TiDBIndexConfig + +log = logging.getLogger(__name__) + + +class TiDB(VectorDB): + def __init__( + self, + dim: int, + db_config: dict, + db_case_config: TiDBIndexConfig, + collection_name: str = "vector_bench_test", + drop_old: bool = False, + **kwargs, + ): + self.name = "TiDB" + self.db_config = db_config + self.case_config = db_case_config + self.table_name = collection_name + self.dim = dim + self.conn = None # To be inited by init() + self.cursor = None # To be inited by init() + + self.search_fn = db_case_config.search_param()["metric_fn"] + + if drop_old: + self._drop_table() + self._create_table() + + @contextmanager + def init(self): + with self._get_connection() as (conn, cursor): + self.conn = conn + self.cursor = cursor + try: + yield + finally: + self.conn = None + self.cursor = None + + @contextmanager + def _get_connection(self): + with pymysql.connect(**self.db_config) as conn: + conn.autocommit = False + with conn.cursor() as cursor: + yield conn, cursor + + def _drop_table(self): + try: + with self._get_connection() as (conn, cursor): + cursor.execute(f"DROP TABLE IF EXISTS {self.table_name}") + conn.commit() + except Exception as e: + log.warning("Failed to drop table: %s error: %s", self.table_name, e) + raise e + + def _create_table(self): + try: + index_param = self.case_config.index_param() + with self._get_connection() as (conn, cursor): + cursor.execute( + f""" + CREATE TABLE {self.table_name} ( + id BIGINT PRIMARY KEY, + embedding VECTOR({self.dim}) NOT NULL, + VECTOR INDEX (({index_param["metric_fn"]}(embedding))) + ); + """ + ) + conn.commit() + except Exception as e: + log.warning("Failed to create table: %s error: %s", self.table_name, e) + raise e + + def ready_to_load(self) -> bool: + pass + + def optimize(self, data_size: int | None = None) -> None: + while True: + progress = self._optimize_check_tiflash_replica_progress() + if progress != 1: + log.info("Data replication not ready, progress: %d", progress) + time.sleep(2) + else: + break + + log.info("Waiting TiFlash to catch up...") + self._optimize_wait_tiflash_catch_up() + + log.info("Start compacting TiFlash replica...") + self._optimize_compact_tiflash() + + log.info("Waiting index build to finish...") + log_reduce_seq = 0 + while True: + pending_rows = self._optimize_get_tiflash_index_pending_rows() + if pending_rows > 0: + if log_reduce_seq % 15 == 0: + log.info("Index not fully built, pending rows: %d", pending_rows) + log_reduce_seq += 1 + time.sleep(2) + else: + break + + log.info("Index build finished successfully.") + + def _optimize_check_tiflash_replica_progress(self): + try: + database = self.db_config["database"] + with self._get_connection() as (_, cursor): + cursor.execute( + f""" + SELECT PROGRESS FROM information_schema.tiflash_replica + WHERE TABLE_SCHEMA = "{database}" AND TABLE_NAME = "{self.table_name}" + """ + ) + result = cursor.fetchone() + return result[0] + except Exception as e: + log.warning("Failed to check TiFlash replica progress: %s", e) + raise e + + def _optimize_wait_tiflash_catch_up(self): + try: + with self._get_connection() as (conn, cursor): + cursor.execute('SET @@TIDB_ISOLATION_READ_ENGINES="tidb,tiflash"') + conn.commit() + cursor.execute(f"SELECT COUNT(*) FROM {self.table_name}") + result = cursor.fetchone() + return result[0] + except Exception as e: + log.warning("Failed to wait TiFlash to catch up: %s", e) + raise e + + def _optimize_compact_tiflash(self): + try: + with self._get_connection() as (conn, cursor): + cursor.execute(f"ALTER TABLE {self.table_name} COMPACT") + conn.commit() + except Exception as e: + log.warning("Failed to compact table: %s", e) + raise e + + def _optimize_get_tiflash_index_pending_rows(self): + try: + database = self.db_config["database"] + with self._get_connection() as (_, cursor): + cursor.execute( + f""" + SELECT SUM(ROWS_STABLE_NOT_INDEXED) + FROM information_schema.tiflash_indexes + WHERE TIDB_DATABASE = "{database}" AND TIDB_TABLE = "{self.table_name}" + """ + ) + result = cursor.fetchone() + return result[0] + except Exception as e: + log.warning("Failed to read TiFlash index pending rows: %s", e) + raise e + + def _insert_embeddings_serial( + self, + embeddings: list[list[float]], + metadata: list[int], + offset: int, + size: int, + ) -> Exception: + try: + with self._get_connection() as (conn, cursor): + buf = io.StringIO() + buf.write(f"INSERT INTO {self.table_name} (id, embedding) VALUES ") + for i in range(offset, offset + size): + if i > offset: + buf.write(",") + buf.write(f'({metadata[i]}, "{str(embeddings[i])}")') + cursor.execute(buf.getvalue()) + conn.commit() + except Exception as e: + log.warning("Failed to insert data into table: %s", e) + raise e + + def insert_embeddings( + self, + embeddings: list[list[float]], + metadata: list[int], + **kwargs: Any, + ) -> Tuple[int, Optional[Exception]]: + workers = 10 + # Avoid exceeding MAX_ALLOWED_PACKET (default=64MB) + max_batch_size = 64 * 1024 * 1024 // 24 // self.dim + batch_size = len(embeddings) // workers + if batch_size > max_batch_size: + batch_size = max_batch_size + with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor: + futures = [] + for i in range(0, len(embeddings), batch_size): + offset = i + size = min(batch_size, len(embeddings) - i) + future = executor.submit(self._insert_embeddings_serial, embeddings, metadata, offset, size) + futures.append(future) + done, pending = concurrent.futures.wait(futures, return_when=concurrent.futures.FIRST_EXCEPTION) + executor.shutdown(wait=False) + for future in done: + future.result() + for future in pending: + future.cancel() + return len(metadata), None + + def search_embedding( + self, + query: list[float], + k: int = 100, + filters: dict | None = None, + timeout: int | None = None, + **kwargs: Any, + ) -> list[int]: + self.cursor.execute( + f""" + SELECT id FROM {self.table_name} + ORDER BY {self.search_fn}(embedding, "{str(query)}") LIMIT {k}; + """ + ) + result = self.cursor.fetchall() + return [int(i[0]) for i in result] diff --git a/vectordb_bench/cli/vectordbbench.py b/vectordb_bench/cli/vectordbbench.py index 7934b3871..49428b678 100644 --- a/vectordb_bench/cli/vectordbbench.py +++ b/vectordb_bench/cli/vectordbbench.py @@ -11,6 +11,7 @@ from ..backend.clients.test.cli import Test from ..backend.clients.weaviate_cloud.cli import Weaviate from ..backend.clients.zilliz_cloud.cli import ZillizAutoIndex +from ..backend.clients.tidb.cli import TiDB from .cli import cli cli.add_command(PgVectorHNSW) @@ -27,6 +28,7 @@ cli.add_command(PgDiskAnn) cli.add_command(AlloyDBScaNN) cli.add_command(MariaDBHNSW) +cli.add_command(TiDB) if __name__ == "__main__": diff --git a/vectordb_bench/frontend/config/styles.py b/vectordb_bench/frontend/config/styles.py index 3e0fdb112..57456722f 100644 --- a/vectordb_bench/frontend/config/styles.py +++ b/vectordb_bench/frontend/config/styles.py @@ -47,6 +47,7 @@ def getPatternShape(i): DB.Redis: "https://assets.zilliz.com/Redis_Cloud_74b8bfef39.png", DB.Chroma: "https://assets.zilliz.com/chroma_ceb3f06ed7.png", DB.AWSOpenSearch: "https://assets.zilliz.com/opensearch_1eee37584e.jpeg", + DB.TiDB: "https://img2.pingcap.com/forms/3/d/3d7fd5f9767323d6f037795704211ac44b4923d6.png", } # RedisCloud color: #0D6EFD @@ -61,4 +62,5 @@ def getPatternShape(i): DB.PgVector.value: "#4C779A", DB.Redis.value: "#0D6EFD", DB.AWSOpenSearch.value: "#0DCAF0", + DB.TiDB.value: "#0D6EFD", } From d2f102e884466f2d7bc6e4d89f3b4a5d93869715 Mon Sep 17 00:00:00 2001 From: Rachit Chaudhary <65501028+Rachit-Chaudhary11@users.noreply.github.com> Date: Fri, 14 Mar 2025 07:14:36 +0530 Subject: [PATCH 148/327] CLI fix for GPU index (#485) * Support GPU_BRUTE_FORCE index for Milvus Signed-off-by: Rachit Chaudhary * MilvusGPUBruteForceTypedDict addition Signed-off-by: Rachit Chaudhary --------- Signed-off-by: Rachit Chaudhary Co-authored-by: Signed-off-by: Rachit Chaudhary - r0c0axe --- vectordb_bench/backend/clients/milvus/cli.py | 30 ++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/vectordb_bench/backend/clients/milvus/cli.py b/vectordb_bench/backend/clients/milvus/cli.py index 303eec5f9..1bec8ebe5 100644 --- a/vectordb_bench/backend/clients/milvus/cli.py +++ b/vectordb_bench/backend/clients/milvus/cli.py @@ -214,6 +214,36 @@ def MilvusGPUBruteForce(**parameters: Unpack[MilvusGPUBruteForceTypedDict]): **parameters, ) +class MilvusGPUBruteForceTypedDict(CommonTypedDict, MilvusTypedDict): + metric_type: Annotated[ + str, + click.option("--metric-type", type=str, required=True, help="Metric type for brute force search"), + ] + limit: Annotated[ + int, + click.option("--limit", type=int, required=True, help="Top-k limit for search"), + ] + +@cli.command() +@click_parameter_decorators_from_typed_dict(MilvusGPUBruteForceTypedDict) +def MilvusGPUBruteForce(**parameters: Unpack[MilvusGPUBruteForceTypedDict]): + from .config import GPUBruteForceConfig, MilvusConfig + + run( + db=DBTYPE, + db_config=MilvusConfig( + db_label=parameters["db_label"], + uri=SecretStr(parameters["uri"]), + user=parameters["user_name"], + password=SecretStr(parameters["password"]), + ), + db_case_config=GPUBruteForceConfig( + metric_type=parameters["metric_type"], + limit=parameters["limit"], # top-k for search + ), + **parameters, + ) + class MilvusGPUIVFPQTypedDict( CommonTypedDict, From 4d0cedd9aaae2cc151483caf5b3a03d20ef8a40f Mon Sep 17 00:00:00 2001 From: yuyuankang Date: Tue, 25 Mar 2025 20:28:00 +0000 Subject: [PATCH 149/327] remove duplicated code --- vectordb_bench/backend/clients/milvus/cli.py | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/vectordb_bench/backend/clients/milvus/cli.py b/vectordb_bench/backend/clients/milvus/cli.py index 1bec8ebe5..52524e785 100644 --- a/vectordb_bench/backend/clients/milvus/cli.py +++ b/vectordb_bench/backend/clients/milvus/cli.py @@ -194,26 +194,6 @@ def MilvusGPUIVFFlat(**parameters: Unpack[MilvusGPUIVFTypedDict]): **parameters, ) -@cli.command() -@click_parameter_decorators_from_typed_dict(MilvusGPUBruteForceTypedDict) -def MilvusGPUBruteForce(**parameters: Unpack[MilvusGPUBruteForceTypedDict]): - from .config import GPUBruteForceConfig, MilvusConfig - - run( - db=DBTYPE, - db_config=MilvusConfig( - db_label=parameters["db_label"], - uri=SecretStr(parameters["uri"]), - user=parameters["user_name"], - password=SecretStr(parameters["password"]), - ), - db_case_config=GPUBruteForceConfig( - metric_type=parameters["metric_type"], - limit=parameters["limit"], # top-k for search - ), - **parameters, - ) - class MilvusGPUBruteForceTypedDict(CommonTypedDict, MilvusTypedDict): metric_type: Annotated[ str, From 809024fe4224a8208fa263a95fbd68782b64269c Mon Sep 17 00:00:00 2001 From: Arseniy Ahtaryanov Date: Tue, 8 Apr 2025 17:56:23 +0300 Subject: [PATCH 150/327] feat: initial commit --- install/requirements_py3.11.txt | 1 + pyproject.toml | 2 + vectordb_bench/backend/clients/__init__.py | 16 ++ .../backend/clients/clickhouse/cli.py | 66 ++++++++ .../backend/clients/clickhouse/clickhouse.py | 149 ++++++++++++++++++ .../backend/clients/clickhouse/config.py | 56 +++++++ vectordb_bench/cli/vectordbbench.py | 2 + 7 files changed, 292 insertions(+) create mode 100644 vectordb_bench/backend/clients/clickhouse/cli.py create mode 100644 vectordb_bench/backend/clients/clickhouse/clickhouse.py create mode 100644 vectordb_bench/backend/clients/clickhouse/config.py diff --git a/install/requirements_py3.11.txt b/install/requirements_py3.11.txt index 18138382f..6969bf9dd 100644 --- a/install/requirements_py3.11.txt +++ b/install/requirements_py3.11.txt @@ -22,3 +22,4 @@ environs pydantic type[VectorDB]: # noqa: PLR0911, PLR0912, C901 @@ -117,6 +118,11 @@ def init_cls(self) -> type[VectorDB]: # noqa: PLR0911, PLR0912, C901 return AWSOpenSearch + if self == DB.Clickhouse: + from .clickhouse.clickhouse import Clickhouse + + return Clickhouse + if self == DB.AlloyDB: from .alloydb.alloydb import AlloyDB @@ -228,6 +234,11 @@ def config_cls(self) -> type[DBConfig]: # noqa: PLR0911, PLR0912, C901 return AWSOpenSearchConfig + if self == DB.Clickhouse: + from .clickhouse.config import ClickhouseConfig + + return ClickhouseConfig + if self == DB.AlloyDB: from .alloydb.config import AlloyDBConfig @@ -310,6 +321,11 @@ def case_config_cls( # noqa: PLR0911 return AWSOpenSearchIndexConfig + if self == DB.Clickhouse: + from .clickhouse.config import ClickhouseHNSWConfig + + return ClickhouseHNSWConfig + if self == DB.PgVectorScale: from .pgvectorscale.config import _pgvectorscale_case_config diff --git a/vectordb_bench/backend/clients/clickhouse/cli.py b/vectordb_bench/backend/clients/clickhouse/cli.py new file mode 100644 index 000000000..e454f5a75 --- /dev/null +++ b/vectordb_bench/backend/clients/clickhouse/cli.py @@ -0,0 +1,66 @@ +from typing import Annotated, TypedDict, Unpack + +import click +from pydantic import SecretStr + +from ....cli.cli import ( + CommonTypedDict, + HNSWFlavor2, + cli, + click_parameter_decorators_from_typed_dict, + run, +) +from .. import DB +from .config import ClickhouseHNSWConfig + + +class ClickhouseTypedDict(TypedDict): + password: Annotated[str, click.option("--password", type=str, help="DB password")] + host: Annotated[str, click.option("--host", type=str, help="DB host", required=True)] + port: Annotated[int, click.option("--port", type=int, default=8123, help="DB Port")] + user: Annotated[int, click.option("--user", type=str, default='clickhouse', help="DB user")] + ssl: Annotated[ + bool, + click.option( + "--ssl/--no-ssl", + is_flag=True, + show_default=True, + default=True, + help="Enable or disable SSL for Clickhouse", + ), + ] + ssl_ca_certs: Annotated[ + str, + click.option( + "--ssl-ca-certs", + show_default=True, + help="Path to certificate authority file to use for SSL", + ), + ] + + +class ClickhouseHNSWTypedDict(CommonTypedDict, ClickhouseTypedDict, HNSWFlavor2): ... + + +@cli.command() +@click_parameter_decorators_from_typed_dict(ClickhouseHNSWTypedDict) +def Clickhouse(**parameters: Unpack[ClickhouseHNSWTypedDict]): + from .config import ClickhouseConfig + + run( + db=DB.Clickhouse, + db_config=ClickhouseConfig( + db_label=parameters["db_label"], + password=SecretStr(parameters["password"]) if parameters["password"] else None, + host=parameters["host"], + port=parameters["port"], + ssl=parameters["ssl"], + ssl_ca_certs=parameters["ssl_ca_certs"], + ), + db_case_config=ClickhouseHNSWConfig( + M=parameters["m"], + efConstruction=parameters["ef_construction"], + ef=parameters["ef_runtime"], + ), + **parameters, + ) diff --git a/vectordb_bench/backend/clients/clickhouse/clickhouse.py b/vectordb_bench/backend/clients/clickhouse/clickhouse.py new file mode 100644 index 000000000..e5339170b --- /dev/null +++ b/vectordb_bench/backend/clients/clickhouse/clickhouse.py @@ -0,0 +1,149 @@ +"""Wrapper around the Clickhouse vector database over VectorDB""" + +import io +import logging +from contextlib import contextmanager +from typing import Any +import clickhouse_connect +import numpy as np + +from ..api import VectorDB, DBCaseConfig + +log = logging.getLogger(__name__) + +class Clickhouse(VectorDB): + """Use SQLAlchemy instructions""" + def __init__( + self, + dim: int, + db_config: dict, + db_case_config: DBCaseConfig, + collection_name: str = "CHVectorCollection", + drop_old: bool = False, + **kwargs, + ): + self.db_config = db_config + self.case_config = db_case_config + self.table_name = collection_name + self.dim = dim + + self._index_name = "clickhouse_index" + self._primary_field = "id" + self._vector_field = "embedding" + + # construct basic units + self.conn = clickhouse_connect.get_client( + host=self.db_config["host"], + port=self.db_config["port"], + username=self.db_config["user"], + password=self.db_config["password"], + database=self.db_config["dbname"]) + + if drop_old: + log.info(f"Clickhouse client drop table : {self.table_name}") + self._drop_table() + self._create_table(dim) + + self.conn.close() + self.conn = None + + @contextmanager + def init(self) -> None: + """ + Examples: + >>> with self.init(): + >>> self.insert_embeddings() + >>> self.search_embedding() + """ + + self.conn = clickhouse_connect.get_client( + host=self.db_config["host"], + port=self.db_config["port"], + username=self.db_config["user"], + password=self.db_config["password"], + database=self.db_config["dbname"]) + + try: + yield + finally: + self.conn.close() + self.conn = None + + def _drop_table(self): + assert self.conn is not None, "Connection is not initialized" + + self.conn.command(f'DROP TABLE IF EXISTS {self.db_config["dbname"]}.{self.table_name}') + + def _create_table(self, dim: int): + assert self.conn is not None, "Connection is not initialized" + + try: + # create table + self.conn.command( + f'CREATE TABLE IF NOT EXISTS {self.db_config["dbname"]}.{self.table_name} \ + (id UInt32, embedding Array(Float64)) ENGINE = MergeTree() ORDER BY id;' + ) + + except Exception as e: + log.warning( + f"Failed to create Clickhouse table: {self.table_name} error: {e}" + ) + raise e from None + + def ready_to_load(self): + pass + + def optimize(self, data_size: int | None = None): + pass + + def ready_to_search(self): + pass + + def insert_embeddings( + self, + embeddings: list[list[float]], + metadata: list[int], + **kwargs: Any, + ) -> (int, Exception): + assert self.conn is not None, "Connection is not initialized" + + try: + # do not iterate for bulk insert + items = [metadata, embeddings] + + self.conn.insert(table=self.table_name, data=items, + column_names=['id', 'embedding'], column_type_names=['UInt32', 'Array(Float64)'], + column_oriented=True) + return len(metadata), None + except Exception as e: + log.warning(f"Failed to insert data into Clickhouse table ({self.table_name}), error: {e}") + return 0, e + + def search_embedding( + self, + query: list[float], + k: int = 100, + filters: dict | None = None, + timeout: int | None = None, + ) -> list[int]: + assert self.conn is not None, "Connection is not initialized" + + index_param = self.case_config.index_param() + search_param = self.case_config.search_param() + + if filters: + gt = filters.get("id") + filterSql = (f'SELECT id, {search_param["metric_type"]}(embedding,{query}) AS score ' + f'FROM {self.db_config["dbname"]}.{self.table_name} ' + f'WHERE id > {gt} ' + f'ORDER BY score LIMIT {k};' + ) + result = self.conn.query(filterSql).result_rows + return [int(row[0]) for row in result] + else: + selectSql = (f'SELECT id, {search_param["metric_type"]}(embedding,{query}) AS score ' + f'FROM {self.db_config["dbname"]}.{self.table_name} ' + f'ORDER BY score LIMIT {k};' + ) + result = self.conn.query(selectSql).result_rows + return [int(row[0]) for row in result] diff --git a/vectordb_bench/backend/clients/clickhouse/config.py b/vectordb_bench/backend/clients/clickhouse/config.py new file mode 100644 index 000000000..7ce0919ea --- /dev/null +++ b/vectordb_bench/backend/clients/clickhouse/config.py @@ -0,0 +1,56 @@ +from typing import TypedDict +from pydantic import BaseModel, SecretStr +from ..api import DBConfig, DBCaseConfig, MetricType, IndexType + +class ClickhouseConfig(DBConfig): + user_name: str = "clickhouse" + password: SecretStr + host: str = "localhost" + port: int = 8123 + db_name: str = "default" + + def to_dict(self) -> dict: + pwd_str = self.password.get_secret_value() + return { + "host": self.host, + "port": self.port, + "dbname": self.db_name, + "user": self.user_name, + "password": pwd_str + } + + +class ClickhouseIndexConfig(BaseModel): + + metric_type: MetricType | None = None + + def parse_metric(self) -> str: + if not self.metric_type: + return "" + return self.metric_type.value + + def parse_metric_str(self) -> str: + if self.metric_type == MetricType.L2: + return "L2Distance" + elif self.metric_type == MetricType.COSINE: + return "cosineDistance" + + +class ClickhouseHNSWConfig(ClickhouseIndexConfig, DBCaseConfig): + M: int | None + efConstruction: int | None + ef: int | None = None + index: IndexType = IndexType.HNSW + + def index_param(self) -> dict: + return { + "metric_type": self.parse_metric_str(), + "index_type": self.index.value, + "params": {"M": self.M, "efConstruction": self.efConstruction}, + } + + def search_param(self) -> dict: + return { + "metric_type": self.parse_metric_str(), + "params": {"ef": self.ef}, + } \ No newline at end of file diff --git a/vectordb_bench/cli/vectordbbench.py b/vectordb_bench/cli/vectordbbench.py index 49428b678..7f5e24fc2 100644 --- a/vectordb_bench/cli/vectordbbench.py +++ b/vectordb_bench/cli/vectordbbench.py @@ -12,6 +12,7 @@ from ..backend.clients.weaviate_cloud.cli import Weaviate from ..backend.clients.zilliz_cloud.cli import ZillizAutoIndex from ..backend.clients.tidb.cli import TiDB +from ..backend.clients.clickhouse.cli import Clickhouse from .cli import cli cli.add_command(PgVectorHNSW) @@ -29,6 +30,7 @@ cli.add_command(AlloyDBScaNN) cli.add_command(MariaDBHNSW) cli.add_command(TiDB) +cli.add_command(Clickhouse) if __name__ == "__main__": From d6364b7c635a8e2bf234bcbb6e4b6043cfea14dc Mon Sep 17 00:00:00 2001 From: nuvotex-tk <161840620+nuvotex-tk@users.noreply.github.com> Date: Tue, 8 Apr 2025 11:28:41 +0200 Subject: [PATCH 151/327] Add vespa integration --- README.md | 1 + install/requirements_py3.11.txt | 1 + pyproject.toml | 4 +- vectordb_bench/backend/clients/__init__.py | 17 ++ vectordb_bench/backend/clients/vespa/cli.py | 47 ++++ .../backend/clients/vespa/config.py | 51 ++++ vectordb_bench/backend/clients/vespa/util.py | 16 ++ vectordb_bench/backend/clients/vespa/vespa.py | 254 ++++++++++++++++++ vectordb_bench/cli/vectordbbench.py | 4 +- .../frontend/config/dbCaseConfigs.py | 57 ++++ vectordb_bench/frontend/config/styles.py | 2 + 11 files changed, 452 insertions(+), 2 deletions(-) create mode 100644 vectordb_bench/backend/clients/vespa/cli.py create mode 100644 vectordb_bench/backend/clients/vespa/config.py create mode 100644 vectordb_bench/backend/clients/vespa/util.py create mode 100644 vectordb_bench/backend/clients/vespa/vespa.py diff --git a/README.md b/README.md index 7a52c3242..6d83671da 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,7 @@ All the database client supported | aliyun_opensearch | `pip install vectordb-bench[aliyun_opensearch]` | | mongodb | `pip install vectordb-bench[mongodb]` | | tidb | `pip install vectordb-bench[tidb]` | +| vespa | `pip install vectordb-bench[vespa]` | ### Run diff --git a/install/requirements_py3.11.txt b/install/requirements_py3.11.txt index 6969bf9dd..86958ada2 100644 --- a/install/requirements_py3.11.txt +++ b/install/requirements_py3.11.txt @@ -23,3 +23,4 @@ pydantic type[VectorDB]: # noqa: PLR0911, PLR0912, C901 @@ -157,6 +158,11 @@ def init_cls(self) -> type[VectorDB]: # noqa: PLR0911, PLR0912, C901 from .test.test import Test return Test + + if self == DB.Vespa: + from .vespa.vespa import Vespa + + return Vespa msg = f"Unknown DB: {self.name}" raise ValueError(msg) @@ -273,6 +279,12 @@ def config_cls(self) -> type[DBConfig]: # noqa: PLR0911, PLR0912, C901 from .test.config import TestConfig return TestConfig + + if self == DB.Vespa: + from .vespa.config import VespaConfig + + return VespaConfig + msg = f"Unknown DB: {self.name}" raise ValueError(msg) @@ -365,6 +377,11 @@ def case_config_cls( # noqa: PLR0911 from .tidb.config import TiDBIndexConfig return TiDBIndexConfig + + if self == DB.Vespa: + from .vespa.config import VespaHNSWConfig + + return VespaHNSWConfig # DB.Pinecone, DB.Chroma, DB.Redis return EmptyDBCaseConfig diff --git a/vectordb_bench/backend/clients/vespa/cli.py b/vectordb_bench/backend/clients/vespa/cli.py new file mode 100644 index 000000000..616d46067 --- /dev/null +++ b/vectordb_bench/backend/clients/vespa/cli.py @@ -0,0 +1,47 @@ +from typing import Annotated, Unpack + +import click +from pydantic import SecretStr + +from vectordb_bench.backend.clients import DB +from vectordb_bench.cli.cli import ( + CommonTypedDict, + HNSWFlavor1, + cli, + click_parameter_decorators_from_typed_dict, + run, +) + + +class VespaTypedDict(CommonTypedDict, HNSWFlavor1): + uri: Annotated[ + str, + click.option("--uri", "-u", type=str, help="uri connection string", default="http://127.0.0.1"), + ] + port: Annotated[ + int, + click.option("--port", "-p", type=int, help="connection port", default=8080), + ] + quantization: Annotated[ + str, click.option("--quantization", type=click.Choice(["none", "binary"], case_sensitive=False), default="none") + ] + + +@cli.command() +@click_parameter_decorators_from_typed_dict(VespaTypedDict) +def Vespa(**params: Unpack[VespaTypedDict]): + from .config import VespaConfig, VespaHNSWConfig + + case_params = { + "quantization_type": params["quantization"], + "M": params["m"], + "efConstruction": params["ef_construction"], + "ef": params["ef_search"], + } + + run( + db=DB.Vespa, + db_config=VespaConfig(url=SecretStr(params["uri"]), port=params["port"]), + db_case_config=VespaHNSWConfig(**{k: v for k, v in case_params.items() if v}), + **params, + ) diff --git a/vectordb_bench/backend/clients/vespa/config.py b/vectordb_bench/backend/clients/vespa/config.py new file mode 100644 index 000000000..3d4a1deaf --- /dev/null +++ b/vectordb_bench/backend/clients/vespa/config.py @@ -0,0 +1,51 @@ +from typing import Literal, TypeAlias + +from pydantic import BaseModel, SecretStr + +from ..api import DBCaseConfig, DBConfig, MetricType + +VespaMetric: TypeAlias = Literal["euclidean", "angular", "dotproduct", "prenormalized-angular", "hamming", "geodegrees"] + +VespaQuantizationType: TypeAlias = Literal["none", "binary"] + + +class VespaConfig(DBConfig): + url: SecretStr = "http://127.0.0.1" + port: int = 8080 + + def to_dict(self): + return { + "url": self.url.get_secret_value(), + "port": self.port, + } + + +class VespaHNSWConfig(BaseModel, DBCaseConfig): + metric_type: MetricType = MetricType.COSINE + quantization_type: VespaQuantizationType = "none" + M: int = 16 + efConstruction: int = 200 + ef: int = 100 + + def index_param(self) -> dict: + return { + "distance_metric": self.parse_metric(self.metric_type), + "max_links_per_node": self.M, + "neighbors_to_explore_at_insert": self.efConstruction, + } + + def search_param(self) -> dict: + return {} + + def parse_metric(self, metric_type: MetricType) -> VespaMetric: + match metric_type: + case MetricType.COSINE: + return "angular" + case MetricType.L2: + return "euclidean" + case MetricType.DP | MetricType.IP: + return "dotproduct" + case MetricType.HAMMING: + return "hamming" + case _: + raise NotImplementedError diff --git a/vectordb_bench/backend/clients/vespa/util.py b/vectordb_bench/backend/clients/vespa/util.py new file mode 100644 index 000000000..97d03d644 --- /dev/null +++ b/vectordb_bench/backend/clients/vespa/util.py @@ -0,0 +1,16 @@ +"""Utility functions for supporting binary quantization + +From https://docs.vespa.ai/en/binarizing-vectors.html#appendix-conversion-to-int8 +""" +import numpy as np + + +def binarize_tensor(tensor: list[float]) -> list[int]: + """ + Binarize a floating-point list by thresholding at zero + and packing the bits into bytes. + """ + tensor = np.array(tensor) + return ( + np.packbits(np.where(tensor > 0, 1, 0), axis=0).astype(np.int8).tolist() + ) diff --git a/vectordb_bench/backend/clients/vespa/vespa.py b/vectordb_bench/backend/clients/vespa/vespa.py new file mode 100644 index 000000000..19bb3f5a4 --- /dev/null +++ b/vectordb_bench/backend/clients/vespa/vespa.py @@ -0,0 +1,254 @@ +import datetime +import logging +import math +from collections.abc import Generator +from contextlib import contextmanager + +from vespa import application + +from ..api import VectorDB +from .config import VespaHNSWConfig +from . import util + +log = logging.getLogger(__name__) + + +class Vespa(VectorDB): + def __init__( + self, + dim: int, + db_config: dict[str, str], + db_case_config: VespaHNSWConfig | None = None, + collection_name: str = "VectorDBBenchCollection", + drop_old: bool = False, + **kwargs, + ) -> None: + self.dim = dim + self.db_config = db_config + self.case_config = db_case_config or VespaHNSWConfig() + self.schema_name = collection_name + + client = self.deploy_http() + client.wait_for_application_up() + + if drop_old: + try: + client.delete_all_docs("vectordbbench_content", self.schema_name) + except Exception: + drop_old = False + log.exception(f"Vespa client drop_old schema: {self.schema_name}") + + @contextmanager + def init(self) -> Generator[None, None, None]: + """create and destory connections to database. + Why contextmanager: + + In multiprocessing search tasks, vectordbbench might init + totally hundreds of thousands of connections with DB server. + + Too many connections may drain local FDs or server connection resources. + If the DB client doesn't have `close()` method, just set the object to None. + + Examples: + >>> with self.init(): + >>> self.insert_embeddings() + """ + self.client = application.Vespa(self.db_config["url"], port=self.db_config["port"]) + yield + self.client = None + + def need_normalize_cosine(self) -> bool: + """Wheather this database need to normalize dataset to support COSINE""" + return False + + def insert_embeddings( + self, + embeddings: list[list[float]], + metadata: list[int], + **kwargs, + ) -> tuple[int, Exception | None]: + """Insert the embeddings to the vector database. The default number of embeddings for + each insert_embeddings is 5000. + + Args: + embeddings(list[list[float]]): list of embedding to add to the vector database. + metadatas(list[int]): metadata associated with the embeddings, for filtering. + **kwargs(Any): vector database specific parameters. + + Returns: + int: inserted data count + """ + assert self.client is not None + + data = ({"id": str(i), "fields": {"id": i, "embedding": e}} for i, e in zip(metadata, embeddings, strict=True)) + self.client.feed_iterable(data, self.schema_name) + return len(embeddings), None + + def search_embedding( + self, + query: list[float], + k: int = 100, + filters: dict | None = None, + ) -> list[int]: + """Get k most similar embeddings to query vector. + + Args: + query(list[float]): query embedding to look up documents similar to. + k(int): Number of most similar embeddings to return. Defaults to 100. + filters(dict, optional): filtering expression to filter the data while searching. + + Returns: + list[int]: list of k most similar embeddings IDs to the query embedding. + """ + assert self.client is not None + + ef = self.case_config.ef + extra_ef = max(0, ef - k) + embedding_field = "embedding" if self.case_config.quantization_type == "none" else "embedding_binary" + + yql = ( + f"select id from {self.schema_name} where " # noqa: S608 + f"{{targetHits: {k}, hnsw.exploreAdditionalHits: {extra_ef}}}" + f"nearestNeighbor({embedding_field}, query_embedding)" + ) + + if filters: + id_filter = filters.get("id") + yql += f" and id >= {id_filter}" + + query_embedding = ( + query + if self.case_config.quantization_type == "none" + else util.binarize_tensor(query) + ) + + ranking = self.case_config.quantization_type + + result = self.client.query({"yql": yql, "input.query(query_embedding)": query_embedding, "ranking": ranking}) + result_ids = [child["fields"]["id"] for child in result.get_json()["root"]["children"]] + return result_ids + + def optimize(self, data_size: int | None = None): + """optimize will be called between insertion and search in performance cases. + + Should be blocked until the vectorDB is ready to be tested on + heavy performance cases. + + Time(insert the dataset) + Time(optimize) will be recorded as "load_duration" metric + Optimize's execution time is limited, the limited time is based on cases. + """ + + @property + def application_package(self): + if getattr(self, "_application_package", None) is None: + self._application_package = self._create_application_package() + return self._application_package + + def _create_application_package(self): + from vespa.package import ( + HNSW, + ApplicationPackage, + Document, + Field, + RankProfile, + Schema, + Validation, + ValidationID, + ) + + fields = [ + Field( + "id", + "int", + indexing=["summary", "attribute"], + ), + Field( + "embedding", + f"tensor(x[{self.dim}])", + indexing=["summary", "attribute", "index"], + ann=HNSW(**self.case_config.index_param()), + ), + ] + + if self.case_config.quantization_type == "binary": + fields.append( + Field( + "embedding_binary", + f"tensor(x[{math.ceil(self.dim / 8)}])", + indexing=[ + "input embedding", + # convert 32 bit float to 1 bit + "binarize", + # pack 8 bits into one int8 + "pack_bits", + "summary", + "attribute", + "index", + ], + ann=HNSW(**{**self.case_config.index_param(), "distance_metric": "hamming"}), + is_document_field=False, + ) + ) + + tomorrow = datetime.date.today() + datetime.timedelta(days=1) + + return ApplicationPackage( + "vectordbbench", + [ + Schema( + self.schema_name, + Document( + fields, + ), + rank_profiles=[ + RankProfile( + name="none", + first_phase="", + inherits="default", + inputs=[("query(query_embedding)", f"tensor(x[{self.dim}])")], + ), + RankProfile( + name="binary", + first_phase="", + inherits="default", + inputs=[("query(query_embedding)", f"tensor(x[{math.ceil(self.dim / 8)}])")], + ), + ], + ) + ], + validations=[ + Validation(ValidationID.tensorTypeChange, until=tomorrow), + Validation(ValidationID.fieldTypeChange, until=tomorrow), + ], + ) + + def deploy_http(self) -> application.Vespa: + """ + Deploy a Vespa application package via HTTP REST API. + + Returns: + application.Vespa: The deployed Vespa application instance + """ + import requests + + url = self.db_config["url"] + ":19071/application/v2/tenant/default/prepareandactivate" + package_data = self.application_package.to_zip() + headers = {"Content-Type": "application/zip"} + + try: + response = requests.post(url=url, data=package_data, headers=headers, timeout=10) + + response.raise_for_status() + result = response.json() + return application.Vespa( + url=self.db_config["url"], + port=self.db_config["port"], + deployment_message=result.get("message"), + application_package=self.application_package, + ) + + except requests.exceptions.RequestException as e: + error_msg = f"Failed to deploy Vespa application: {e!s}" + if hasattr(e, "response") and e.response is not None: + error_msg += f" - Response: {e.response.text}" + raise RuntimeError(error_msg) from e diff --git a/vectordb_bench/cli/vectordbbench.py b/vectordb_bench/cli/vectordbbench.py index 7f5e24fc2..7469ec9d5 100644 --- a/vectordb_bench/cli/vectordbbench.py +++ b/vectordb_bench/cli/vectordbbench.py @@ -9,9 +9,10 @@ from ..backend.clients.pgvectorscale.cli import PgVectorScaleDiskAnn from ..backend.clients.redis.cli import Redis from ..backend.clients.test.cli import Test +from ..backend.clients.tidb.cli import TiDB +from ..backend.clients.vespa.cli import Vespa from ..backend.clients.weaviate_cloud.cli import Weaviate from ..backend.clients.zilliz_cloud.cli import ZillizAutoIndex -from ..backend.clients.tidb.cli import TiDB from ..backend.clients.clickhouse.cli import Clickhouse from .cli import cli @@ -31,6 +32,7 @@ cli.add_command(MariaDBHNSW) cli.add_command(TiDB) cli.add_command(Clickhouse) +cli.add_command(Vespa) if __name__ == "__main__": diff --git a/vectordb_bench/frontend/config/dbCaseConfigs.py b/vectordb_bench/frontend/config/dbCaseConfigs.py index 0ab3a932b..e85b42ff3 100644 --- a/vectordb_bench/frontend/config/dbCaseConfigs.py +++ b/vectordb_bench/frontend/config/dbCaseConfigs.py @@ -1137,6 +1137,50 @@ class CaseConfigInput(BaseModel): ) +CaseConfigParamInput_M_Vespa = CaseConfigInput( + label=CaseConfigParamType.M, + inputType=InputType.Number, + inputConfig={ + "min": 4, + "max": 64, + "value": 16, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.HNSW.value, +) + +CaseConfigParamInput_IndexType_Vespa = CaseConfigInput( + label=CaseConfigParamType.IndexType, + inputType=InputType.Option, + inputConfig={ + "options": [ + IndexType.HNSW.value, + ], + }, +) + +CaseConfigParamInput_QuantizationType_Vespa = CaseConfigInput( + label=CaseConfigParamType.quantizationType, + inputType=InputType.Option, + inputConfig={ + "options": [ + "none", + "binary" + ], + }, +) + +CaseConfigParamInput_EFConstruction_Vespa = CaseConfigInput( + label=CaseConfigParamType.EFConstruction, + inputType=InputType.Number, + inputConfig={ + "min": 8, + "max": 512, + "value": 200, + }, + isDisplayed=lambda config: config[CaseConfigParamType.IndexType] == IndexType.HNSW.value, +) + + MilvusLoadConfig = [ CaseConfigParamInput_IndexType, CaseConfigParamInput_M, @@ -1344,6 +1388,15 @@ class CaseConfigInput(BaseModel): CaseConfigParamInput_EFSearch_MariaDB, ] +VespaLoadingConfig = [ + CaseConfigParamInput_IndexType_Vespa, + CaseConfigParamInput_QuantizationType_Vespa, + CaseConfigParamInput_M_Vespa, + CaseConfigParamInput_EF_Milvus, + CaseConfigParamInput_EFConstruction_Vespa, +] +VespaPerformanceConfig = VespaLoadingConfig + CASE_CONFIG_MAP = { DB.Milvus: { CaseLabel.Load: MilvusLoadConfig, @@ -1400,4 +1453,8 @@ class CaseConfigInput(BaseModel): CaseLabel.Load: MariaDBLoadingConfig, CaseLabel.Performance: MariaDBPerformanceConfig, }, + DB.Vespa: { + CaseLabel.Load: VespaLoadingConfig, + CaseLabel.Performance: VespaPerformanceConfig, + }, } diff --git a/vectordb_bench/frontend/config/styles.py b/vectordb_bench/frontend/config/styles.py index 57456722f..4418c19da 100644 --- a/vectordb_bench/frontend/config/styles.py +++ b/vectordb_bench/frontend/config/styles.py @@ -48,6 +48,7 @@ def getPatternShape(i): DB.Chroma: "https://assets.zilliz.com/chroma_ceb3f06ed7.png", DB.AWSOpenSearch: "https://assets.zilliz.com/opensearch_1eee37584e.jpeg", DB.TiDB: "https://img2.pingcap.com/forms/3/d/3d7fd5f9767323d6f037795704211ac44b4923d6.png", + DB.Vespa: "https://vespa.ai/vespa-content/uploads/2025/01/Vespa-symbol-green-rgb.png.webp", } # RedisCloud color: #0D6EFD @@ -63,4 +64,5 @@ def getPatternShape(i): DB.Redis.value: "#0D6EFD", DB.AWSOpenSearch.value: "#0DCAF0", DB.TiDB.value: "#0D6EFD", + DB.Vespa.value: "#61d790" } From 8432f6f1b45fe74d03577e7d84ac790db76f6a61 Mon Sep 17 00:00:00 2001 From: "min.tian" Date: Mon, 14 Apr 2025 02:49:50 +0000 Subject: [PATCH 152/327] remove redundant empty_field config check for qdrant and tidb Signed-off-by: min.tian --- .../backend/clients/clickhouse/config.py | 18 +++++++++++------- vectordb_bench/backend/clients/mariadb/cli.py | 5 ++--- .../backend/clients/mariadb/config.py | 13 ++++++++----- .../backend/clients/qdrant_cloud/config.py | 10 +--------- vectordb_bench/backend/clients/tidb/config.py | 15 ++++++--------- 5 files changed, 28 insertions(+), 33 deletions(-) diff --git a/vectordb_bench/backend/clients/clickhouse/config.py b/vectordb_bench/backend/clients/clickhouse/config.py index 7ce0919ea..fad446049 100644 --- a/vectordb_bench/backend/clients/clickhouse/config.py +++ b/vectordb_bench/backend/clients/clickhouse/config.py @@ -1,9 +1,10 @@ -from typing import TypedDict from pydantic import BaseModel, SecretStr -from ..api import DBConfig, DBCaseConfig, MetricType, IndexType + +from ..api import DBCaseConfig, DBConfig, IndexType, MetricType + class ClickhouseConfig(DBConfig): - user_name: str = "clickhouse" + user_name: str = "clickhouse" password: SecretStr host: str = "localhost" port: int = 8123 @@ -16,7 +17,7 @@ def to_dict(self) -> dict: "port": self.port, "dbname": self.db_name, "user": self.user_name, - "password": pwd_str + "password": pwd_str, } @@ -32,8 +33,11 @@ def parse_metric(self) -> str: def parse_metric_str(self) -> str: if self.metric_type == MetricType.L2: return "L2Distance" - elif self.metric_type == MetricType.COSINE: + if self.metric_type == MetricType.COSINE: return "cosineDistance" + msg = f"Not Support for {self.metric_type}" + raise RuntimeError(msg) + return None class ClickhouseHNSWConfig(ClickhouseIndexConfig, DBCaseConfig): @@ -51,6 +55,6 @@ def index_param(self) -> dict: def search_param(self) -> dict: return { - "metric_type": self.parse_metric_str(), + "met˝ric_type": self.parse_metric_str(), "params": {"ef": self.ef}, - } \ No newline at end of file + } diff --git a/vectordb_bench/backend/clients/mariadb/cli.py b/vectordb_bench/backend/clients/mariadb/cli.py index c5439f37d..17717c38d 100644 --- a/vectordb_bench/backend/clients/mariadb/cli.py +++ b/vectordb_bench/backend/clients/mariadb/cli.py @@ -1,17 +1,16 @@ from typing import Annotated, Optional, Unpack import click -import os from pydantic import SecretStr +from vectordb_bench.backend.clients import DB + from ....cli.cli import ( CommonTypedDict, - HNSWFlavor1, cli, click_parameter_decorators_from_typed_dict, run, ) -from vectordb_bench.backend.clients import DB class MariaDBTypedDict(CommonTypedDict): diff --git a/vectordb_bench/backend/clients/mariadb/config.py b/vectordb_bench/backend/clients/mariadb/config.py index c7b2cd5fe..50d0b55c5 100644 --- a/vectordb_bench/backend/clients/mariadb/config.py +++ b/vectordb_bench/backend/clients/mariadb/config.py @@ -1,6 +1,9 @@ -from pydantic import SecretStr, BaseModel from typing import TypedDict -from ..api import DBConfig, DBCaseConfig, MetricType, IndexType + +from pydantic import BaseModel, SecretStr + +from ..api import DBCaseConfig, DBConfig, IndexType, MetricType + class MariaDBConfigDict(TypedDict): """These keys will be directly used as kwargs in mariadb connection string, @@ -36,10 +39,10 @@ class MariaDBIndexConfig(BaseModel): def parse_metric(self) -> str: if self.metric_type == MetricType.L2: return "euclidean" - elif self.metric_type == MetricType.COSINE: + if self.metric_type == MetricType.COSINE: return "cosine" - else: - raise ValueError(f"Metric type {self.metric_type} is not supported!") + msg = f"Metric type {self.metric_type} is not supported!" + raise ValueError(msg) class MariaDBHNSWConfig(MariaDBIndexConfig, DBCaseConfig): M: int | None diff --git a/vectordb_bench/backend/clients/qdrant_cloud/config.py b/vectordb_bench/backend/clients/qdrant_cloud/config.py index c1d6882c0..d4e27cb3c 100644 --- a/vectordb_bench/backend/clients/qdrant_cloud/config.py +++ b/vectordb_bench/backend/clients/qdrant_cloud/config.py @@ -1,4 +1,4 @@ -from pydantic import BaseModel, SecretStr, validator +from pydantic import BaseModel, SecretStr from ..api import DBCaseConfig, DBConfig, MetricType @@ -20,14 +20,6 @@ def to_dict(self) -> dict: "url": self.url.get_secret_value(), } - @validator("*") - def not_empty_field(cls, v: any, field: any): - if field.name in ["api_key", "db_label"]: - return v - if isinstance(v, str | SecretStr) and len(v) == 0: - raise ValueError("Empty string!") - return v - class QdrantIndexConfig(BaseModel, DBCaseConfig): metric_type: MetricType | None = None diff --git a/vectordb_bench/backend/clients/tidb/config.py b/vectordb_bench/backend/clients/tidb/config.py index 213a18bc5..693551045 100644 --- a/vectordb_bench/backend/clients/tidb/config.py +++ b/vectordb_bench/backend/clients/tidb/config.py @@ -1,5 +1,6 @@ -from pydantic import SecretStr, BaseModel, validator -from ..api import DBConfig, DBCaseConfig, MetricType +from pydantic import BaseModel, SecretStr + +from ..api import DBCaseConfig, DBConfig, MetricType class TiDBConfig(DBConfig): @@ -10,10 +11,6 @@ class TiDBConfig(DBConfig): db_name: str = "test" ssl: bool = False - @validator("*") - def not_empty_field(cls, v: any, field: any): - return v - def to_dict(self) -> dict: pwd_str = self.password.get_secret_value() return { @@ -33,10 +30,10 @@ class TiDBIndexConfig(BaseModel, DBCaseConfig): def get_metric_fn(self) -> str: if self.metric_type == MetricType.L2: return "vec_l2_distance" - elif self.metric_type == MetricType.COSINE: + if self.metric_type == MetricType.COSINE: return "vec_cosine_distance" - else: - raise ValueError(f"Unsupported metric type: {self.metric_type}") + msg = f"Unsupported metric type: {self.metric_type}" + raise ValueError(msg) def index_param(self) -> dict: return { From 0a96299dae7885d783b3e5a57786065751938723 Mon Sep 17 00:00:00 2001 From: "min.tian" Date: Mon, 14 Apr 2025 03:39:01 +0000 Subject: [PATCH 153/327] reformat all Signed-off-by: min.tian --- vectordb_bench/backend/clients/__init__.py | 9 +- vectordb_bench/backend/clients/api.py | 2 +- .../backend/clients/chroma/chroma.py | 4 +- .../backend/clients/clickhouse/cli.py | 2 +- .../backend/clients/clickhouse/clickhouse.py | 79 +++++++------ .../clients/elastic_cloud/elastic_cloud.py | 2 +- vectordb_bench/backend/clients/mariadb/cli.py | 100 +++++++++------- .../backend/clients/mariadb/config.py | 7 +- .../backend/clients/mariadb/mariadb.py | 110 +++++++++--------- vectordb_bench/backend/clients/milvus/cli.py | 2 + .../backend/clients/milvus/config.py | 1 - .../backend/clients/milvus/milvus.py | 2 +- .../backend/clients/pgvector/cli.py | 3 +- .../backend/clients/pinecone/pinecone.py | 2 +- .../clients/qdrant_cloud/qdrant_cloud.py | 2 +- vectordb_bench/backend/clients/tidb/tidb.py | 35 +++--- vectordb_bench/backend/clients/vespa/util.py | 5 +- vectordb_bench/backend/clients/vespa/vespa.py | 11 +- .../clients/weaviate_cloud/weaviate_cloud.py | 2 +- vectordb_bench/cli/cli.py | 37 +++--- vectordb_bench/cli/vectordbbench.py | 2 +- .../frontend/config/dbCaseConfigs.py | 16 +-- vectordb_bench/frontend/config/styles.py | 2 +- vectordb_bench/models.py | 11 +- 24 files changed, 226 insertions(+), 222 deletions(-) diff --git a/vectordb_bench/backend/clients/__init__.py b/vectordb_bench/backend/clients/__init__.py index 732f427a7..cfef0283f 100644 --- a/vectordb_bench/backend/clients/__init__.py +++ b/vectordb_bench/backend/clients/__init__.py @@ -158,7 +158,7 @@ def init_cls(self) -> type[VectorDB]: # noqa: PLR0911, PLR0912, C901 from .test.test import Test return Test - + if self == DB.Vespa: from .vespa.vespa import Vespa @@ -279,17 +279,16 @@ def config_cls(self) -> type[DBConfig]: # noqa: PLR0911, PLR0912, C901 from .test.config import TestConfig return TestConfig - + if self == DB.Vespa: from .vespa.config import VespaConfig return VespaConfig - msg = f"Unknown DB: {self.name}" raise ValueError(msg) - def case_config_cls( # noqa: PLR0911 + def case_config_cls( # noqa: C901, PLR0911, PLR0912 self, index_type: IndexType | None = None, ) -> type[DBCaseConfig]: @@ -377,7 +376,7 @@ def case_config_cls( # noqa: PLR0911 from .tidb.config import TiDBIndexConfig return TiDBIndexConfig - + if self == DB.Vespa: from .vespa.config import VespaHNSWConfig diff --git a/vectordb_bench/backend/clients/api.py b/vectordb_bench/backend/clients/api.py index e498ab077..ce2b05650 100644 --- a/vectordb_bench/backend/clients/api.py +++ b/vectordb_bench/backend/clients/api.py @@ -162,7 +162,7 @@ def insert_embeddings( embeddings: list[list[float]], metadata: list[int], **kwargs, - ) -> (int, Exception): + ) -> tuple[int, Exception]: """Insert the embeddings to the vector database. The default number of embeddings for each insert_embeddings is 5000. diff --git a/vectordb_bench/backend/clients/chroma/chroma.py b/vectordb_bench/backend/clients/chroma/chroma.py index 76c810263..26a810065 100644 --- a/vectordb_bench/backend/clients/chroma/chroma.py +++ b/vectordb_bench/backend/clients/chroma/chroma.py @@ -65,7 +65,7 @@ def insert_embeddings( embeddings: list[list[float]], metadata: list[int], **kwargs: Any, - ) -> (int, Exception): + ) -> tuple[int, Exception]: """Insert embeddings into the database. Args: @@ -74,7 +74,7 @@ def insert_embeddings( kwargs: other arguments Returns: - (int, Exception): number of embeddings inserted and exception if any + tuple[int, Exception]: number of embeddings inserted and exception if any """ ids = [str(i) for i in metadata] metadata = [{"id": int(i)} for i in metadata] diff --git a/vectordb_bench/backend/clients/clickhouse/cli.py b/vectordb_bench/backend/clients/clickhouse/cli.py index e454f5a75..6fc1a84d7 100644 --- a/vectordb_bench/backend/clients/clickhouse/cli.py +++ b/vectordb_bench/backend/clients/clickhouse/cli.py @@ -18,7 +18,7 @@ class ClickhouseTypedDict(TypedDict): password: Annotated[str, click.option("--password", type=str, help="DB password")] host: Annotated[str, click.option("--host", type=str, help="DB host", required=True)] port: Annotated[int, click.option("--port", type=int, default=8123, help="DB Port")] - user: Annotated[int, click.option("--user", type=str, default='clickhouse', help="DB user")] + user: Annotated[int, click.option("--user", type=str, default="clickhouse", help="DB user")] ssl: Annotated[ bool, click.option( diff --git a/vectordb_bench/backend/clients/clickhouse/clickhouse.py b/vectordb_bench/backend/clients/clickhouse/clickhouse.py index e5339170b..e241cdb8d 100644 --- a/vectordb_bench/backend/clients/clickhouse/clickhouse.py +++ b/vectordb_bench/backend/clients/clickhouse/clickhouse.py @@ -1,18 +1,19 @@ """Wrapper around the Clickhouse vector database over VectorDB""" -import io import logging from contextlib import contextmanager from typing import Any + import clickhouse_connect -import numpy as np -from ..api import VectorDB, DBCaseConfig +from ..api import DBCaseConfig, VectorDB log = logging.getLogger(__name__) + class Clickhouse(VectorDB): """Use SQLAlchemy instructions""" + def __init__( self, dim: int, @@ -32,12 +33,13 @@ def __init__( self._vector_field = "embedding" # construct basic units - self.conn = clickhouse_connect.get_client( - host=self.db_config["host"], - port=self.db_config["port"], - username=self.db_config["user"], - password=self.db_config["password"], - database=self.db_config["dbname"]) + self.conn = clickhouse_connect.get_client( + host=self.db_config["host"], + port=self.db_config["port"], + username=self.db_config["user"], + password=self.db_config["password"], + database=self.db_config["dbname"], + ) if drop_old: log.info(f"Clickhouse client drop table : {self.table_name}") @@ -48,7 +50,7 @@ def __init__( self.conn = None @contextmanager - def init(self) -> None: + def init(self): """ Examples: >>> with self.init(): @@ -56,12 +58,13 @@ def init(self) -> None: >>> self.search_embedding() """ - self.conn = clickhouse_connect.get_client( - host=self.db_config["host"], - port=self.db_config["port"], - username=self.db_config["user"], - password=self.db_config["password"], - database=self.db_config["dbname"]) + self.conn = clickhouse_connect.get_client( + host=self.db_config["host"], + port=self.db_config["port"], + username=self.db_config["user"], + password=self.db_config["password"], + database=self.db_config["dbname"], + ) try: yield @@ -85,9 +88,7 @@ def _create_table(self, dim: int): ) except Exception as e: - log.warning( - f"Failed to create Clickhouse table: {self.table_name} error: {e}" - ) + log.warning(f"Failed to create Clickhouse table: {self.table_name} error: {e}") raise e from None def ready_to_load(self): @@ -104,16 +105,20 @@ def insert_embeddings( embeddings: list[list[float]], metadata: list[int], **kwargs: Any, - ) -> (int, Exception): + ) -> tuple[int, Exception]: assert self.conn is not None, "Connection is not initialized" try: # do not iterate for bulk insert items = [metadata, embeddings] - self.conn.insert(table=self.table_name, data=items, - column_names=['id', 'embedding'], column_type_names=['UInt32', 'Array(Float64)'], - column_oriented=True) + self.conn.insert( + table=self.table_name, + data=items, + column_names=["id", "embedding"], + column_type_names=["UInt32", "Array(Float64)"], + column_oriented=True, + ) return len(metadata), None except Exception as e: log.warning(f"Failed to insert data into Clickhouse table ({self.table_name}), error: {e}") @@ -128,22 +133,24 @@ def search_embedding( ) -> list[int]: assert self.conn is not None, "Connection is not initialized" - index_param = self.case_config.index_param() + index_param = self.case_config.index_param() # noqa: F841 search_param = self.case_config.search_param() if filters: gt = filters.get("id") - filterSql = (f'SELECT id, {search_param["metric_type"]}(embedding,{query}) AS score ' - f'FROM {self.db_config["dbname"]}.{self.table_name} ' - f'WHERE id > {gt} ' - f'ORDER BY score LIMIT {k};' - ) - result = self.conn.query(filterSql).result_rows + filter_sql = ( + f'SELECT id, {search_param["metric_type"]}(embedding,{query}) AS score ' # noqa: S608 + f'FROM {self.db_config["dbname"]}.{self.table_name} ' + f"WHERE id > {gt} " + f"ORDER BY score LIMIT {k};" + ) + result = self.conn.query(filter_sql).result_rows return [int(row[0]) for row in result] - else: - selectSql = (f'SELECT id, {search_param["metric_type"]}(embedding,{query}) AS score ' - f'FROM {self.db_config["dbname"]}.{self.table_name} ' - f'ORDER BY score LIMIT {k};' - ) - result = self.conn.query(selectSql).result_rows + else: # noqa: RET505 + select_sql = ( + f'SELECT id, {search_param["metric_type"]}(embedding,{query}) AS score ' # noqa: S608 + f'FROM {self.db_config["dbname"]}.{self.table_name} ' + f"ORDER BY score LIMIT {k};" + ) + result = self.conn.query(select_sql).result_rows return [int(row[0]) for row in result] diff --git a/vectordb_bench/backend/clients/elastic_cloud/elastic_cloud.py b/vectordb_bench/backend/clients/elastic_cloud/elastic_cloud.py index ea038c587..7d201729f 100644 --- a/vectordb_bench/backend/clients/elastic_cloud/elastic_cloud.py +++ b/vectordb_bench/backend/clients/elastic_cloud/elastic_cloud.py @@ -81,7 +81,7 @@ def insert_embeddings( embeddings: Iterable[list[float]], metadata: list[int], **kwargs, - ) -> (int, Exception): + ) -> tuple[int, Exception]: """Insert the embeddings to the elasticsearch.""" assert self.client is not None, "should self.init() first" diff --git a/vectordb_bench/backend/clients/mariadb/cli.py b/vectordb_bench/backend/clients/mariadb/cli.py index 17717c38d..969247271 100644 --- a/vectordb_bench/backend/clients/mariadb/cli.py +++ b/vectordb_bench/backend/clients/mariadb/cli.py @@ -1,4 +1,4 @@ -from typing import Annotated, Optional, Unpack +from typing import Annotated, Unpack import click from pydantic import SecretStr @@ -15,68 +15,84 @@ class MariaDBTypedDict(CommonTypedDict): user_name: Annotated[ - str, click.option("--username", - type=str, - help="Username", - required=True, - ), + str, + click.option( + "--username", + type=str, + help="Username", + required=True, + ), ] password: Annotated[ - str, click.option("--password", - type=str, - help="Password", - required=True, - ), + str, + click.option( + "--password", + type=str, + help="Password", + required=True, + ), ] host: Annotated[ - str, click.option("--host", - type=str, - help="Db host", - default="127.0.0.1", - ), + str, + click.option( + "--host", + type=str, + help="Db host", + default="127.0.0.1", + ), ] port: Annotated[ - int, click.option("--port", - type=int, - default=3306, - help="Db Port", - ), + int, + click.option( + "--port", + type=int, + default=3306, + help="Db Port", + ), ] storage_engine: Annotated[ - int, click.option("--storage-engine", - type=click.Choice(["InnoDB", "MyISAM"]), - help="DB storage engine", - required=True, - ), + int, + click.option( + "--storage-engine", + type=click.Choice(["InnoDB", "MyISAM"]), + help="DB storage engine", + required=True, + ), ] + class MariaDBHNSWTypedDict(MariaDBTypedDict): - ... m: Annotated[ - Optional[int], click.option("--m", - type=int, - help="M parameter in MHNSW vector indexing", - required=False, - ), + int | None, + click.option( + "--m", + type=int, + help="M parameter in MHNSW vector indexing", + required=False, + ), ] ef_search: Annotated[ - Optional[int], click.option("--ef-search", - type=int, - help="MariaDB system variable mhnsw_min_limit", - required=False, - ), + int | None, + click.option( + "--ef-search", + type=int, + help="MariaDB system variable mhnsw_min_limit", + required=False, + ), ] max_cache_size: Annotated[ - Optional[int], click.option("--max-cache-size", - type=int, - help="MariaDB system variable mhnsw_max_cache_size", - required=False, - ), + int | None, + click.option( + "--max-cache-size", + type=int, + help="MariaDB system variable mhnsw_max_cache_size", + required=False, + ), ] diff --git a/vectordb_bench/backend/clients/mariadb/config.py b/vectordb_bench/backend/clients/mariadb/config.py index 50d0b55c5..d183adc76 100644 --- a/vectordb_bench/backend/clients/mariadb/config.py +++ b/vectordb_bench/backend/clients/mariadb/config.py @@ -7,7 +7,7 @@ class MariaDBConfigDict(TypedDict): """These keys will be directly used as kwargs in mariadb connection string, - so the names must match exactly mariadb API""" + so the names must match exactly mariadb API""" user: str password: str @@ -44,6 +44,7 @@ def parse_metric(self) -> str: msg = f"Metric type {self.metric_type} is not supported!" raise ValueError(msg) + class MariaDBHNSWConfig(MariaDBIndexConfig, DBCaseConfig): M: int | None ef_search: int | None @@ -68,7 +69,5 @@ def search_param(self) -> dict: _mariadb_case_config = { - IndexType.HNSW: MariaDBHNSWConfig, + IndexType.HNSW: MariaDBHNSWConfig, } - - diff --git a/vectordb_bench/backend/clients/mariadb/mariadb.py b/vectordb_bench/backend/clients/mariadb/mariadb.py index 42b621d9c..5ccddfe7a 100644 --- a/vectordb_bench/backend/clients/mariadb/mariadb.py +++ b/vectordb_bench/backend/clients/mariadb/mariadb.py @@ -1,27 +1,25 @@ -from ..api import VectorDB - import logging from contextlib import contextmanager -from typing import Any, Optional, Tuple -from ..api import VectorDB -from .config import MariaDBConfigDict, MariaDBIndexConfig -import numpy as np import mariadb +import numpy as np + +from ..api import VectorDB +from .config import MariaDBConfigDict, MariaDBIndexConfig log = logging.getLogger(__name__) + class MariaDB(VectorDB): def __init__( - self, - dim: int, - db_config: MariaDBConfigDict, - db_case_config: MariaDBIndexConfig, - collection_name: str = "vec_collection", - drop_old: bool = False, - **kwargs, - ): - + self, + dim: int, + db_config: MariaDBConfigDict, + db_case_config: MariaDBIndexConfig, + collection_name: str = "vec_collection", + drop_old: bool = False, + **kwargs, + ): self.name = "MariaDB" self.db_config = db_config self.case_config = db_case_config @@ -31,7 +29,7 @@ def __init__( # construct basic units self.conn, self.cursor = self._create_connection(**self.db_config) - + if drop_old: self._drop_db() self._create_db_table(dim) @@ -41,9 +39,8 @@ def __init__( self.cursor = None self.conn = None - @staticmethod - def _create_connection(**kwargs) -> Tuple[mariadb.Connection, mariadb.Cursor]: + def _create_connection(**kwargs) -> tuple[mariadb.Connection, mariadb.Cursor]: conn = mariadb.connect(**kwargs) cursor = conn.cursor() @@ -52,7 +49,6 @@ def _create_connection(**kwargs) -> Tuple[mariadb.Connection, mariadb.Cursor]: return conn, cursor - def _drop_db(self): assert self.conn is not None, "Connection is not initialized" assert self.cursor is not None, "Cursor is not initialized" @@ -77,24 +73,23 @@ def _create_db_table(self, dim: int): log.info(f"{self.name} client create table : {self.table_name}") self.cursor.execute(f"USE {self.db_name}") - self.cursor.execute(f""" + self.cursor.execute( + f""" CREATE TABLE {self.table_name} ( id INT PRIMARY KEY, v VECTOR({self.dim}) NOT NULL ) ENGINE={index_param["storage_engine"]} - """) + """ + ) self.cursor.execute("COMMIT") except Exception as e: - log.warning( - f"Failed to create table: {self.table_name} error: {e}" - ) + log.warning(f"Failed to create table: {self.table_name} error: {e}") raise e from None - @contextmanager - def init(self) -> None: - """ create and destory connections to database. + def init(self): + """create and destory connections to database. Examples: >>> with self.init(): @@ -109,15 +104,21 @@ def init(self) -> None: self.cursor.execute("SET GLOBAL max_allowed_packet = 1073741824") if index_param["index_type"] == "HNSW": - if index_param["max_cache_size"] != None: - self.cursor.execute(f"SET GLOBAL mhnsw_max_cache_size = {index_param["max_cache_size"]}") - if search_param["ef_search"] != None: - self.cursor.execute(f"SET mhnsw_ef_search = {search_param["ef_search"]}") + if index_param["max_cache_size"] is not None: + self.cursor.execute(f"SET GLOBAL mhnsw_max_cache_size = {index_param['max_cache_size']}") + if search_param["ef_search"] is not None: + self.cursor.execute(f"SET mhnsw_ef_search = {search_param['ef_search']}") self.cursor.execute("COMMIT") - self.insert_sql = f"INSERT INTO {self.db_name}.{self.table_name} (id, v) VALUES (%s, %s)" - self.select_sql = f"SELECT id FROM {self.db_name}.{self.table_name} ORDER by vec_distance_{search_param["metric_type"]}(v, %s) LIMIT %d" - self.select_sql_with_filter = f"SELECT id FROM {self.db_name}.{self.table_name} WHERE id >= %d ORDER by vec_distance_{search_param["metric_type"]}(v, %s) LIMIT %d" + self.insert_sql = f"INSERT INTO {self.db_name}.{self.table_name} (id, v) VALUES (%s, %s)" # noqa: S608 + self.select_sql = ( + f"SELECT id FROM {self.db_name}.{self.table_name}" # noqa: S608 + f"ORDER by vec_distance_{search_param['metric_type']}(v, %s) LIMIT %d" + ) + self.select_sql_with_filter = ( + f"SELECT id FROM {self.db_name}.{self.table_name} WHERE id >= %d " # noqa: S608 + f"ORDER by vec_distance_{search_param['metric_type']}(v, %s) LIMIT %d" + ) try: yield @@ -126,7 +127,6 @@ def init(self) -> None: self.conn.close() self.cursor = None self.conn = None - def ready_to_load(self) -> bool: pass @@ -139,33 +139,31 @@ def optimize(self) -> None: try: index_options = f"DISTANCE={index_param['metric_type']}" - if index_param["index_type"] == "HNSW" and index_param["M"] != None: + if index_param["index_type"] == "HNSW" and index_param["M"] is not None: index_options += f" M={index_param['M']}" - self.cursor.execute(f""" + self.cursor.execute( + f""" ALTER TABLE {self.db_name}.{self.table_name} ADD VECTOR KEY v(v) {index_options} - """) + """ + ) self.cursor.execute("COMMIT") except Exception as e: - log.warning( - f"Failed to create index: {self.table_name} error: {e}" - ) + log.warning(f"Failed to create index: {self.table_name} error: {e}") raise e from None - pass - @staticmethod - def vector_to_hex(v): - return np.array(v, 'float32').tobytes() + def vector_to_hex(v): # noqa: ANN001 + return np.array(v, "float32").tobytes() def insert_embeddings( self, embeddings: list[list[float]], metadata: list[int], - **kwargs: Any, - ) -> Tuple[int, Optional[Exception]]: + **kwargs, + ) -> tuple[int, Exception]: """Insert embeddings into the database. Should call self.init() first. """ @@ -178,7 +176,7 @@ def insert_embeddings( batch_data = [] for i, row in enumerate(metadata_arr): - batch_data.append((int(row), self.vector_to_hex(embeddings_arr[i]))); + batch_data.append((int(row), self.vector_to_hex(embeddings_arr[i]))) self.cursor.executemany(self.insert_sql, batch_data) self.cursor.execute("COMMIT") @@ -186,11 +184,8 @@ def insert_embeddings( return len(metadata), None except Exception as e: - log.warning( - f"Failed to insert data into Vector table ({self.table_name}), error: {e}" - ) + log.warning(f"Failed to insert data into Vector table ({self.table_name}), error: {e}") return 0, e - def search_embedding( self, @@ -198,17 +193,16 @@ def search_embedding( k: int = 100, filters: dict | None = None, timeout: int | None = None, - **kwargs: Any, - ) -> (list[int]): + **kwargs, + ) -> list[int]: assert self.conn is not None, "Connection is not initialized" assert self.cursor is not None, "Cursor is not initialized" - search_param = self.case_config.search_param() + search_param = self.case_config.search_param() # noqa: F841 if filters: - self.cursor.execute(self.select_sql_with_filter, (filters.get('id'), self.vector_to_hex(query), k)) + self.cursor.execute(self.select_sql_with_filter, (filters.get("id"), self.vector_to_hex(query), k)) else: self.cursor.execute(self.select_sql, (self.vector_to_hex(query), k)) - return [id for id, in self.cursor.fetchall()] - + return [id for (id,) in self.cursor.fetchall()] # noqa: A001 diff --git a/vectordb_bench/backend/clients/milvus/cli.py b/vectordb_bench/backend/clients/milvus/cli.py index 52524e785..24a61566f 100644 --- a/vectordb_bench/backend/clients/milvus/cli.py +++ b/vectordb_bench/backend/clients/milvus/cli.py @@ -194,6 +194,7 @@ def MilvusGPUIVFFlat(**parameters: Unpack[MilvusGPUIVFTypedDict]): **parameters, ) + class MilvusGPUBruteForceTypedDict(CommonTypedDict, MilvusTypedDict): metric_type: Annotated[ str, @@ -204,6 +205,7 @@ class MilvusGPUBruteForceTypedDict(CommonTypedDict, MilvusTypedDict): click.option("--limit", type=int, required=True, help="Top-k limit for search"), ] + @cli.command() @click_parameter_decorators_from_typed_dict(MilvusGPUBruteForceTypedDict) def MilvusGPUBruteForce(**parameters: Unpack[MilvusGPUBruteForceTypedDict]): diff --git a/vectordb_bench/backend/clients/milvus/config.py b/vectordb_bench/backend/clients/milvus/config.py index 1ff3bea5f..e3a3f9b19 100644 --- a/vectordb_bench/backend/clients/milvus/config.py +++ b/vectordb_bench/backend/clients/milvus/config.py @@ -215,7 +215,6 @@ def search_param(self) -> dict: } - class GPUIVFPQConfig(MilvusIndexConfig, DBCaseConfig): nlist: int = 1024 m: int = 0 diff --git a/vectordb_bench/backend/clients/milvus/milvus.py b/vectordb_bench/backend/clients/milvus/milvus.py index 4015eb1f3..c812698fe 100644 --- a/vectordb_bench/backend/clients/milvus/milvus.py +++ b/vectordb_bench/backend/clients/milvus/milvus.py @@ -155,7 +155,7 @@ def insert_embeddings( embeddings: Iterable[list[float]], metadata: list[int], **kwargs, - ) -> (int, Exception): + ) -> tuple[int, Exception]: """Insert embeddings into Milvus. should call self.init() first""" # use the first insert_embeddings to init collection assert self.col is not None diff --git a/vectordb_bench/backend/clients/pgvector/cli.py b/vectordb_bench/backend/clients/pgvector/cli.py index 1780af991..f8c138802 100644 --- a/vectordb_bench/backend/clients/pgvector/cli.py +++ b/vectordb_bench/backend/clients/pgvector/cli.py @@ -18,8 +18,7 @@ ) -# ruff: noqa -def set_default_quantized_fetch_limit(ctx: any, param: any, value: any): +def set_default_quantized_fetch_limit(ctx: any, param: any, value: any): # noqa: ARG001 if ctx.params.get("reranking") and value is None: # ef_search is the default value for quantized_fetch_limit as it's bound by ef_search. # 100 is default value for quantized_fetch_limit for IVFFlat. diff --git a/vectordb_bench/backend/clients/pinecone/pinecone.py b/vectordb_bench/backend/clients/pinecone/pinecone.py index 1a681b33f..9fd0afc7c 100644 --- a/vectordb_bench/backend/clients/pinecone/pinecone.py +++ b/vectordb_bench/backend/clients/pinecone/pinecone.py @@ -67,7 +67,7 @@ def insert_embeddings( embeddings: list[list[float]], metadata: list[int], **kwargs, - ) -> (int, Exception): + ) -> tuple[int, Exception]: assert len(embeddings) == len(metadata) insert_count = 0 try: diff --git a/vectordb_bench/backend/clients/qdrant_cloud/qdrant_cloud.py b/vectordb_bench/backend/clients/qdrant_cloud/qdrant_cloud.py index 5de72798b..f618c3ba4 100644 --- a/vectordb_bench/backend/clients/qdrant_cloud/qdrant_cloud.py +++ b/vectordb_bench/backend/clients/qdrant_cloud/qdrant_cloud.py @@ -111,7 +111,7 @@ def insert_embeddings( embeddings: list[list[float]], metadata: list[int], **kwargs, - ) -> (int, Exception): + ) -> tuple[int, Exception]: """Insert embeddings into Milvus. should call self.init() first""" assert self.qdrant_client is not None try: diff --git a/vectordb_bench/backend/clients/tidb/tidb.py b/vectordb_bench/backend/clients/tidb/tidb.py index d1f26084e..b75605eda 100644 --- a/vectordb_bench/backend/clients/tidb/tidb.py +++ b/vectordb_bench/backend/clients/tidb/tidb.py @@ -3,7 +3,7 @@ import logging import time from contextlib import contextmanager -from typing import Any, Optional, Tuple +from typing import Any import pymysql @@ -62,7 +62,7 @@ def _drop_table(self): conn.commit() except Exception as e: log.warning("Failed to drop table: %s error: %s", self.table_name, e) - raise e + raise def _create_table(self): try: @@ -80,7 +80,7 @@ def _create_table(self): conn.commit() except Exception as e: log.warning("Failed to create table: %s error: %s", self.table_name, e) - raise e + raise def ready_to_load(self) -> bool: pass @@ -122,25 +122,25 @@ def _optimize_check_tiflash_replica_progress(self): f""" SELECT PROGRESS FROM information_schema.tiflash_replica WHERE TABLE_SCHEMA = "{database}" AND TABLE_NAME = "{self.table_name}" - """ + """ # noqa: S608 ) result = cursor.fetchone() return result[0] except Exception as e: log.warning("Failed to check TiFlash replica progress: %s", e) - raise e + raise def _optimize_wait_tiflash_catch_up(self): try: with self._get_connection() as (conn, cursor): cursor.execute('SET @@TIDB_ISOLATION_READ_ENGINES="tidb,tiflash"') conn.commit() - cursor.execute(f"SELECT COUNT(*) FROM {self.table_name}") + cursor.execute(f"SELECT COUNT(*) FROM {self.table_name}") # noqa: S608 result = cursor.fetchone() return result[0] except Exception as e: log.warning("Failed to wait TiFlash to catch up: %s", e) - raise e + raise def _optimize_compact_tiflash(self): try: @@ -149,7 +149,7 @@ def _optimize_compact_tiflash(self): conn.commit() except Exception as e: log.warning("Failed to compact table: %s", e) - raise e + raise def _optimize_get_tiflash_index_pending_rows(self): try: @@ -160,13 +160,13 @@ def _optimize_get_tiflash_index_pending_rows(self): SELECT SUM(ROWS_STABLE_NOT_INDEXED) FROM information_schema.tiflash_indexes WHERE TIDB_DATABASE = "{database}" AND TIDB_TABLE = "{self.table_name}" - """ + """ # noqa: S608 ) result = cursor.fetchone() return result[0] except Exception as e: log.warning("Failed to read TiFlash index pending rows: %s", e) - raise e + raise def _insert_embeddings_serial( self, @@ -178,29 +178,28 @@ def _insert_embeddings_serial( try: with self._get_connection() as (conn, cursor): buf = io.StringIO() - buf.write(f"INSERT INTO {self.table_name} (id, embedding) VALUES ") + buf.write(f"INSERT INTO {self.table_name} (id, embedding) VALUES ") # noqa: S608 for i in range(offset, offset + size): if i > offset: buf.write(",") - buf.write(f'({metadata[i]}, "{str(embeddings[i])}")') + buf.write(f'({metadata[i]}, "{embeddings[i]!s}")') cursor.execute(buf.getvalue()) conn.commit() except Exception as e: log.warning("Failed to insert data into table: %s", e) - raise e + raise def insert_embeddings( self, embeddings: list[list[float]], metadata: list[int], **kwargs: Any, - ) -> Tuple[int, Optional[Exception]]: + ) -> tuple[int, Exception]: workers = 10 # Avoid exceeding MAX_ALLOWED_PACKET (default=64MB) max_batch_size = 64 * 1024 * 1024 // 24 // self.dim batch_size = len(embeddings) // workers - if batch_size > max_batch_size: - batch_size = max_batch_size + batch_size = min(batch_size, max_batch_size) with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor: futures = [] for i in range(0, len(embeddings), batch_size): @@ -227,8 +226,8 @@ def search_embedding( self.cursor.execute( f""" SELECT id FROM {self.table_name} - ORDER BY {self.search_fn}(embedding, "{str(query)}") LIMIT {k}; - """ + ORDER BY {self.search_fn}(embedding, "{query!s}") LIMIT {k}; + """ # noqa: S608 ) result = self.cursor.fetchall() return [int(i[0]) for i in result] diff --git a/vectordb_bench/backend/clients/vespa/util.py b/vectordb_bench/backend/clients/vespa/util.py index 97d03d644..7a64cc30d 100644 --- a/vectordb_bench/backend/clients/vespa/util.py +++ b/vectordb_bench/backend/clients/vespa/util.py @@ -2,6 +2,7 @@ From https://docs.vespa.ai/en/binarizing-vectors.html#appendix-conversion-to-int8 """ + import numpy as np @@ -11,6 +12,4 @@ def binarize_tensor(tensor: list[float]) -> list[int]: and packing the bits into bytes. """ tensor = np.array(tensor) - return ( - np.packbits(np.where(tensor > 0, 1, 0), axis=0).astype(np.int8).tolist() - ) + return np.packbits(np.where(tensor > 0, 1, 0), axis=0).astype(np.int8).tolist() diff --git a/vectordb_bench/backend/clients/vespa/vespa.py b/vectordb_bench/backend/clients/vespa/vespa.py index 19bb3f5a4..5288bc04c 100644 --- a/vectordb_bench/backend/clients/vespa/vespa.py +++ b/vectordb_bench/backend/clients/vespa/vespa.py @@ -7,8 +7,8 @@ from vespa import application from ..api import VectorDB -from .config import VespaHNSWConfig from . import util +from .config import VespaHNSWConfig log = logging.getLogger(__name__) @@ -116,17 +116,12 @@ def search_embedding( id_filter = filters.get("id") yql += f" and id >= {id_filter}" - query_embedding = ( - query - if self.case_config.quantization_type == "none" - else util.binarize_tensor(query) - ) + query_embedding = query if self.case_config.quantization_type == "none" else util.binarize_tensor(query) ranking = self.case_config.quantization_type result = self.client.query({"yql": yql, "input.query(query_embedding)": query_embedding, "ranking": ranking}) - result_ids = [child["fields"]["id"] for child in result.get_json()["root"]["children"]] - return result_ids + return [child["fields"]["id"] for child in result.get_json()["root"]["children"]] def optimize(self, data_size: int | None = None): """optimize will be called between insertion and search in performance cases. diff --git a/vectordb_bench/backend/clients/weaviate_cloud/weaviate_cloud.py b/vectordb_bench/backend/clients/weaviate_cloud/weaviate_cloud.py index aa4368bb7..c31104d8b 100644 --- a/vectordb_bench/backend/clients/weaviate_cloud/weaviate_cloud.py +++ b/vectordb_bench/backend/clients/weaviate_cloud/weaviate_cloud.py @@ -99,7 +99,7 @@ def insert_embeddings( embeddings: Iterable[list[float]], metadata: list[int], **kwargs, - ) -> (int, Exception): + ) -> tuple[int, Exception]: """Insert embeddings into Weaviate""" assert self.client.schema.exists(self.collection_name) insert_count = 0 diff --git a/vectordb_bench/cli/cli.py b/vectordb_bench/cli/cli.py index 3bb7763d8..bdf1a25f4 100644 --- a/vectordb_bench/cli/cli.py +++ b/vectordb_bench/cli/cli.py @@ -1,9 +1,9 @@ import logging -import os import time from collections.abc import Callable from concurrent.futures import wait from datetime import datetime +from pathlib import Path from pprint import pformat from typing import ( Annotated, @@ -38,18 +38,16 @@ from yaml import Loader -def click_get_defaults_from_file(ctx, param, value): +def click_get_defaults_from_file(ctx, param, value): # noqa: ANN001, ARG001 if value: - if os.path.exists(value): - input_file = value - else: - input_file = os.path.join(config.CONFIG_LOCAL_DIR, value) + input_file = value if Path.exists(value) else Path.join(config.CONFIG_LOCAL_DIR, value) try: - with open(input_file) as f: - _config: dict[str, dict[str, Any]] = load(f.read(), Loader=Loader) + with Path.open(input_file) as f: + _config: dict[str, dict[str, Any]] = load(f.read(), Loader=Loader) # noqa: S506 ctx.default_map = _config.get(ctx.command.name, {}) except Exception as e: - raise click.BadParameter(f"Failed to load config file: {e}") + msg = f"Failed to load config file: {e}" + raise click.BadParameter(msg) from e return value @@ -68,12 +66,16 @@ def click_parameter_decorators_from_typed_dict( For clarity, the key names of the TypedDict will be used to determine the type hints for the input parameters. - The actual function parameters are controlled by the click.option definitions. You must manually ensure these are aligned in a sensible way! + The actual function parameters are controlled by the click.option definitions. + You must manually ensure these are aligned in a sensible way! Example: ``` class CommonTypedDict(TypedDict): - z: Annotated[int, click.option("--z/--no-z", is_flag=True, type=bool, help="help z", default=True, show_default=True)] + z: Annotated[ + int, + click.option("--z/--no-z", is_flag=True, type=bool, help="help z", default=True, show_default=True) + ] name: Annotated[str, click.argument("name", required=False, default="Jeff")] class FooTypedDict(CommonTypedDict): @@ -91,14 +93,16 @@ def foo(**parameters: Unpack[FooTypedDict]): for _, t in get_type_hints(typed_dict, include_extras=True).items(): assert get_origin(t) is Annotated if len(t.__metadata__) == 1 and t.__metadata__[0].__module__ == "click.decorators": - # happy path -- only accept Annotated[..., Union[click.option,click.argument,...]] with no additional metadata defined (len=1) + # happy path -- only accept Annotated[..., Union[click.option,click.argument,...]] + # with no additional metadata defined (len=1) decorators.append(t.__metadata__[0]) else: raise RuntimeError( - "Click-TypedDict decorator parsing must only contain root type and a click decorator like click.option. See docstring", + "Click-TypedDict decorator parsing must only contain root type " + "and a click decorator like click.option. See docstring", ) - def deco(f): + def deco(f): # noqa: ANN001 for dec in reversed(decorators): f = dec(f) return f @@ -106,7 +110,7 @@ def deco(f): return deco -def click_arg_split(ctx: click.Context, param: click.core.Option, value): +def click_arg_split(ctx: click.Context, param: click.core.Option, value): # noqa: ANN001, ARG001 """Will split a comma-separated list input into an actual list. Args: @@ -145,8 +149,7 @@ def parse_task_stages( return stages -# ruff: noqa -def check_custom_case_parameters(ctx: any, param: any, value: any): +def check_custom_case_parameters(ctx: any, param: any, value: any): # noqa: ARG001 if ctx.params.get("case_type") == "PerformanceCustomDataset" and value is None: raise click.BadParameter( """ Custom case parameters diff --git a/vectordb_bench/cli/vectordbbench.py b/vectordb_bench/cli/vectordbbench.py index 7469ec9d5..1c6ff1260 100644 --- a/vectordb_bench/cli/vectordbbench.py +++ b/vectordb_bench/cli/vectordbbench.py @@ -1,5 +1,6 @@ from ..backend.clients.alloydb.cli import AlloyDBScaNN from ..backend.clients.aws_opensearch.cli import AWSOpenSearch +from ..backend.clients.clickhouse.cli import Clickhouse from ..backend.clients.mariadb.cli import MariaDBHNSW from ..backend.clients.memorydb.cli import MemoryDB from ..backend.clients.milvus.cli import MilvusAutoIndex @@ -13,7 +14,6 @@ from ..backend.clients.vespa.cli import Vespa from ..backend.clients.weaviate_cloud.cli import Weaviate from ..backend.clients.zilliz_cloud.cli import ZillizAutoIndex -from ..backend.clients.clickhouse.cli import Clickhouse from .cli import cli cli.add_command(PgVectorHNSW) diff --git a/vectordb_bench/frontend/config/dbCaseConfigs.py b/vectordb_bench/frontend/config/dbCaseConfigs.py index e85b42ff3..bb9cfa44b 100644 --- a/vectordb_bench/frontend/config/dbCaseConfigs.py +++ b/vectordb_bench/frontend/config/dbCaseConfigs.py @@ -1087,8 +1087,7 @@ class CaseConfigInput(BaseModel): "max": 200, "value": 6, }, - isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) - == IndexType.HNSW.value, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.HNSW.value, ) CaseConfigParamInput_EFSearch_MariaDB = CaseConfigInput( @@ -1100,8 +1099,7 @@ class CaseConfigInput(BaseModel): "max": 10000, "value": 20, }, - isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) - == IndexType.HNSW.value, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.HNSW.value, ) CaseConfigParamInput_CacheSize_MariaDB = CaseConfigInput( @@ -1111,10 +1109,9 @@ class CaseConfigInput(BaseModel): inputConfig={ "min": 1048576, "max": (1 << 53) - 1, - "value": 16 * 1024 ** 3, + "value": 16 * 1024**3, }, - isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) - == IndexType.HNSW.value, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.HNSW.value, ) CaseConfigParamInput_MongoDBQuantizationType = CaseConfigInput( @@ -1162,10 +1159,7 @@ class CaseConfigInput(BaseModel): label=CaseConfigParamType.quantizationType, inputType=InputType.Option, inputConfig={ - "options": [ - "none", - "binary" - ], + "options": ["none", "binary"], }, ) diff --git a/vectordb_bench/frontend/config/styles.py b/vectordb_bench/frontend/config/styles.py index 4418c19da..03bda0fec 100644 --- a/vectordb_bench/frontend/config/styles.py +++ b/vectordb_bench/frontend/config/styles.py @@ -64,5 +64,5 @@ def getPatternShape(i): DB.Redis.value: "#0D6EFD", DB.AWSOpenSearch.value: "#0DCAF0", DB.TiDB.value: "#0D6EFD", - DB.Vespa.value: "#61d790" + DB.Vespa.value: "#61d790", } diff --git a/vectordb_bench/models.py b/vectordb_bench/models.py index e206919ac..b28521096 100644 --- a/vectordb_bench/models.py +++ b/vectordb_bench/models.py @@ -263,7 +263,6 @@ def read_file(cls, full_path: pathlib.Path, trans_unit: bool = False) -> Self: ) return TestResult.validate(test_result) - # ruff: noqa def display(self, dbs: list[DB] | None = None): filter_list = dbs if dbs and isinstance(dbs, list) else None sorted_results = sorted( @@ -294,7 +293,7 @@ def append_return(x: any, y: any): max_qps = 10 if max_qps < 10 else max_qps max_recall = 13 if max_recall < 13 else max_recall - LENGTH = ( + LENGTH = ( # noqa: N806 max_db, max_db_labels, max_case, @@ -307,13 +306,13 @@ def append_return(x: any, y: any): 5, ) - DATA_FORMAT = ( + DATA_FORMAT = ( # noqa: N806 f"%-{max_db}s | %-{max_db_labels}s %-{max_case}s %-{len(self.task_label)}s" f" | %-{max_load_dur}s %-{max_qps}s %-15s %-{max_recall}s %-14s" f" | %-5s" ) - TITLE = DATA_FORMAT % ( + TITLE = DATA_FORMAT % ( # noqa: N806 "DB", "db_label", "case", @@ -325,8 +324,8 @@ def append_return(x: any, y: any): "max_load_count", "label", ) - SPLIT = DATA_FORMAT % tuple(map(lambda x: "-" * x, LENGTH)) - SUMMARY_FORMAT = ("Task summary: run_id=%s, task_label=%s") % ( + SPLIT = DATA_FORMAT % tuple(map(lambda x: "-" * x, LENGTH)) # noqa: C417, N806 + SUMMARY_FORMAT = ("Task summary: run_id=%s, task_label=%s") % ( # noqa: N806 self.run_id[:5], self.task_label, ) From 6be5c2bb625f76be50af2914a4518bff6f8779b3 Mon Sep 17 00:00:00 2001 From: "min.tian" Date: Wed, 16 Apr 2025 10:43:36 +0800 Subject: [PATCH 154/327] fix cli crush Signed-off-by: min.tian --- vectordb_bench/cli/cli.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/vectordb_bench/cli/cli.py b/vectordb_bench/cli/cli.py index bdf1a25f4..4b42a912c 100644 --- a/vectordb_bench/cli/cli.py +++ b/vectordb_bench/cli/cli.py @@ -40,9 +40,10 @@ def click_get_defaults_from_file(ctx, param, value): # noqa: ANN001, ARG001 if value: - input_file = value if Path.exists(value) else Path.join(config.CONFIG_LOCAL_DIR, value) + path = Path(value) + input_file = path if path.exists() else Path(config.CONFIG_LOCAL_DIR, path) try: - with Path.open(input_file) as f: + with input_file.open() as f: _config: dict[str, dict[str, Any]] = load(f.read(), Loader=Loader) # noqa: S506 ctx.default_map = _config.get(ctx.command.name, {}) except Exception as e: From 6d3f4a45d57c00e037bdff28662f943c74f5576f Mon Sep 17 00:00:00 2001 From: Polo Vezia Date: Thu, 17 Apr 2025 08:19:39 +0000 Subject: [PATCH 155/327] downgrade streamlit version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index a66cfac1a..09fa66972 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,7 @@ dependencies = [ "click", "pytz", "streamlit-autorefresh", - "streamlit!=1.34.0", + "streamlit<1.44,!=1.34.0", # There is a breaking change in 1.44 related to get_page https://discuss.streamlit.io/t/from-streamlit-source-util-import-get-pages-gone-in-v-1-44-0-need-urgent-help/98399 "streamlit_extras", "tqdm", "s3fs", From b979e79aa3fb8d0cce100c8f4291def3310bd16c Mon Sep 17 00:00:00 2001 From: "min.tian" Date: Fri, 18 Apr 2025 11:30:30 +0800 Subject: [PATCH 156/327] add more milvus index types: hnsw sq/pq/prq; ivf rabitq Signed-off-by: min.tian --- vectordb_bench/backend/clients/api.py | 12 ++ .../backend/clients/milvus/config.py | 113 +++++++++++++++++- .../backend/clients/milvus/milvus.py | 4 +- .../frontend/config/dbCaseConfigs.py | 111 +++++++++++++++-- vectordb_bench/models.py | 6 + 5 files changed, 237 insertions(+), 9 deletions(-) diff --git a/vectordb_bench/backend/clients/api.py b/vectordb_bench/backend/clients/api.py index ce2b05650..8070164dd 100644 --- a/vectordb_bench/backend/clients/api.py +++ b/vectordb_bench/backend/clients/api.py @@ -16,10 +16,14 @@ class MetricType(str, Enum): class IndexType(str, Enum): HNSW = "HNSW" + HNSW_SQ = "HNSW_SQ" + HNSW_PQ = "HNSW_PQ" + HNSW_PRQ = "HNSW_PRQ" DISKANN = "DISKANN" STREAMING_DISKANN = "DISKANN" IVFFlat = "IVF_FLAT" IVFSQ8 = "IVF_SQ8" + IVF_RABITQ = "IVF_RABITQ" Flat = "FLAT" AUTOINDEX = "AUTOINDEX" ES_HNSW = "hnsw" @@ -31,6 +35,14 @@ class IndexType(str, Enum): SCANN = "scann" +class SQType(str, Enum): + SQ6 = "SQ6" + SQ8 = "SQ8" + BF16 = "BF16" + FP16 = "FP16" + FP32 = "FP32" + + class DBConfig(ABC, BaseModel): """DBConfig contains the connection info of vector database diff --git a/vectordb_bench/backend/clients/milvus/config.py b/vectordb_bench/backend/clients/milvus/config.py index e3a3f9b19..07cd9aad8 100644 --- a/vectordb_bench/backend/clients/milvus/config.py +++ b/vectordb_bench/backend/clients/milvus/config.py @@ -1,6 +1,6 @@ from pydantic import BaseModel, SecretStr, validator -from ..api import DBCaseConfig, DBConfig, IndexType, MetricType +from ..api import DBCaseConfig, DBConfig, IndexType, MetricType, SQType class MilvusConfig(DBConfig): @@ -88,6 +88,88 @@ def search_param(self) -> dict: } +class HNSWSQConfig(HNSWConfig, DBCaseConfig): + index: IndexType = IndexType.HNSW_SQ + sq_type: SQType = SQType.SQ8 + refine: bool = True + refine_type: SQType = SQType.FP32 + refine_k: float = 1 + + def index_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "index_type": self.index.value, + "params": { + "M": self.M, + "efConstruction": self.efConstruction, + "sq_type": self.sq_type.value, + "refine": self.refine, + "refine_type": self.refine_type.value, + }, + } + + def search_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "params": {"ef": self.ef, "refine_k": self.refine_k}, + } + + +class HNSWPQConfig(HNSWConfig): + index: IndexType = IndexType.HNSW_PQ + m: int = 32 + nbits: int = 8 + refine: bool = True + refine_type: SQType = SQType.FP32 + refine_k: float = 1 + + def index_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "index_type": self.index.value, + "params": { + "M": self.M, + "efConstruction": self.efConstruction, + "m": self.m, + "nbits": self.nbits, + "refine": self.refine, + "refine_type": self.refine_type.value, + }, + } + + def search_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "params": {"ef": self.ef, "refine_k": self.refine_k}, + } + + +class HNSWPRQConfig(HNSWPQConfig): + index: IndexType = IndexType.HNSW_PRQ + nrq: int = 2 + + def index_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "index_type": self.index.value, + "params": { + "M": self.M, + "efConstruction": self.efConstruction, + "m": self.m, + "nbits": self.nbits, + "nrq": self.nrq, + "refine": self.refine, + "refine_type": self.refine_type.value, + }, + } + + def search_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "params": {"ef": self.ef, "refine_k": self.refine_k}, + } + + class DISKANNConfig(MilvusIndexConfig, DBCaseConfig): search_list: int | None = None index: IndexType = IndexType.DISKANN @@ -144,6 +226,31 @@ def search_param(self) -> dict: } +class IVFRABITQConfig(IVFSQ8Config): + index: IndexType = IndexType.IVF_RABITQ + rbq_bits_query: int = 0 # 0, 1, 2, ..., 8 + refine: bool = True + refine_type: SQType = SQType.FP32 + refine_k: float = 1 + + def index_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "index_type": self.index.value, + "params": { + "nlist": self.nlist, + "refine": self.refine, + "refine_type": self.refine_type.value, + }, + } + + def search_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "params": {"nprobe": self.nprobe, "rbq_bits_query": self.rbq_bits_query, "refine_k": self.refine_k}, + } + + class FLATConfig(MilvusIndexConfig, DBCaseConfig): index: IndexType = IndexType.Flat @@ -285,9 +392,13 @@ def search_param(self) -> dict: _milvus_case_config = { IndexType.AUTOINDEX: AutoIndexConfig, IndexType.HNSW: HNSWConfig, + IndexType.HNSW_SQ: HNSWSQConfig, + IndexType.HNSW_PQ: HNSWPQConfig, + IndexType.HNSW_PRQ: HNSWPRQConfig, IndexType.DISKANN: DISKANNConfig, IndexType.IVFFlat: IVFFlatConfig, IndexType.IVFSQ8: IVFSQ8Config, + IndexType.IVF_RABITQ: IVFRABITQConfig, IndexType.Flat: FLATConfig, IndexType.GPU_IVF_FLAT: GPUIVFFlatConfig, IndexType.GPU_IVF_PQ: GPUIVFPQConfig, diff --git a/vectordb_bench/backend/clients/milvus/milvus.py b/vectordb_bench/backend/clients/milvus/milvus.py index c812698fe..465c51179 100644 --- a/vectordb_bench/backend/clients/milvus/milvus.py +++ b/vectordb_bench/backend/clients/milvus/milvus.py @@ -61,6 +61,7 @@ def __init__( consistency_level="Session", ) + log.info(f"{self.name} create index: index_params: {self.case_config.index_param()}") col.create_index( self._vector_field, self.case_config.index_param(), @@ -71,7 +72,7 @@ def __init__( connections.disconnect("default") @contextmanager - def init(self) -> None: + def init(self): """ Examples: >>> with self.init(): @@ -126,6 +127,7 @@ def wait_index(): try: self.col.compact() self.col.wait_for_compaction_completed() + log.info("compactation completed. waiting for the rest of index buliding.") except Exception as e: log.warning(f"{self.name} compact error: {e}") if hasattr(e, "code"): diff --git a/vectordb_bench/frontend/config/dbCaseConfigs.py b/vectordb_bench/frontend/config/dbCaseConfigs.py index bb9cfa44b..bd59c3470 100644 --- a/vectordb_bench/frontend/config/dbCaseConfigs.py +++ b/vectordb_bench/frontend/config/dbCaseConfigs.py @@ -3,7 +3,7 @@ from pydantic import BaseModel from vectordb_bench.backend.cases import CaseLabel, CaseType from vectordb_bench.backend.clients import DB -from vectordb_bench.backend.clients.api import IndexType, MetricType +from vectordb_bench.backend.clients.api import IndexType, MetricType, SQType from vectordb_bench.frontend.components.custom.getCustomConfig import get_custom_configs from vectordb_bench.models import CaseConfig, CaseConfigParamType @@ -164,10 +164,13 @@ class CaseConfigInput(BaseModel): inputConfig={ "options": [ IndexType.HNSW.value, + IndexType.HNSW_SQ.value, + IndexType.HNSW_PQ.value, + IndexType.HNSW_PRQ.value, IndexType.IVFFlat.value, IndexType.IVFSQ8.value, + IndexType.IVF_RABITQ.value, IndexType.DISKANN.value, - IndexType.STREAMING_DISKANN.value, IndexType.Flat.value, IndexType.AUTOINDEX.value, IndexType.GPU_IVF_FLAT.value, @@ -346,9 +349,16 @@ class CaseConfigInput(BaseModel): "max": 64, "value": 30, }, - isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.HNSW.value, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) + in [ + IndexType.HNSW.value, + IndexType.HNSW_SQ.value, + IndexType.HNSW_PQ.value, + IndexType.HNSW_PRQ.value, + ], ) + CaseConfigParamInput_m = CaseConfigInput( label=CaseConfigParamType.m, inputType=InputType.Number, @@ -369,7 +379,62 @@ class CaseConfigInput(BaseModel): "max": 512, "value": 360, }, - isDisplayed=lambda config: config[CaseConfigParamType.IndexType] == IndexType.HNSW.value, + isDisplayed=lambda config: config[CaseConfigParamType.IndexType] + in [ + IndexType.HNSW.value, + IndexType.HNSW_SQ.value, + IndexType.HNSW_PQ.value, + IndexType.HNSW_PRQ.value, + ], +) + +CaseConfigParamInput_SQType = CaseConfigInput( + label=CaseConfigParamType.sq_type, + inputType=InputType.Option, + inputHelp="Scalar quantizer type.", + inputConfig={ + "options": [SQType.SQ6.value, SQType.SQ8.value, SQType.BF16.value, SQType.FP16.value, SQType.FP32.value] + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) in [IndexType.HNSW_SQ.value], +) + +CaseConfigParamInput_Refine = CaseConfigInput( + label=CaseConfigParamType.refine, + inputType=InputType.Option, + inputHelp="Whether refined data is reserved during index building.", + inputConfig={"options": [True, False]}, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) + in [IndexType.HNSW_SQ.value, IndexType.HNSW_PQ.value, IndexType.HNSW_PRQ.value, IndexType.IVF_RABITQ.value], +) + +CaseConfigParamInput_RefineType = CaseConfigInput( + label=CaseConfigParamType.refine_type, + inputType=InputType.Option, + inputHelp="The data type of the refine index.", + inputConfig={ + "options": [SQType.FP32.value, SQType.FP16.value, SQType.BF16.value, SQType.SQ8.value, SQType.SQ6.value] + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) + in [IndexType.HNSW_SQ.value, IndexType.HNSW_PQ.value, IndexType.HNSW_PRQ.value, IndexType.IVF_RABITQ.value] + and config.get(CaseConfigParamType.refine, True), +) + +CaseConfigParamInput_RefineK = CaseConfigInput( + label=CaseConfigParamType.refine_k, + inputType=InputType.Float, + inputHelp="The magnification factor of refine compared to k.", + inputConfig={"min": 1.0, "max": 10000.0, "value": 1.0}, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) + in [IndexType.HNSW_SQ.value, IndexType.HNSW_PQ.value, IndexType.HNSW_PRQ.value, IndexType.IVF_RABITQ.value] + and config.get(CaseConfigParamType.refine, True), +) + +CaseConfigParamInput_RBQBitsQuery = CaseConfigInput( + label=CaseConfigParamType.rbq_bits_query, + inputType=InputType.Number, + inputHelp="The magnification factor of refine compared to k.", + inputConfig={"min": 0, "max": 8, "value": 0}, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) in [IndexType.IVF_RABITQ.value], ) CaseConfigParamInput_EFConstruction_Weaviate = CaseConfigInput( @@ -519,7 +584,13 @@ class CaseConfigInput(BaseModel): "max": MAX_STREAMLIT_INT, "value": 100, }, - isDisplayed=lambda config: config[CaseConfigParamType.IndexType] == IndexType.HNSW.value, + isDisplayed=lambda config: config[CaseConfigParamType.IndexType] + in [ + IndexType.HNSW.value, + IndexType.HNSW_SQ.value, + IndexType.HNSW_PQ.value, + IndexType.HNSW_PRQ.value, + ], ) CaseConfigParamInput_EF_Weaviate = CaseConfigInput( @@ -561,6 +632,7 @@ class CaseConfigInput(BaseModel): in [ IndexType.IVFFlat.value, IndexType.IVFSQ8.value, + IndexType.IVF_RABITQ.value, IndexType.GPU_IVF_FLAT.value, IndexType.GPU_IVF_PQ.value, IndexType.GPU_BRUTE_FORCE.value, @@ -579,6 +651,7 @@ class CaseConfigInput(BaseModel): in [ IndexType.IVFFlat.value, IndexType.IVFSQ8.value, + IndexType.IVF_RABITQ.value, IndexType.GPU_IVF_FLAT.value, IndexType.GPU_IVF_PQ.value, IndexType.GPU_BRUTE_FORCE.value, @@ -593,7 +666,8 @@ class CaseConfigInput(BaseModel): "max": 65536, "value": 0, }, - isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) in [IndexType.GPU_IVF_PQ.value], + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) + in [IndexType.GPU_IVF_PQ.value, IndexType.HNSW_PQ.value, IndexType.HNSW_PRQ.value], ) @@ -605,7 +679,20 @@ class CaseConfigInput(BaseModel): "max": 65536, "value": 8, }, - isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) in [IndexType.GPU_IVF_PQ.value], + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) + in [IndexType.GPU_IVF_PQ.value, IndexType.HNSW_PQ.value, IndexType.HNSW_PRQ.value], +) + +CaseConfigParamInput_NRQ = CaseConfigInput( + label=CaseConfigParamType.nrq, + inputType=InputType.Number, + inputHelp="The number of residual subquantizers.", + inputConfig={ + "min": 1, + "max": 16, + "value": 2, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) in [IndexType.HNSW_PRQ.value], ) CaseConfigParamInput_intermediate_graph_degree = CaseConfigInput( @@ -1186,6 +1273,10 @@ class CaseConfigInput(BaseModel): CaseConfigParamInput_graph_degree, CaseConfigParamInput_build_algo, CaseConfigParamInput_cache_dataset_on_device, + CaseConfigParamInput_SQType, + CaseConfigParamInput_Refine, + CaseConfigParamInput_RefineType, + CaseConfigParamInput_NRQ, ] MilvusPerformanceConfig = [ CaseConfigParamInput_IndexType, @@ -1197,6 +1288,8 @@ class CaseConfigInput(BaseModel): CaseConfigParamInput_Nprobe, CaseConfigParamInput_M_PQ, CaseConfigParamInput_Nbits_PQ, + CaseConfigParamInput_RBQBitsQuery, + CaseConfigParamInput_NRQ, CaseConfigParamInput_intermediate_graph_degree, CaseConfigParamInput_graph_degree, CaseConfigParamInput_itopk_size, @@ -1207,6 +1300,10 @@ class CaseConfigInput(BaseModel): CaseConfigParamInput_build_algo, CaseConfigParamInput_cache_dataset_on_device, CaseConfigParamInput_refine_ratio, + CaseConfigParamInput_SQType, + CaseConfigParamInput_Refine, + CaseConfigParamInput_RefineType, + CaseConfigParamInput_RefineK, ] WeaviateLoadConfig = [ diff --git a/vectordb_bench/models.py b/vectordb_bench/models.py index b28521096..ca00c4b55 100644 --- a/vectordb_bench/models.py +++ b/vectordb_bench/models.py @@ -55,6 +55,7 @@ class CaseConfigParamType(Enum): quantizedFetchLimit = "quantized_fetch_limit" m = "m" nbits = "nbits" + nrq = "nrq" intermediate_graph_degree = "intermediate_graph_degree" graph_degree = "graph_degree" itopk_size = "itopk_size" @@ -65,6 +66,11 @@ class CaseConfigParamType(Enum): build_algo = "build_algo" cache_dataset_on_device = "cache_dataset_on_device" refine_ratio = "refine_ratio" + refine = "refine" + refine_type = "refine_type" + refine_k = "refine_k" + rbq_bits_query = "rbq_bits_query" + sq_type = "sq_type" level = "level" maintenance_work_mem = "maintenance_work_mem" max_parallel_workers = "max_parallel_workers" From 3f9c498f1a2173aa1c58dbe0ebe4f412b0b00dd8 Mon Sep 17 00:00:00 2001 From: "min.tian" Date: Wed, 23 Apr 2025 10:14:15 +0800 Subject: [PATCH 157/327] add more milvus index types: ivf_pq Signed-off-by: min.tian --- vectordb_bench/backend/clients/api.py | 1 + .../backend/clients/milvus/config.py | 22 +++++++++++++++++++ .../frontend/config/dbCaseConfigs.py | 11 ++++++---- 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/vectordb_bench/backend/clients/api.py b/vectordb_bench/backend/clients/api.py index 8070164dd..790da891b 100644 --- a/vectordb_bench/backend/clients/api.py +++ b/vectordb_bench/backend/clients/api.py @@ -22,6 +22,7 @@ class IndexType(str, Enum): DISKANN = "DISKANN" STREAMING_DISKANN = "DISKANN" IVFFlat = "IVF_FLAT" + IVFPQ = "IVF_PQ" IVFSQ8 = "IVF_SQ8" IVF_RABITQ = "IVF_RABITQ" Flat = "FLAT" diff --git a/vectordb_bench/backend/clients/milvus/config.py b/vectordb_bench/backend/clients/milvus/config.py index 07cd9aad8..672becf1b 100644 --- a/vectordb_bench/backend/clients/milvus/config.py +++ b/vectordb_bench/backend/clients/milvus/config.py @@ -207,6 +207,27 @@ def search_param(self) -> dict: } +class IVFPQConfig(MilvusIndexConfig, DBCaseConfig): + nlist: int + nprobe: int | None = None + m: int = 32 + nbits: int = 8 + index: IndexType = IndexType.IVFPQ + + def index_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "index_type": self.index.value, + "params": {"nlist": self.nlist, "m": self.m, "nbits": self.nbits}, + } + + def search_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "params": {"nprobe": self.nprobe}, + } + + class IVFSQ8Config(MilvusIndexConfig, DBCaseConfig): nlist: int nprobe: int | None = None @@ -397,6 +418,7 @@ def search_param(self) -> dict: IndexType.HNSW_PRQ: HNSWPRQConfig, IndexType.DISKANN: DISKANNConfig, IndexType.IVFFlat: IVFFlatConfig, + IndexType.IVFPQ: IVFPQConfig, IndexType.IVFSQ8: IVFSQ8Config, IndexType.IVF_RABITQ: IVFRABITQConfig, IndexType.Flat: FLATConfig, diff --git a/vectordb_bench/frontend/config/dbCaseConfigs.py b/vectordb_bench/frontend/config/dbCaseConfigs.py index bd59c3470..da5e91d91 100644 --- a/vectordb_bench/frontend/config/dbCaseConfigs.py +++ b/vectordb_bench/frontend/config/dbCaseConfigs.py @@ -168,6 +168,7 @@ class CaseConfigInput(BaseModel): IndexType.HNSW_PQ.value, IndexType.HNSW_PRQ.value, IndexType.IVFFlat.value, + IndexType.IVFPQ.value, IndexType.IVFSQ8.value, IndexType.IVF_RABITQ.value, IndexType.DISKANN.value, @@ -631,6 +632,7 @@ class CaseConfigInput(BaseModel): isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) in [ IndexType.IVFFlat.value, + IndexType.IVFPQ.value, IndexType.IVFSQ8.value, IndexType.IVF_RABITQ.value, IndexType.GPU_IVF_FLAT.value, @@ -650,6 +652,7 @@ class CaseConfigInput(BaseModel): isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) in [ IndexType.IVFFlat.value, + IndexType.IVFPQ.value, IndexType.IVFSQ8.value, IndexType.IVF_RABITQ.value, IndexType.GPU_IVF_FLAT.value, @@ -662,12 +665,12 @@ class CaseConfigInput(BaseModel): label=CaseConfigParamType.m, inputType=InputType.Number, inputConfig={ - "min": 0, + "min": 1, "max": 65536, - "value": 0, + "value": 32, }, isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) - in [IndexType.GPU_IVF_PQ.value, IndexType.HNSW_PQ.value, IndexType.HNSW_PRQ.value], + in [IndexType.GPU_IVF_PQ.value, IndexType.HNSW_PQ.value, IndexType.HNSW_PRQ.value, IndexType.IVFPQ.value], ) @@ -680,7 +683,7 @@ class CaseConfigInput(BaseModel): "value": 8, }, isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) - in [IndexType.GPU_IVF_PQ.value, IndexType.HNSW_PQ.value, IndexType.HNSW_PRQ.value], + in [IndexType.GPU_IVF_PQ.value, IndexType.HNSW_PQ.value, IndexType.HNSW_PRQ.value, IndexType.IVFPQ.value], ) CaseConfigParamInput_NRQ = CaseConfigInput( From f7f551e77400f463b64852c1276205b00b692d16 Mon Sep 17 00:00:00 2001 From: MansorY <119126888+MansorY23@users.noreply.github.com> Date: Thu, 24 Apr 2025 04:49:40 +0300 Subject: [PATCH 158/327] Add HNSW support for Clickhouse client (#500) * feat: add hnsw support * refactor: minor fixes * feat: reformat code * fix: remove sql injections, reformat code --- .../backend/clients/clickhouse/clickhouse.py | 170 +++++++++++++----- .../backend/clients/clickhouse/config.py | 49 +++-- 2 files changed, 162 insertions(+), 57 deletions(-) diff --git a/vectordb_bench/backend/clients/clickhouse/clickhouse.py b/vectordb_bench/backend/clients/clickhouse/clickhouse.py index e241cdb8d..498132ca9 100644 --- a/vectordb_bench/backend/clients/clickhouse/clickhouse.py +++ b/vectordb_bench/backend/clients/clickhouse/clickhouse.py @@ -5,8 +5,11 @@ from typing import Any import clickhouse_connect +from clickhouse_connect.driver import Client -from ..api import DBCaseConfig, VectorDB +from .. import IndexType +from ..api import VectorDB +from .config import ClickhouseConfigDict, ClickhouseIndexConfig log = logging.getLogger(__name__) @@ -17,8 +20,8 @@ class Clickhouse(VectorDB): def __init__( self, dim: int, - db_config: dict, - db_case_config: DBCaseConfig, + db_config: ClickhouseConfigDict, + db_case_config: ClickhouseIndexConfig, collection_name: str = "CHVectorCollection", drop_old: bool = False, **kwargs, @@ -28,29 +31,29 @@ def __init__( self.table_name = collection_name self.dim = dim + self.index_param = self.case_config.index_param() + self.search_param = self.case_config.search_param() + self.session_param = self.case_config.session_param() + self._index_name = "clickhouse_index" self._primary_field = "id" self._vector_field = "embedding" # construct basic units - self.conn = clickhouse_connect.get_client( - host=self.db_config["host"], - port=self.db_config["port"], - username=self.db_config["user"], - password=self.db_config["password"], - database=self.db_config["dbname"], - ) + self.conn = self._create_connection(**self.db_config, settings=self.session_param) if drop_old: log.info(f"Clickhouse client drop table : {self.table_name}") self._drop_table() self._create_table(dim) + if self.case_config.create_index_before_load: + self._create_index() self.conn.close() self.conn = None @contextmanager - def init(self): + def init(self) -> None: """ Examples: >>> with self.init(): @@ -58,13 +61,7 @@ def init(self): >>> self.search_embedding() """ - self.conn = clickhouse_connect.get_client( - host=self.db_config["host"], - port=self.db_config["port"], - username=self.db_config["user"], - password=self.db_config["password"], - database=self.db_config["dbname"], - ) + self.conn = self._create_connection(**self.db_config, settings=self.session_param) try: yield @@ -72,10 +69,61 @@ def init(self): self.conn.close() self.conn = None + def _create_connection(self, settings: dict | None, **kwargs) -> Client: + return clickhouse_connect.get_client(**self.db_config, settings=settings) + + def _drop_index(self): + assert self.conn is not None, "Connection is not initialized" + try: + self.conn.command( + f'ALTER TABLE {self.db_config["database"]}.{self.table_name} DROP INDEX {self._index_name}' + ) + except Exception as e: + log.warning(f"Failed to drop index on table {self.db_config['database']}.{self.table_name}: {e}") + raise e from None + def _drop_table(self): assert self.conn is not None, "Connection is not initialized" - self.conn.command(f'DROP TABLE IF EXISTS {self.db_config["dbname"]}.{self.table_name}') + try: + self.conn.command(f'DROP TABLE IF EXISTS {self.db_config["database"]}.{self.table_name}') + except Exception as e: + log.warning(f"Failed to drop table {self.db_config['database']}.{self.table_name}: {e}") + raise e from None + + def _perfomance_tuning(self): + self.conn.command("SET materialize_skip_indexes_on_insert = 1") + + def _create_index(self): + assert self.conn is not None, "Connection is not initialized" + try: + if self.index_param["index_type"] == IndexType.HNSW.value: + if ( + self.index_param["quantization"] + and self.index_param["params"]["M"] + and self.index_param["params"]["efConstruction"] + ): + query = f""" + ALTER TABLE {self.db_config["database"]}.{self.table_name} + ADD INDEX {self._index_name} {self._vector_field} + TYPE vector_similarity('hnsw', '{self.index_param["metric_type"]}', + '{self.index_param["quantization"]}', + {self.index_param["params"]["M"]}, {self.index_param["params"]["efConstruction"]}) + GRANULARITY {self.index_param["granularity"]} + """ + else: + query = f""" + ALTER TABLE {self.db_config["database"]}.{self.table_name} + ADD INDEX {self._index_name} {self._vector_field} + TYPE vector_similarity('hnsw', '{self.index_param["metric_type"]}') + GRANULARITY {self.index_param["granularity"]} + """ + self.conn.command(cmd=query) + else: + log.warning("HNSW is only avaliable method in clickhouse now") + except Exception as e: + log.warning(f"Failed to create Clickhouse vector index on table: {self.table_name} error: {e}") + raise e from None def _create_table(self, dim: int): assert self.conn is not None, "Connection is not initialized" @@ -83,21 +131,22 @@ def _create_table(self, dim: int): try: # create table self.conn.command( - f'CREATE TABLE IF NOT EXISTS {self.db_config["dbname"]}.{self.table_name} \ - (id UInt32, embedding Array(Float64)) ENGINE = MergeTree() ORDER BY id;' + f'CREATE TABLE IF NOT EXISTS {self.db_config["database"]}.{self.table_name} ' + f"({self._primary_field} UInt32, " + f'{self._vector_field} Array({self.index_param["vector_data_type"]}) CODEC(NONE), ' + f"CONSTRAINT same_length CHECK length(embedding) = {dim}) " + f"ENGINE = MergeTree() " + f"ORDER BY {self._primary_field}" ) except Exception as e: log.warning(f"Failed to create Clickhouse table: {self.table_name} error: {e}") raise e from None - def ready_to_load(self): - pass - def optimize(self, data_size: int | None = None): pass - def ready_to_search(self): + def _post_insert(self): pass def insert_embeddings( @@ -105,7 +154,7 @@ def insert_embeddings( embeddings: list[list[float]], metadata: list[int], **kwargs: Any, - ) -> tuple[int, Exception]: + ) -> (int, Exception): assert self.conn is not None, "Connection is not initialized" try: @@ -116,7 +165,7 @@ def insert_embeddings( table=self.table_name, data=items, column_names=["id", "embedding"], - column_type_names=["UInt32", "Array(Float64)"], + column_type_names=["UInt32", f'Array({self.index_param["vector_data_type"]})'], column_oriented=True, ) return len(metadata), None @@ -132,25 +181,52 @@ def search_embedding( timeout: int | None = None, ) -> list[int]: assert self.conn is not None, "Connection is not initialized" - - index_param = self.case_config.index_param() # noqa: F841 - search_param = self.case_config.search_param() - - if filters: - gt = filters.get("id") - filter_sql = ( - f'SELECT id, {search_param["metric_type"]}(embedding,{query}) AS score ' # noqa: S608 - f'FROM {self.db_config["dbname"]}.{self.table_name} ' - f"WHERE id > {gt} " - f"ORDER BY score LIMIT {k};" - ) - result = self.conn.query(filter_sql).result_rows + parameters = { + "primary_field": self._primary_field, + "vector_field": self._vector_field, + "schema": self.db_config["database"], + "table": self.table_name, + "gt": filters.get("id"), + "k": k, + "metric_type": self.search_param["metric_type"], + "query": query, + } + if self.case_config.metric_type == "COSINE": + if filters: + result = self.conn.query( + "SELECT {primary_field:Identifier}, {vector_field:Identifier} " + "FROM {schema:Identifier}.{table:Identifier} " + "WHERE {primary_field:Identifier} > {gt:UInt32} " + "ORDER BY cosineDistance(embedding,{query:Array(Float64)}) " + "LIMIT {k:UInt32}", + parameters=parameters, + ).result_rows + return [int(row[0]) for row in result] + + result = self.conn.query( + "SELECT {primary_field:Identifier}, {vector_field:Identifier} " + "FROM {schema:Identifier}.{table:Identifier} " + "ORDER BY cosineDistance(embedding,{query:Array(Float64)}) " + "LIMIT {k:UInt32}", + parameters=parameters, + ).result_rows return [int(row[0]) for row in result] - else: # noqa: RET505 - select_sql = ( - f'SELECT id, {search_param["metric_type"]}(embedding,{query}) AS score ' # noqa: S608 - f'FROM {self.db_config["dbname"]}.{self.table_name} ' - f"ORDER BY score LIMIT {k};" - ) - result = self.conn.query(select_sql).result_rows + if filters: + result = self.conn.query( + "SELECT {primary_field:Identifier}, {vector_field:Identifier} " + "FROM {schema:Identifier}.{table:Identifier} " + "WHERE {primary_field:Identifier} > {gt:UInt32} " + "ORDER BY L2Distance(embedding,{query:Array(Float64)}) " + "LIMIT {k:UInt32}", + parameters=parameters, + ).result_rows return [int(row[0]) for row in result] + + result = self.conn.query( + "SELECT {primary_field:Identifier}, {vector_field:Identifier} " + "FROM {schema:Identifier}.{table:Identifier} " + "ORDER BY L2Distance(embedding,{query:Array(Float64)}) " + "LIMIT {k:UInt32}", + parameters=parameters, + ).result_rows + return [int(row[0]) for row in result] diff --git a/vectordb_bench/backend/clients/clickhouse/config.py b/vectordb_bench/backend/clients/clickhouse/config.py index fad446049..a4c5fe499 100644 --- a/vectordb_bench/backend/clients/clickhouse/config.py +++ b/vectordb_bench/backend/clients/clickhouse/config.py @@ -1,29 +1,46 @@ +from abc import abstractmethod +from typing import TypedDict + from pydantic import BaseModel, SecretStr from ..api import DBCaseConfig, DBConfig, IndexType, MetricType +class ClickhouseConfigDict(TypedDict): + user: str + password: str + host: str + port: int + database: str + secure: bool + + class ClickhouseConfig(DBConfig): user_name: str = "clickhouse" password: SecretStr host: str = "localhost" port: int = 8123 db_name: str = "default" + secure: bool = False - def to_dict(self) -> dict: + def to_dict(self) -> ClickhouseConfigDict: pwd_str = self.password.get_secret_value() return { "host": self.host, "port": self.port, - "dbname": self.db_name, + "database": self.db_name, "user": self.user_name, "password": pwd_str, + "secure": self.secure, } -class ClickhouseIndexConfig(BaseModel): +class ClickhouseIndexConfig(BaseModel, DBCaseConfig): metric_type: MetricType | None = None + vector_data_type: str | None = "Float32" # Data type of vectors. Can be Float32 or Float64 or BFloat16 + create_index_before_load: bool = True + create_index_after_load: bool = False def parse_metric(self) -> str: if not self.metric_type: @@ -35,26 +52,38 @@ def parse_metric_str(self) -> str: return "L2Distance" if self.metric_type == MetricType.COSINE: return "cosineDistance" - msg = f"Not Support for {self.metric_type}" - raise RuntimeError(msg) - return None + return "cosineDistance" + + @abstractmethod + def session_param(self): + pass -class ClickhouseHNSWConfig(ClickhouseIndexConfig, DBCaseConfig): - M: int | None - efConstruction: int | None +class ClickhouseHNSWConfig(ClickhouseIndexConfig): + M: int | None # Default in clickhouse in 32 + efConstruction: int | None # Default in clickhouse in 128 ef: int | None = None index: IndexType = IndexType.HNSW + quantization: str | None = "bf16" # Default is bf16. Possible values are f64, f32, f16, bf16, or i8 + granularity: int | None = 10_000_000 # Size of the index granules. By default, in CH it's equal 10.000.000 def index_param(self) -> dict: return { + "vector_data_type": self.vector_data_type, "metric_type": self.parse_metric_str(), "index_type": self.index.value, + "quantization": self.quantization, + "granularity": self.granularity, "params": {"M": self.M, "efConstruction": self.efConstruction}, } def search_param(self) -> dict: return { - "met˝ric_type": self.parse_metric_str(), + "metric_type": self.parse_metric_str(), "params": {"ef": self.ef}, } + + def session_param(self) -> dict: + return { + "allow_experimental_vector_similarity_index": 1, + } From 42af186e5fe342c95f3e4b6abf5caa4789ebda8b Mon Sep 17 00:00:00 2001 From: "min.tian" Date: Wed, 30 Apr 2025 13:01:13 +0800 Subject: [PATCH 159/327] fix bugs when use custom_dataset without groundtruth file Signed-off-by: min.tian --- vectordb_bench/backend/dataset.py | 12 ++++++++---- vectordb_bench/backend/runner/serial_runner.py | 13 +++++++++---- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/vectordb_bench/backend/dataset.py b/vectordb_bench/backend/dataset.py index 62700b0fa..f90580dc6 100644 --- a/vectordb_bench/backend/dataset.py +++ b/vectordb_bench/backend/dataset.py @@ -220,10 +220,12 @@ def prepare( train_files = utils.compose_train_files(file_count, use_shuffled) all_files = train_files - gt_file, test_file = None, None + test_file = "test.parquet" + all_files.extend([test_file]) + gt_file = None if self.data.with_gt: - gt_file, test_file = utils.compose_gt_file(filters), "test.parquet" - all_files.extend([gt_file, test_file]) + gt_file = utils.compose_gt_file(filters) + all_files.extend([gt_file]) if not self.data.is_custom: source.reader().read( @@ -232,8 +234,10 @@ def prepare( local_ds_root=self.data_dir, ) - if gt_file is not None and test_file is not None: + if test_file is not None: self.test_data = self._read_file(test_file) + + if gt_file is not None: self.gt_data = self._read_file(gt_file) prefix = "shuffle_train" if use_shuffled else "train" diff --git a/vectordb_bench/backend/runner/serial_runner.py b/vectordb_bench/backend/runner/serial_runner.py index 365641132..5b418c886 100644 --- a/vectordb_bench/backend/runner/serial_runner.py +++ b/vectordb_bench/backend/runner/serial_runner.py @@ -209,7 +209,8 @@ def search(self, args: tuple[list, pd.DataFrame]) -> tuple[float, float, float]: ideal_dcg = get_ideal_dcg(self.k) log.debug(f"test dataset size: {len(test_data)}") - log.debug(f"ground truth size: {ground_truth.columns}, shape: {ground_truth.shape}") + if ground_truth is not None: + log.debug(f"ground truth size: {ground_truth.columns}, shape: {ground_truth.shape}") latencies, recalls, ndcgs = [], [], [] for idx, emb in enumerate(test_data): @@ -228,9 +229,13 @@ def search(self, args: tuple[list, pd.DataFrame]) -> tuple[float, float, float]: latencies.append(time.perf_counter() - s) - gt = ground_truth["neighbors_id"][idx] - recalls.append(calc_recall(self.k, gt[: self.k], results)) - ndcgs.append(calc_ndcg(gt[: self.k], results, ideal_dcg)) + if ground_truth is not None: + gt = ground_truth["neighbors_id"][idx] + recalls.append(calc_recall(self.k, gt[: self.k], results)) + ndcgs.append(calc_ndcg(gt[: self.k], results, ideal_dcg)) + else: + recalls.append(0) + ndcgs.append(0) if len(latencies) % 100 == 0: log.debug( From 9a912f6d1db608bab707f5ab6766ec19fcd610ed Mon Sep 17 00:00:00 2001 From: Andreas Opferkuch Date: Sat, 3 May 2025 12:46:44 +0200 Subject: [PATCH 160/327] fix: prevent the frontend from crashing on invalid indexes in results --- vectordb_bench/models.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/vectordb_bench/models.py b/vectordb_bench/models.py index ca00c4b55..e99b3c789 100644 --- a/vectordb_bench/models.py +++ b/vectordb_bench/models.py @@ -6,12 +6,15 @@ import ujson +from vectordb_bench.backend.clients.api import EmptyDBCaseConfig + from . import config from .backend.cases import CaseType from .backend.clients import ( DB, DBCaseConfig, DBConfig, + EmptyDBCaseConfig, ) from .base import BaseModel from .metric import Metric @@ -247,13 +250,21 @@ def read_file(cls, full_path: pathlib.Path, trans_unit: bool = False) -> Self: test_result["task_label"] = test_result["run_id"] for case_result in test_result["results"]: - task_config = case_result.get("task_config") - db = DB(task_config.get("db")) + task_config = case_result["task_config"] + db = DB(task_config["db"]) task_config["db_config"] = db.config_cls(**task_config["db_config"]) - task_config["db_case_config"] = db.case_config_cls( - index_type=task_config["db_case_config"].get("index", None), - )(**task_config["db_case_config"]) + + # Safely instantiate DBCaseConfig (fallback to EmptyDBCaseConfig on None) + raw_case_cfg = task_config.get("db_case_config") or {} + index_value = raw_case_cfg.get("index", None) + try: + task_config["db_case_config"] = db.case_config_cls(index_type=index_value)(**raw_case_cfg) + except: + log.error( + f"Couldn't get class for index '{index_value}' ({full_path})" + ) + task_config["db_case_config"] = EmptyDBCaseConfig(**raw_case_cfg) case_result["task_config"] = task_config From 7720bd4e2d2b5358e9f678c838462bdb7888919e Mon Sep 17 00:00:00 2001 From: Andreas Opferkuch Date: Tue, 6 May 2025 11:45:15 +0200 Subject: [PATCH 161/327] fix ruff warnings --- vectordb_bench/models.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/vectordb_bench/models.py b/vectordb_bench/models.py index e99b3c789..001469f3d 100644 --- a/vectordb_bench/models.py +++ b/vectordb_bench/models.py @@ -6,8 +6,6 @@ import ujson -from vectordb_bench.backend.clients.api import EmptyDBCaseConfig - from . import config from .backend.cases import CaseType from .backend.clients import ( @@ -260,8 +258,8 @@ def read_file(cls, full_path: pathlib.Path, trans_unit: bool = False) -> Self: index_value = raw_case_cfg.get("index", None) try: task_config["db_case_config"] = db.case_config_cls(index_type=index_value)(**raw_case_cfg) - except: - log.error( + except Exception: + log.exception( f"Couldn't get class for index '{index_value}' ({full_path})" ) task_config["db_case_config"] = EmptyDBCaseConfig(**raw_case_cfg) @@ -368,3 +366,4 @@ def append_return(x: any, y: any): tmp_logger = logging.getLogger("no_color") for f in fmt: tmp_logger.info(f) + From de9aa903a36a75b479b02abc1a98ec2550fb2fff Mon Sep 17 00:00:00 2001 From: Andreas Opferkuch Date: Tue, 6 May 2025 12:02:21 +0200 Subject: [PATCH 162/327] Fix formatting --- vectordb_bench/models.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/vectordb_bench/models.py b/vectordb_bench/models.py index 001469f3d..76ceaaddc 100644 --- a/vectordb_bench/models.py +++ b/vectordb_bench/models.py @@ -259,9 +259,7 @@ def read_file(cls, full_path: pathlib.Path, trans_unit: bool = False) -> Self: try: task_config["db_case_config"] = db.case_config_cls(index_type=index_value)(**raw_case_cfg) except Exception: - log.exception( - f"Couldn't get class for index '{index_value}' ({full_path})" - ) + log.exception(f"Couldn't get class for index '{index_value}' ({full_path})") task_config["db_case_config"] = EmptyDBCaseConfig(**raw_case_cfg) case_result["task_config"] = task_config @@ -366,4 +364,3 @@ def append_return(x: any, y: any): tmp_logger = logging.getLogger("no_color") for f in fmt: tmp_logger.info(f) - From 0122126eafaf9f328847cce01fa142655fd35fbe Mon Sep 17 00:00:00 2001 From: Andreas Opferkuch Date: Sat, 26 Apr 2025 19:07:26 +0200 Subject: [PATCH 163/327] Add lancedb --- README.md | 8 +- pyproject.toml | 2 + vectordb_bench/backend/clients/__init__.py | 20 ++- vectordb_bench/backend/clients/api.py | 1 + vectordb_bench/backend/clients/lancedb/cli.py | 92 +++++++++++++ .../backend/clients/lancedb/config.py | 103 +++++++++++++++ .../backend/clients/lancedb/lancedb.py | 91 +++++++++++++ vectordb_bench/cli/vectordbbench.py | 2 + .../frontend/config/dbCaseConfigs.py | 125 ++++++++++++++++++ vectordb_bench/frontend/config/styles.py | 1 + vectordb_bench/models.py | 3 + 11 files changed, 442 insertions(+), 6 deletions(-) create mode 100644 vectordb_bench/backend/clients/lancedb/cli.py create mode 100644 vectordb_bench/backend/clients/lancedb/config.py create mode 100644 vectordb_bench/backend/clients/lancedb/lancedb.py diff --git a/README.md b/README.md index 6d83671da..662461a73 100644 --- a/README.md +++ b/README.md @@ -267,13 +267,13 @@ pip install -e '.[pinecone]' ``` ### Run test server ``` -$ python -m vectordb_bench +python -m vectordb_bench ``` OR: ```shell -$ init_bench +init_bench ``` OR: @@ -290,13 +290,13 @@ After reopen the repository in container, run `python -m vectordb_bench` in the ### Check coding styles ```shell -$ make lint +make lint ``` To fix the coding styles automatically ```shell -$ make format +make format ``` ## How does it work? diff --git a/pyproject.toml b/pyproject.toml index 09fa66972..8cab39194 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,6 +72,7 @@ all = [ "PyMySQL", "clickhouse-connect", "pyvespa", + "lancedb", ] qdrant = [ "qdrant-client" ] @@ -94,6 +95,7 @@ mariadb = [ "mariadb" ] tidb = [ "PyMySQL" ] clickhouse = [ "clickhouse-connect" ] vespa = [ "pyvespa" ] +lancedb = [ "lancedb" ] [project.urls] "repository" = "https://github.com/zilliztech/VectorDBBench" diff --git a/vectordb_bench/backend/clients/__init__.py b/vectordb_bench/backend/clients/__init__.py index cfef0283f..f05913a06 100644 --- a/vectordb_bench/backend/clients/__init__.py +++ b/vectordb_bench/backend/clients/__init__.py @@ -45,9 +45,10 @@ class DB(Enum): TiDB = "TiDB" Clickhouse = "Clickhouse" Vespa = "Vespa" + LanceDB = "LanceDB" @property - def init_cls(self) -> type[VectorDB]: # noqa: PLR0911, PLR0912, C901 + def init_cls(self) -> type[VectorDB]: # noqa: PLR0911, PLR0912, C901, PLR0915 """Import while in use""" if self == DB.Milvus: from .milvus.milvus import Milvus @@ -164,11 +165,16 @@ def init_cls(self) -> type[VectorDB]: # noqa: PLR0911, PLR0912, C901 return Vespa + if self == DB.LanceDB: + from .lancedb.lancedb import LanceDB + + return LanceDB + msg = f"Unknown DB: {self.name}" raise ValueError(msg) @property - def config_cls(self) -> type[DBConfig]: # noqa: PLR0911, PLR0912, C901 + def config_cls(self) -> type[DBConfig]: # noqa: PLR0911, PLR0912, C901, PLR0915 """Import while in use""" if self == DB.Milvus: from .milvus.config import MilvusConfig @@ -285,6 +291,11 @@ def config_cls(self) -> type[DBConfig]: # noqa: PLR0911, PLR0912, C901 return VespaConfig + if self == DB.LanceDB: + from .lancedb.config import LanceDBConfig + + return LanceDBConfig + msg = f"Unknown DB: {self.name}" raise ValueError(msg) @@ -382,6 +393,11 @@ def case_config_cls( # noqa: C901, PLR0911, PLR0912 return VespaHNSWConfig + if self == DB.LanceDB: + from .lancedb.config import _lancedb_case_config + + return _lancedb_case_config.get(index_type) + # DB.Pinecone, DB.Chroma, DB.Redis return EmptyDBCaseConfig diff --git a/vectordb_bench/backend/clients/api.py b/vectordb_bench/backend/clients/api.py index 790da891b..ff7b378a7 100644 --- a/vectordb_bench/backend/clients/api.py +++ b/vectordb_bench/backend/clients/api.py @@ -34,6 +34,7 @@ class IndexType(str, Enum): GPU_IVF_PQ = "GPU_IVF_PQ" GPU_CAGRA = "GPU_CAGRA" SCANN = "scann" + NONE = "NONE" class SQType(str, Enum): diff --git a/vectordb_bench/backend/clients/lancedb/cli.py b/vectordb_bench/backend/clients/lancedb/cli.py new file mode 100644 index 000000000..573c64d05 --- /dev/null +++ b/vectordb_bench/backend/clients/lancedb/cli.py @@ -0,0 +1,92 @@ +from typing import Annotated, Unpack + +import click +from pydantic import SecretStr + +from ....cli.cli import ( + CommonTypedDict, + cli, + click_parameter_decorators_from_typed_dict, + run, +) +from .. import DB +from ..api import IndexType + + +class LanceDBTypedDict(CommonTypedDict): + uri: Annotated[ + str, + click.option("--uri", type=str, help="URI connection string", required=True), + ] + token: Annotated[ + str | None, + click.option("--token", type=str, help="Authentication token", required=False), + ] + + +@cli.command() +@click_parameter_decorators_from_typed_dict(LanceDBTypedDict) +def LanceDB(**parameters: Unpack[LanceDBTypedDict]): + from .config import LanceDBConfig, _lancedb_case_config + + run( + db=DB.LanceDB, + db_config=LanceDBConfig( + db_label=parameters["db_label"], + uri=parameters["uri"], + token=SecretStr(parameters["token"]) if parameters.get("token") else None, + ), + db_case_config=_lancedb_case_config.get("NONE")(), + **parameters, + ) + + +@cli.command() +@click_parameter_decorators_from_typed_dict(LanceDBTypedDict) +def LanceDBAutoIndex(**parameters: Unpack[LanceDBTypedDict]): + from .config import LanceDBConfig, _lancedb_case_config + + run( + db=DB.LanceDB, + db_config=LanceDBConfig( + db_label=parameters["db_label"], + uri=parameters["uri"], + token=SecretStr(parameters["token"]) if parameters.get("token") else None, + ), + db_case_config=_lancedb_case_config.get(IndexType.AUTOINDEX)(), + **parameters, + ) + + +@cli.command() +@click_parameter_decorators_from_typed_dict(LanceDBTypedDict) +def LanceDBIVFPQ(**parameters: Unpack[LanceDBTypedDict]): + from .config import LanceDBConfig, _lancedb_case_config + + run( + db=DB.LanceDB, + db_config=LanceDBConfig( + db_label=parameters["db_label"], + uri=parameters["uri"], + token=SecretStr(parameters["token"]) if parameters.get("token") else None, + ), + db_case_config=_lancedb_case_config.get(IndexType.IVFPQ)(), + **parameters, + ) + + +@cli.command() +@click_parameter_decorators_from_typed_dict(LanceDBTypedDict) +def LanceDBHNSW(**parameters: Unpack[LanceDBTypedDict]): + from .config import LanceDBConfig, _lancedb_case_config + + run( + db=DB.LanceDB, + db_config=LanceDBConfig( + db_label=parameters["db_label"], + uri=parameters["uri"], + token=SecretStr(parameters["token"]) if parameters.get("token") else None, + ), + db_case_config=_lancedb_case_config.get(IndexType.HNSW)(), + **parameters, + ) diff --git a/vectordb_bench/backend/clients/lancedb/config.py b/vectordb_bench/backend/clients/lancedb/config.py new file mode 100644 index 000000000..0bbdfc4c9 --- /dev/null +++ b/vectordb_bench/backend/clients/lancedb/config.py @@ -0,0 +1,103 @@ +from pydantic import BaseModel, SecretStr + +from ..api import DBCaseConfig, DBConfig, IndexType, MetricType + + +class LanceDBConfig(DBConfig): + """LanceDB connection configuration.""" + + db_label: str + uri: str + token: SecretStr | None = None + + def to_dict(self) -> dict: + return { + "uri": self.uri, + "token": self.token.get_secret_value() if self.token else None, + } + + +class LanceDBIndexConfig(BaseModel, DBCaseConfig): + index: IndexType = IndexType.IVFPQ + metric_type: MetricType = MetricType.L2 + num_partitions: int = 0 + num_sub_vectors: int = 0 + nbits: int = 8 # Must be 4 or 8 + sample_rate: int = 256 + max_iterations: int = 50 + + def index_param(self) -> dict: + if self.index not in [ + IndexType.IVFPQ, + IndexType.HNSW, + IndexType.AUTOINDEX, + IndexType.NONE, + ]: + msg = f"Index type {self.index} is not supported for LanceDB!" + raise ValueError(msg) + + # See https://lancedb.github.io/lancedb/python/python/#lancedb.table.Table.create_index + params = { + "metric": self.parse_metric(), + "num_bits": self.nbits, + "sample_rate": self.sample_rate, + "max_iterations": self.max_iterations, + } + + if self.num_partitions > 0: + params["num_partitions"] = self.num_partitions + if self.num_sub_vectors > 0: + params["num_sub_vectors"] = self.num_sub_vectors + + return params + + def search_param(self) -> dict: + pass + + def parse_metric(self) -> str: + if self.metric_type in [MetricType.L2, MetricType.COSINE]: + return self.metric_type.value.lower() + if self.metric_type in [MetricType.IP, MetricType.DP]: + return "dot" + msg = f"Metric type {self.metric_type} is not supported for LanceDB!" + raise ValueError(msg) + + +class LanceDBNoIndexConfig(LanceDBIndexConfig): + index: IndexType = IndexType.NONE + + def index_param(self) -> dict: + return {} + + +class LanceDBAutoIndexConfig(LanceDBIndexConfig): + index: IndexType = IndexType.AUTOINDEX + + def index_param(self) -> dict: + return {} + + +class LanceDBHNSWIndexConfig(LanceDBIndexConfig): + index: IndexType = IndexType.HNSW + m: int = 0 + ef_construction: int = 0 + + def index_param(self) -> dict: + params = LanceDBIndexConfig.index_param(self) + + # See https://lancedb.github.io/lancedb/python/python/#lancedb.index.HnswSq + params["index_type"] = "IVF_HNSW_SQ" + if self.m > 0: + params["m"] = self.m + if self.ef_construction > 0: + params["ef_construction"] = self.ef_construction + + return params + + +_lancedb_case_config = { + IndexType.IVFPQ: LanceDBIndexConfig, + IndexType.AUTOINDEX: LanceDBAutoIndexConfig, + IndexType.HNSW: LanceDBHNSWIndexConfig, + IndexType.NONE: LanceDBNoIndexConfig, +} diff --git a/vectordb_bench/backend/clients/lancedb/lancedb.py b/vectordb_bench/backend/clients/lancedb/lancedb.py new file mode 100644 index 000000000..d93871de2 --- /dev/null +++ b/vectordb_bench/backend/clients/lancedb/lancedb.py @@ -0,0 +1,91 @@ +import logging +from contextlib import contextmanager + +import lancedb +import pyarrow as pa +from lancedb.pydantic import LanceModel + +from ..api import IndexType, VectorDB +from .config import LanceDBConfig, LanceDBIndexConfig + +log = logging.getLogger(__name__) + + +class VectorModel(LanceModel): + id: int + vector: list[float] + + +class LanceDB(VectorDB): + def __init__( + self, + dim: int, + db_config: LanceDBConfig, + db_case_config: LanceDBIndexConfig, + collection_name: str = "vector_bench_test", + drop_old: bool = False, + **kwargs, + ): + self.name = "LanceDB" + self.db_config = db_config + self.case_config = db_case_config + self.table_name = collection_name + self.dim = dim + self.uri = db_config["uri"] + + db = lancedb.connect(self.uri) + + if drop_old: + try: + db.drop_table(self.table_name) + except Exception as e: + log.warning(f"Failed to drop table {self.table_name}: {e}") + + try: + db.open_table(self.table_name) + except Exception: + schema = pa.schema( + [pa.field("id", pa.int64()), pa.field("vector", pa.list_(pa.float64(), list_size=self.dim))] + ) + db.create_table(self.table_name, schema=schema, mode="overwrite") + + @contextmanager + def init(self): + self.db = lancedb.connect(self.uri) + self.table = self.db.open_table(self.table_name) + yield + self.db = None + self.table = None + + def insert_embeddings( + self, + embeddings: list[list[float]], + metadata: list[int], + ) -> tuple[int, Exception | None]: + try: + data = [{"id": meta, "vector": emb} for meta, emb in zip(metadata, embeddings, strict=False)] + self.table.add(data) + return len(metadata), None + except Exception as e: + log.warning(f"Failed to insert data into LanceDB table ({self.table_name}), error: {e}") + return 0, e + + def search_embedding( + self, + query: list[float], + k: int = 100, + filters: dict | None = None, + ) -> list[int]: + if filters: + results = self.table.search(query).where(f"id >= {filters['id']}", prefilter=True).limit(k).to_list() + else: + results = self.table.search(query).limit(k).to_list() + return [int(result["id"]) for result in results] + + def optimize(self, data_size: int | None = None): + if self.table and hasattr(self, "case_config") and self.case_config.index != IndexType.NONE: + log.info(f"Creating index for LanceDB table ({self.table_name})") + self.table.create_index(**self.case_config.index_param()) + # Better recall with IVF_PQ (though still bad) but breaks HNSW: https://github.com/lancedb/lancedb/issues/2369 + if self.case_config.index in (IndexType.IVFPQ, IndexType.AUTOINDEX): + self.table.optimize() diff --git a/vectordb_bench/cli/vectordbbench.py b/vectordb_bench/cli/vectordbbench.py index 1c6ff1260..210a3ccb7 100644 --- a/vectordb_bench/cli/vectordbbench.py +++ b/vectordb_bench/cli/vectordbbench.py @@ -1,6 +1,7 @@ from ..backend.clients.alloydb.cli import AlloyDBScaNN from ..backend.clients.aws_opensearch.cli import AWSOpenSearch from ..backend.clients.clickhouse.cli import Clickhouse +from ..backend.clients.lancedb.cli import LanceDB from ..backend.clients.mariadb.cli import MariaDBHNSW from ..backend.clients.memorydb.cli import MemoryDB from ..backend.clients.milvus.cli import MilvusAutoIndex @@ -33,6 +34,7 @@ cli.add_command(TiDB) cli.add_command(Clickhouse) cli.add_command(Vespa) +cli.add_command(LanceDB) if __name__ == "__main__": diff --git a/vectordb_bench/frontend/config/dbCaseConfigs.py b/vectordb_bench/frontend/config/dbCaseConfigs.py index da5e91d91..3c9430b2b 100644 --- a/vectordb_bench/frontend/config/dbCaseConfigs.py +++ b/vectordb_bench/frontend/config/dbCaseConfigs.py @@ -1491,6 +1491,127 @@ class CaseConfigInput(BaseModel): ] VespaPerformanceConfig = VespaLoadingConfig +CaseConfigParamInput_IndexType_LanceDB = CaseConfigInput( + label=CaseConfigParamType.IndexType, + inputHelp="AUTOINDEX = IVFPQ with default parameters", + inputType=InputType.Option, + inputConfig={ + "options": [ + IndexType.NONE.value, + IndexType.AUTOINDEX.value, + IndexType.IVFPQ.value, + IndexType.HNSW.value, + ], + }, +) + +CaseConfigParamInput_num_partitions_LanceDB = CaseConfigInput( + label=CaseConfigParamType.num_partitions, + displayLabel="Number of Partitions", + inputHelp="Number of partitions (clusters) for IVF_PQ. Default (when 0): sqrt(num_rows)", + inputType=InputType.Number, + inputConfig={ + "min": 0, + "max": 10000, + "value": 0, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.IVFPQ.value + or config.get(CaseConfigParamType.IndexType, None) == IndexType.HNSW.value, +) + +CaseConfigParamInput_num_sub_vectors_LanceDB = CaseConfigInput( + label=CaseConfigParamType.num_sub_vectors, + displayLabel="Number of Sub-vectors", + inputHelp="Number of sub-vectors for PQ. Default (when 0): dim/16 or dim/8", + inputType=InputType.Number, + inputConfig={ + "min": 0, + "max": 1000, + "value": 0, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.IVFPQ.value + or config.get(CaseConfigParamType.IndexType, None) == IndexType.HNSW.value, +) + +CaseConfigParamInput_num_bits_LanceDB = CaseConfigInput( + label=CaseConfigParamType.nbits, + displayLabel="Number of Bits", + inputHelp="Number of bits per sub-vector.", + inputType=InputType.Option, + inputConfig={ + "options": [4, 8], + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.IVFPQ.value + or config.get(CaseConfigParamType.IndexType, None) == IndexType.HNSW.value, +) + +CaseConfigParamInput_sample_rate_LanceDB = CaseConfigInput( + label=CaseConfigParamType.sample_rate, + displayLabel="Sample Rate", + inputHelp="Sample rate for training. Higher values are more accurate but slower", + inputType=InputType.Number, + inputConfig={ + "min": 16, + "max": 1024, + "value": 256, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.IVFPQ.value + or config.get(CaseConfigParamType.IndexType, None) == IndexType.HNSW.value, +) + +CaseConfigParamInput_max_iterations_LanceDB = CaseConfigInput( + label=CaseConfigParamType.max_iterations, + displayLabel="Max Iterations", + inputHelp="Maximum iterations for k-means clustering", + inputType=InputType.Number, + inputConfig={ + "min": 10, + "max": 200, + "value": 50, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.IVFPQ.value + or config.get(CaseConfigParamType.IndexType, None) == IndexType.HNSW.value, +) + +CaseConfigParamInput_m_LanceDB = CaseConfigInput( + label=CaseConfigParamType.m, + displayLabel="m", + inputHelp="m parameter in HNSW", + inputType=InputType.Number, + inputConfig={ + "min": 0, + "max": 1000, + "value": 0, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.HNSW.value, +) + +CaseConfigParamInput_ef_construction_LanceDB = CaseConfigInput( + label=CaseConfigParamType.ef_construction, + displayLabel="ef_construction", + inputHelp="ef_construction parameter in HNSW", + inputType=InputType.Number, + inputConfig={ + "min": 0, + "max": 1000, + "value": 0, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.HNSW.value, +) + +LanceDBLoadConfig = [ + CaseConfigParamInput_IndexType_LanceDB, + CaseConfigParamInput_num_partitions_LanceDB, + CaseConfigParamInput_num_sub_vectors_LanceDB, + CaseConfigParamInput_num_bits_LanceDB, + CaseConfigParamInput_sample_rate_LanceDB, + CaseConfigParamInput_max_iterations_LanceDB, + CaseConfigParamInput_m_LanceDB, + CaseConfigParamInput_ef_construction_LanceDB, +] + +LanceDBPerformanceConfig = LanceDBLoadConfig + CASE_CONFIG_MAP = { DB.Milvus: { CaseLabel.Load: MilvusLoadConfig, @@ -1551,4 +1672,8 @@ class CaseConfigInput(BaseModel): CaseLabel.Load: VespaLoadingConfig, CaseLabel.Performance: VespaPerformanceConfig, }, + DB.LanceDB: { + CaseLabel.Load: LanceDBLoadConfig, + CaseLabel.Performance: LanceDBPerformanceConfig, + }, } diff --git a/vectordb_bench/frontend/config/styles.py b/vectordb_bench/frontend/config/styles.py index 03bda0fec..96a5eede4 100644 --- a/vectordb_bench/frontend/config/styles.py +++ b/vectordb_bench/frontend/config/styles.py @@ -49,6 +49,7 @@ def getPatternShape(i): DB.AWSOpenSearch: "https://assets.zilliz.com/opensearch_1eee37584e.jpeg", DB.TiDB: "https://img2.pingcap.com/forms/3/d/3d7fd5f9767323d6f037795704211ac44b4923d6.png", DB.Vespa: "https://vespa.ai/vespa-content/uploads/2025/01/Vespa-symbol-green-rgb.png.webp", + DB.LanceDB: "https://raw.githubusercontent.com/lancedb/lancedb/main/docs/src/assets/logo.png", } # RedisCloud color: #0D6EFD diff --git a/vectordb_bench/models.py b/vectordb_bench/models.py index 76ceaaddc..997831a00 100644 --- a/vectordb_bench/models.py +++ b/vectordb_bench/models.py @@ -97,6 +97,9 @@ class CaseConfigParamType(Enum): maxNumPrefetchDatasets = "max_num_prefetch_datasets" storage_engine = "storage_engine" max_cache_size = "max_cache_size" + num_partitions = "num_partitions" + num_sub_vectors = "num_sub_vectors" + sample_rate = "sample_rate" # mongodb params mongodb_quantization_type = "quantization" From 75bbdfb99d019079b4a2e6d22c0982882cf3624e Mon Sep 17 00:00:00 2001 From: LoveYou3000 <760583490@qq.com> Date: Thu, 8 May 2025 00:09:46 +0800 Subject: [PATCH 164/327] Add --task-label option for cli (#517) * Add --task-label option for cli * Fix lint issues --- vectordb_bench/cli/cli.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/vectordb_bench/cli/cli.py b/vectordb_bench/cli/cli.py index 4b42a912c..3ec3c18cd 100644 --- a/vectordb_bench/cli/cli.py +++ b/vectordb_bench/cli/cli.py @@ -405,6 +405,7 @@ class CommonTypedDict(TypedDict): show_default=True, ), ] + task_label: Annotated[str, click.option("--task-label", help="Task label")] class HNSWBaseTypedDict(TypedDict): @@ -499,10 +500,11 @@ def run( parameters["search_concurrent"], ), ) + task_label = parameters["task_label"] log.info(f"Task:\n{pformat(task)}\n") if not parameters["dry_run"]: - benchmark_runner.run([task]) + benchmark_runner.run([task], task_label) time.sleep(5) if global_result_future: wait([global_result_future]) From 67f0f2edaacec01c2f187c800380089efb1613b2 Mon Sep 17 00:00:00 2001 From: Andreas Opferkuch Date: Tue, 6 May 2025 20:53:30 +0200 Subject: [PATCH 165/327] Add qdrant cli --- .../backend/clients/qdrant_cloud/cli.py | 43 +++++++++++++++++++ .../backend/clients/qdrant_cloud/config.py | 8 ++-- vectordb_bench/cli/vectordbbench.py | 2 + .../components/run_test/dbConfigSetting.py | 14 ++++-- 4 files changed, 59 insertions(+), 8 deletions(-) create mode 100644 vectordb_bench/backend/clients/qdrant_cloud/cli.py diff --git a/vectordb_bench/backend/clients/qdrant_cloud/cli.py b/vectordb_bench/backend/clients/qdrant_cloud/cli.py new file mode 100644 index 000000000..32353472b --- /dev/null +++ b/vectordb_bench/backend/clients/qdrant_cloud/cli.py @@ -0,0 +1,43 @@ +from typing import Annotated, Unpack + +import click +from pydantic import SecretStr + +from ....cli.cli import ( + CommonTypedDict, + cli, + click_parameter_decorators_from_typed_dict, + run, +) +from .. import DB + + +class QdrantTypedDict(CommonTypedDict): + url: Annotated[ + str, + click.option("--url", type=str, help="URL connection string", required=True), + ] + api_key: Annotated[ + str | None, + click.option("--api-key", type=str, help="API key for authentication", required=False), + ] + + +@cli.command() +@click_parameter_decorators_from_typed_dict(QdrantTypedDict) +def QdrantCloud(**parameters: Unpack[QdrantTypedDict]): + from .config import QdrantConfig, QdrantIndexConfig + + config_params = { + "db_label": parameters["db_label"], + "url": SecretStr(parameters["url"]), + } + + config_params["api_key"] = SecretStr(parameters["api_key"]) if parameters["api_key"] else None + + run( + db=DB.QdrantCloud, + db_config=QdrantConfig(**config_params), + db_case_config=QdrantIndexConfig(), + **parameters, + ) diff --git a/vectordb_bench/backend/clients/qdrant_cloud/config.py b/vectordb_bench/backend/clients/qdrant_cloud/config.py index d4e27cb3c..b60733bc3 100644 --- a/vectordb_bench/backend/clients/qdrant_cloud/config.py +++ b/vectordb_bench/backend/clients/qdrant_cloud/config.py @@ -6,14 +6,14 @@ # Allowing `api_key` to be left empty, to ensure compatibility with the open-source Qdrant. class QdrantConfig(DBConfig): url: SecretStr - api_key: SecretStr + api_key: SecretStr | None = None def to_dict(self) -> dict: - api_key = self.api_key.get_secret_value() - if len(api_key) > 0: + api_key_value = self.api_key.get_secret_value() if self.api_key else None + if api_key_value: return { "url": self.url.get_secret_value(), - "api_key": self.api_key.get_secret_value(), + "api_key": api_key_value, "prefer_grpc": True, } return { diff --git a/vectordb_bench/cli/vectordbbench.py b/vectordb_bench/cli/vectordbbench.py index 210a3ccb7..d4153bc1e 100644 --- a/vectordb_bench/cli/vectordbbench.py +++ b/vectordb_bench/cli/vectordbbench.py @@ -9,6 +9,7 @@ from ..backend.clients.pgvecto_rs.cli import PgVectoRSHNSW, PgVectoRSIVFFlat from ..backend.clients.pgvector.cli import PgVectorHNSW from ..backend.clients.pgvectorscale.cli import PgVectorScaleDiskAnn +from ..backend.clients.qdrant_cloud.cli import QdrantCloud from ..backend.clients.redis.cli import Redis from ..backend.clients.test.cli import Test from ..backend.clients.tidb.cli import TiDB @@ -35,6 +36,7 @@ cli.add_command(Clickhouse) cli.add_command(Vespa) cli.add_command(LanceDB) +cli.add_command(QdrantCloud) if __name__ == "__main__": diff --git a/vectordb_bench/frontend/components/run_test/dbConfigSetting.py b/vectordb_bench/frontend/components/run_test/dbConfigSetting.py index 800e6dede..a2d2de77f 100644 --- a/vectordb_bench/frontend/components/run_test/dbConfigSetting.py +++ b/vectordb_bench/frontend/components/run_test/dbConfigSetting.py @@ -36,21 +36,27 @@ def dbConfigSettingItem(st, activeDb: DB): columns = st.columns(DB_CONFIG_SETTING_COLUMNS) dbConfigClass = activeDb.config_cls - properties = dbConfigClass.schema().get("properties") + schema = dbConfigClass.schema() + property_items = schema.get("properties").items() + required_fields = set(schema.get("required", [])) dbConfig = {} idx = 0 # db config (unique) - for key, property in properties.items(): + for key, property in property_items: if key not in dbConfigClass.common_short_configs() and key not in dbConfigClass.common_long_configs(): column = columns[idx % DB_CONFIG_SETTING_COLUMNS] idx += 1 - dbConfig[key] = column.text_input( + input_value = column.text_input( key, - key="%s-%s" % (activeDb.name, key), + key=f"{activeDb.name}-{key}", value=property.get("default", ""), type="password" if inputIsPassword(key) else "default", + placeholder="optional" if key not in required_fields else None, ) + if key in required_fields or input_value: + dbConfig[key] = input_value + # db config (common short labels) for key in dbConfigClass.common_short_configs(): column = columns[idx % DB_CONFIG_SETTING_COLUMNS] From 2b966c39a884a38928244b1fcfa4a08985c2c58d Mon Sep 17 00:00:00 2001 From: Yuyuan Kang <36235611+yuyuankang@users.noreply.github.com> Date: Sun, 11 May 2025 22:43:17 -0500 Subject: [PATCH 166/327] Update README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 662461a73..a17220749 100644 --- a/README.md +++ b/README.md @@ -200,7 +200,8 @@ Options: # Memory Management --cb-threshold TEXT k-NN Memory circuit breaker threshold - --help Show this message and exit.``` + --help Show this message and exit. + ``` #### Using a configuration file. From 4efbe83c73358dcd18833f613203b5797e218c57 Mon Sep 17 00:00:00 2001 From: Yuyuan Kang <36235611+yuyuankang@users.noreply.github.com> Date: Tue, 13 May 2025 01:39:56 -0500 Subject: [PATCH 167/327] Fixing Bugs in Benchmarking ClickHouse with vectordbbench (#523) * Update cli.py * Update clickhouse.py * Update clickhouse.py * Update cli.py * Update config.py * remove space --- vectordb_bench/backend/clients/clickhouse/cli.py | 1 + vectordb_bench/backend/clients/clickhouse/clickhouse.py | 6 +++--- vectordb_bench/backend/clients/clickhouse/config.py | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/vectordb_bench/backend/clients/clickhouse/cli.py b/vectordb_bench/backend/clients/clickhouse/cli.py index 6fc1a84d7..4b50bc55b 100644 --- a/vectordb_bench/backend/clients/clickhouse/cli.py +++ b/vectordb_bench/backend/clients/clickhouse/cli.py @@ -51,6 +51,7 @@ def Clickhouse(**parameters: Unpack[ClickhouseHNSWTypedDict]): db=DB.Clickhouse, db_config=ClickhouseConfig( db_label=parameters["db_label"], + user=parameters["user"], password=SecretStr(parameters["password"]) if parameters["password"] else None, host=parameters["host"], port=parameters["port"], diff --git a/vectordb_bench/backend/clients/clickhouse/clickhouse.py b/vectordb_bench/backend/clients/clickhouse/clickhouse.py index 498132ca9..de09895a8 100644 --- a/vectordb_bench/backend/clients/clickhouse/clickhouse.py +++ b/vectordb_bench/backend/clients/clickhouse/clickhouse.py @@ -106,7 +106,7 @@ def _create_index(self): query = f""" ALTER TABLE {self.db_config["database"]}.{self.table_name} ADD INDEX {self._index_name} {self._vector_field} - TYPE vector_similarity('hnsw', '{self.index_param["metric_type"]}', + TYPE vector_similarity('hnsw', '{self.index_param["metric_type"]}',{self.dim}, '{self.index_param["quantization"]}', {self.index_param["params"]["M"]}, {self.index_param["params"]["efConstruction"]}) GRANULARITY {self.index_param["granularity"]} @@ -115,7 +115,7 @@ def _create_index(self): query = f""" ALTER TABLE {self.db_config["database"]}.{self.table_name} ADD INDEX {self._index_name} {self._vector_field} - TYPE vector_similarity('hnsw', '{self.index_param["metric_type"]}') + TYPE vector_similarity('hnsw', '{self.index_param["metric_type"]}', {self.dim}) GRANULARITY {self.index_param["granularity"]} """ self.conn.command(cmd=query) @@ -186,7 +186,7 @@ def search_embedding( "vector_field": self._vector_field, "schema": self.db_config["database"], "table": self.table_name, - "gt": filters.get("id"), + "gt": 0 if filters is None else filters.get("id", 0), "k": k, "metric_type": self.search_param["metric_type"], "query": query, diff --git a/vectordb_bench/backend/clients/clickhouse/config.py b/vectordb_bench/backend/clients/clickhouse/config.py index a4c5fe499..f9e09812b 100644 --- a/vectordb_bench/backend/clients/clickhouse/config.py +++ b/vectordb_bench/backend/clients/clickhouse/config.py @@ -16,7 +16,7 @@ class ClickhouseConfigDict(TypedDict): class ClickhouseConfig(DBConfig): - user_name: str = "clickhouse" + user: str = "clickhouse" password: SecretStr host: str = "localhost" port: int = 8123 @@ -29,7 +29,7 @@ def to_dict(self) -> ClickhouseConfigDict: "host": self.host, "port": self.port, "database": self.db_name, - "user": self.user_name, + "user": self.user, "password": pwd_str, "secure": self.secure, } From 9f5ea99a17d137b1e7b6b9c859f526b28c5ae2b2 Mon Sep 17 00:00:00 2001 From: LoveYou3000 <760583490@qq.com> Date: Wed, 14 May 2025 12:03:09 +0800 Subject: [PATCH 168/327] Add --concurrency-timeout option to avoid long time waiting (#521) * Add --concurrency-timeout option to avoid long time waiting, by default, it's 3600s. * Fix lint error * Update README.md, add --concurrency-timeout option --- README.md | 4 ++++ vectordb_bench/__init__.py | 4 +++- vectordb_bench/backend/runner/mp_runner.py | 21 ++++++++++++++++----- vectordb_bench/backend/task_runner.py | 1 + vectordb_bench/cli/cli.py | 15 +++++++++++++-- vectordb_bench/models.py | 6 ++++++ 6 files changed, 43 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index a17220749..3d38d9444 100644 --- a/README.md +++ b/README.md @@ -114,6 +114,10 @@ Options: --num-concurrency TEXT Comma-separated list of concurrency values to test during concurrent search [default: 1,10,20] + --concurrency-timeout INTEGER Timeout (in seconds) to wait for a + concurrency slot before failing. Set to a + negative value to wait indefinitely. + [default: 3600] --user-name TEXT Db username [required] --password TEXT Db password [required] --host TEXT Db host [required] diff --git a/vectordb_bench/__init__.py b/vectordb_bench/__init__.py index c07fc855d..52c2094b4 100644 --- a/vectordb_bench/__init__.py +++ b/vectordb_bench/__init__.py @@ -6,7 +6,7 @@ from . import log_util env = environs.Env() -env.read_env(".env", False) +env.read_env(path=".env", recurse=False) class config: @@ -52,6 +52,8 @@ class config: CONCURRENCY_DURATION = 30 + CONCURRENCY_TIMEOUT = 3600 + RESULTS_LOCAL_DIR = env.path( "RESULTS_LOCAL_DIR", pathlib.Path(__file__).parent.joinpath("results"), diff --git a/vectordb_bench/backend/runner/mp_runner.py b/vectordb_bench/backend/runner/mp_runner.py index 687a0ecd7..fd87d7ece 100644 --- a/vectordb_bench/backend/runner/mp_runner.py +++ b/vectordb_bench/backend/runner/mp_runner.py @@ -5,10 +5,12 @@ import time import traceback from collections.abc import Iterable +from multiprocessing.queues import Queue import numpy as np from ... import config +from ...models import ConcurrencySlotTimeoutError from ..clients import api NUM_PER_BATCH = config.NUM_PER_BATCH @@ -28,16 +30,18 @@ def __init__( self, db: api.VectorDB, test_data: list[list[float]], - k: int = 100, + k: int = config.K_DEFAULT, filters: dict | None = None, concurrencies: Iterable[int] = config.NUM_CONCURRENCY, - duration: int = 30, + duration: int = config.CONCURRENCY_DURATION, + concurrency_timeout: int = config.CONCURRENCY_TIMEOUT, ): self.db = db self.k = k self.filters = filters self.concurrencies = concurrencies self.duration = duration + self.concurrency_timeout = concurrency_timeout self.test_data = test_data log.debug(f"test dataset columns: {len(test_data)}") @@ -114,9 +118,7 @@ def _run_all_concurrencies_mem_efficient(self): log.info(f"Start search {self.duration}s in concurrency {conc}, filters: {self.filters}") future_iter = [executor.submit(self.search, self.test_data, q, cond) for i in range(conc)] # Sync all processes - while q.qsize() < conc: - sleep_t = conc if conc < 10 else 10 - time.sleep(sleep_t) + self._wait_for_queue_fill(q, size=conc) with cond: cond.notify_all() @@ -160,6 +162,15 @@ def _run_all_concurrencies_mem_efficient(self): conc_latency_avg_list, ) + def _wait_for_queue_fill(self, q: Queue, size: int): + wait_t = 0 + while q.qsize() < size: + sleep_t = size if size < 10 else 10 + wait_t += sleep_t + if wait_t > self.concurrency_timeout > 0: + raise ConcurrencySlotTimeoutError + time.sleep(sleep_t) + def run(self) -> float: """ Returns: diff --git a/vectordb_bench/backend/task_runner.py b/vectordb_bench/backend/task_runner.py index 2a583b4f5..1da3bb4cd 100644 --- a/vectordb_bench/backend/task_runner.py +++ b/vectordb_bench/backend/task_runner.py @@ -275,6 +275,7 @@ def _init_search_runner(self): filters=self.ca.filters, concurrencies=self.config.case_config.concurrency_search_config.num_concurrency, duration=self.config.case_config.concurrency_search_config.concurrency_duration, + concurrency_timeout=self.config.case_config.concurrency_search_config.concurrency_timeout, k=self.config.case_config.k, ) diff --git a/vectordb_bench/cli/cli.py b/vectordb_bench/cli/cli.py index 3ec3c18cd..1b0eb295b 100644 --- a/vectordb_bench/cli/cli.py +++ b/vectordb_bench/cli/cli.py @@ -17,10 +17,9 @@ import click from yaml import load -from vectordb_bench.backend.clients.api import MetricType - from .. import config from ..backend.clients import DB +from ..backend.clients.api import MetricType from ..interface import benchmark_runner, global_result_future from ..models import ( CaseConfig, @@ -303,6 +302,17 @@ class CommonTypedDict(TypedDict): callback=lambda *args: list(map(int, click_arg_split(*args))), ), ] + concurrency_timeout: Annotated[ + int, + click.option( + "--concurrency-timeout", + type=int, + default=config.CONCURRENCY_TIMEOUT, + show_default=True, + help="Timeout (in seconds) to wait for a concurrency slot before failing. " + "Set to a negative value to wait indefinitely.", + ), + ] custom_case_name: Annotated[ str, click.option( @@ -490,6 +500,7 @@ def run( concurrency_search_config=ConcurrencySearchConfig( concurrency_duration=parameters["concurrency_duration"], num_concurrency=[int(s) for s in parameters["num_concurrency"]], + concurrency_timeout=parameters["concurrency_timeout"], ), custom_case=get_custom_case_config(parameters), ), diff --git a/vectordb_bench/models.py b/vectordb_bench/models.py index 997831a00..c35c21755 100644 --- a/vectordb_bench/models.py +++ b/vectordb_bench/models.py @@ -30,6 +30,11 @@ def __init__(self): super().__init__("Performance case optimize timeout") +class ConcurrencySlotTimeoutError(TimeoutError): + def __init__(self): + super().__init__("Timeout while waiting for a concurrency slot to become available") + + class CaseConfigParamType(Enum): """ Value will be the key of CaseConfig.params and displayed in UI @@ -113,6 +118,7 @@ class CustomizedCase(BaseModel): class ConcurrencySearchConfig(BaseModel): num_concurrency: list[int] = config.NUM_CONCURRENCY concurrency_duration: int = config.CONCURRENCY_DURATION + concurrency_timeout: int = config.CONCURRENCY_TIMEOUT class CaseConfig(BaseModel): From aa13197931764a3c95a335481eade90c02e0f2db Mon Sep 17 00:00:00 2001 From: "min.tian" Date: Thu, 15 May 2025 18:08:21 +0800 Subject: [PATCH 169/327] add alias: VDBBench Signed-off-by: min.tian --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3d38d9444..35c66cac4 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ -# VectorDBBench: A Benchmark Tool for VectorDB +# VectorDBBench(VDBBench): A Benchmark Tool for VectorDB [![version](https://img.shields.io/pypi/v/vectordb-bench.svg?color=blue)](https://pypi.org/project/vectordb-bench/) [![Downloads](https://pepy.tech/badge/vectordb-bench)](https://pepy.tech/project/vectordb-bench) ## What is VectorDBBench -VectorDBBench is not just an offering of benchmark results for mainstream vector databases and cloud services, it's your go-to tool for the ultimate performance and cost-effectiveness comparison. Designed with ease-of-use in mind, VectorDBBench is devised to help users, even non-professionals, reproduce results or test new systems, making the hunt for the optimal choice amongst a plethora of cloud services and open-source vector databases a breeze. +VectorDBBench(VDBBench) is not just an offering of benchmark results for mainstream vector databases and cloud services, it's your go-to tool for the ultimate performance and cost-effectiveness comparison. Designed with ease-of-use in mind, VectorDBBench is devised to help users, even non-professionals, reproduce results or test new systems, making the hunt for the optimal choice amongst a plethora of cloud services and open-source vector databases a breeze. Understanding the importance of user experience, we provide an intuitive visual interface. This not only empowers users to initiate benchmarks at ease, but also to view comparative result reports, thereby reproducing benchmark results effortlessly. To add more relevance and practicality, we provide cost-effectiveness reports particularly for cloud services. This allows for a more realistic and applicable benchmarking process. From 7a1dc5e6e828e50d8ebb015bb03908340437273d Mon Sep 17 00:00:00 2001 From: Andreas Opferkuch Date: Thu, 15 May 2025 22:30:37 +0200 Subject: [PATCH 170/327] LanceDB: Improve serial latency by only selecting id --- vectordb_bench/backend/clients/lancedb/lancedb.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/vectordb_bench/backend/clients/lancedb/lancedb.py b/vectordb_bench/backend/clients/lancedb/lancedb.py index d93871de2..334a7ccf8 100644 --- a/vectordb_bench/backend/clients/lancedb/lancedb.py +++ b/vectordb_bench/backend/clients/lancedb/lancedb.py @@ -77,9 +77,15 @@ def search_embedding( filters: dict | None = None, ) -> list[int]: if filters: - results = self.table.search(query).where(f"id >= {filters['id']}", prefilter=True).limit(k).to_list() + results = ( + self.table.search(query) + .select(["id"]) + .where(f"id >= {filters['id']}", prefilter=True) + .limit(k) + .to_list() + ) else: - results = self.table.search(query).limit(k).to_list() + results = self.table.search(query).select(["id"]).limit(k).to_list() return [int(result["id"]) for result in results] def optimize(self, data_size: int | None = None): From a8980952b25d0e8ae010b352fd60b74655432aac Mon Sep 17 00:00:00 2001 From: LoveYou3000 <760583490@qq.com> Date: Tue, 20 May 2025 18:17:10 +0800 Subject: [PATCH 171/327] add --num-shards option for milvus performance test case (#526) * add --num-shards option for milvus performance test case * fix lint warning --- vectordb_bench/backend/clients/milvus/cli.py | 20 +++++++++++++++++++ .../backend/clients/milvus/config.py | 1 + .../backend/clients/milvus/milvus.py | 8 +++++++- 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/vectordb_bench/backend/clients/milvus/cli.py b/vectordb_bench/backend/clients/milvus/cli.py index 24a61566f..b9394cd06 100644 --- a/vectordb_bench/backend/clients/milvus/cli.py +++ b/vectordb_bench/backend/clients/milvus/cli.py @@ -29,6 +29,17 @@ class MilvusTypedDict(TypedDict): str | None, click.option("--password", type=str, help="Db password", required=False), ] + num_shards: Annotated[ + int, + click.option( + "--num-shards", + type=int, + help="Number of shards", + required=False, + default=1, + show_default=True, + ), + ] class MilvusAutoIndexTypedDict(CommonTypedDict, MilvusTypedDict): ... @@ -46,6 +57,7 @@ def MilvusAutoIndex(**parameters: Unpack[MilvusAutoIndexTypedDict]): uri=SecretStr(parameters["uri"]), user=parameters["user_name"], password=SecretStr(parameters["password"]), + num_shards=int(parameters["num_shards"]), ), db_case_config=AutoIndexConfig(), **parameters, @@ -64,6 +76,7 @@ def MilvusFlat(**parameters: Unpack[MilvusAutoIndexTypedDict]): uri=SecretStr(parameters["uri"]), user=parameters["user_name"], password=SecretStr(parameters["password"]), + num_shards=int(parameters["num_shards"]), ), db_case_config=FLATConfig(), **parameters, @@ -110,6 +123,7 @@ def MilvusIVFFlat(**parameters: Unpack[MilvusIVFFlatTypedDict]): uri=SecretStr(parameters["uri"]), user=parameters["user_name"], password=SecretStr(parameters["password"]), + num_shards=int(parameters["num_shards"]), ), db_case_config=IVFFlatConfig( nlist=parameters["nlist"], @@ -131,6 +145,7 @@ def MilvusIVFSQ8(**parameters: Unpack[MilvusIVFFlatTypedDict]): uri=SecretStr(parameters["uri"]), user=parameters["user_name"], password=SecretStr(parameters["password"]), + num_shards=int(parameters["num_shards"]), ), db_case_config=IVFSQ8Config( nlist=parameters["nlist"], @@ -156,6 +171,7 @@ def MilvusDISKANN(**parameters: Unpack[MilvusDISKANNTypedDict]): uri=SecretStr(parameters["uri"]), user=parameters["user_name"], password=SecretStr(parameters["password"]), + num_shards=int(parameters["num_shards"]), ), db_case_config=DISKANNConfig( search_list=parameters["search_list"], @@ -184,6 +200,7 @@ def MilvusGPUIVFFlat(**parameters: Unpack[MilvusGPUIVFTypedDict]): uri=SecretStr(parameters["uri"]), user=parameters["user_name"], password=SecretStr(parameters["password"]), + num_shards=int(parameters["num_shards"]), ), db_case_config=GPUIVFFlatConfig( nlist=parameters["nlist"], @@ -218,6 +235,7 @@ def MilvusGPUBruteForce(**parameters: Unpack[MilvusGPUBruteForceTypedDict]): uri=SecretStr(parameters["uri"]), user=parameters["user_name"], password=SecretStr(parameters["password"]), + num_shards=int(parameters["num_shards"]), ), db_case_config=GPUBruteForceConfig( metric_type=parameters["metric_type"], @@ -249,6 +267,7 @@ def MilvusGPUIVFPQ(**parameters: Unpack[MilvusGPUIVFPQTypedDict]): uri=SecretStr(parameters["uri"]), user=parameters["user_name"], password=SecretStr(parameters["password"]), + num_shards=int(parameters["num_shards"]), ), db_case_config=GPUIVFPQConfig( nlist=parameters["nlist"], @@ -288,6 +307,7 @@ def MilvusGPUCAGRA(**parameters: Unpack[MilvusGPUCAGRATypedDict]): uri=SecretStr(parameters["uri"]), user=parameters["user_name"], password=SecretStr(parameters["password"]), + num_shards=int(parameters["num_shards"]), ), db_case_config=GPUCAGRAConfig( intermediate_graph_degree=parameters["intermediate_graph_degree"], diff --git a/vectordb_bench/backend/clients/milvus/config.py b/vectordb_bench/backend/clients/milvus/config.py index 672becf1b..bf5920817 100644 --- a/vectordb_bench/backend/clients/milvus/config.py +++ b/vectordb_bench/backend/clients/milvus/config.py @@ -7,6 +7,7 @@ class MilvusConfig(DBConfig): uri: SecretStr = "http://localhost:19530" user: str | None = None password: SecretStr | None = None + num_shards: int = 1 def to_dict(self) -> dict: return { diff --git a/vectordb_bench/backend/clients/milvus/milvus.py b/vectordb_bench/backend/clients/milvus/milvus.py index 465c51179..7ea7308c5 100644 --- a/vectordb_bench/backend/clients/milvus/milvus.py +++ b/vectordb_bench/backend/clients/milvus/milvus.py @@ -40,7 +40,12 @@ def __init__( from pymilvus import connections - connections.connect(**self.db_config, timeout=30) + connections.connect( + uri=self.db_config.get("uri"), + user=self.db_config.get("user"), + password=self.db_config.get("password"), + timeout=30, + ) if drop_old and utility.has_collection(self.collection_name): log.info(f"{self.name} client drop_old collection: {self.collection_name}") utility.drop_collection(self.collection_name) @@ -59,6 +64,7 @@ def __init__( name=self.collection_name, schema=CollectionSchema(fields), consistency_level="Session", + num_shards=self.db_config.get("num_shards"), ) log.info(f"{self.name} create index: index_params: {self.case_config.index_param()}") From bf03df3d4b606859cda0d734de666de6585cd2c0 Mon Sep 17 00:00:00 2001 From: LoveYou3000 <760583490@qq.com> Date: Mon, 9 Jun 2025 09:27:49 +0800 Subject: [PATCH 172/327] Add a batch cli to support the batch execution of multiple cases. (#530) * Add a batch cli to support the batch execution of multiple cases. * Add how to use --- README.md | 43 +++++++ vectordb_bench/cli/batch_cli.py | 121 ++++++++++++++++++ vectordb_bench/cli/vectordbbench.py | 2 + .../config-files/batch_sample_config.yml | 17 +++ 4 files changed, 183 insertions(+) create mode 100644 vectordb_bench/cli/batch_cli.py create mode 100644 vectordb_bench/config-files/batch_sample_config.yml diff --git a/README.md b/README.md index 35c66cac4..6cb3cf7d8 100644 --- a/README.md +++ b/README.md @@ -245,6 +245,49 @@ milvushnsw: > - Options passed on the command line will override the configuration file* > - Parameter names use an _ not - +#### Using a batch configuration file. + +The vectordbbench command can read a batch configuration file to run all the test cases in the yaml formatted configuration file. + +By default, configuration files are expected to be in vectordb_bench/config-files/, this can be overridden by setting +the environment variable CONFIG_LOCAL_DIR or by passing the full path to the file. + +The required format is: +```yaml +commandname: + - parameter_name: parameter_value + another_parameter_name: parameter_value +``` +Example: +```yaml +pgvectorhnsw: + - db_label: pgConfigTest + user_name: vectordbbench + password: vectordbbench + db_name: vectordbbench + host: localhost + m: 16 + ef_construction: 128 + ef_search: 128 +milvushnsw: + - skip_search_serial: True + case_type: Performance1536D50K + uri: http://localhost:19530 + m: 16 + ef_construction: 128 + ef_search: 128 + drop_old: False + load: False +``` +> Notes: +> - Options can only be passed through configuration files +> - Parameter names use an _ not - + +How to use? +```shell +vectordbbench batchcli --batch-config-file +``` + ## Leaderboard ### Introduction To facilitate the presentation of test results and provide a comprehensive performance analysis report, we offer a [leaderboard page](https://zilliz.com/benchmark). It allows us to choose from QPS, QP$, and latency metrics, and provides a comprehensive assessment of a system's performance based on the test results of various cases and a set of scoring mechanisms (to be introduced later). On this leaderboard, we can select the systems and models to be compared, and filter out cases we do not want to consider. Comprehensive scores are always ranked from best to worst, and the specific test results of each query will be presented in the list below. diff --git a/vectordb_bench/cli/batch_cli.py b/vectordb_bench/cli/batch_cli.py new file mode 100644 index 000000000..5ac2b1cf1 --- /dev/null +++ b/vectordb_bench/cli/batch_cli.py @@ -0,0 +1,121 @@ +import logging +import time +from collections.abc import MutableMapping +from concurrent.futures import wait +from pathlib import Path +from typing import Annotated, Any, TypedDict + +import click +from click.testing import CliRunner +from yaml import Loader, load + +from .. import config +from ..cli.cli import ( + cli, + click_parameter_decorators_from_typed_dict, +) + +log = logging.getLogger(__name__) + + +def click_get_defaults_from_file(ctx, param, value): # noqa: ANN001, ARG001 + if not value: + raise click.MissingParameter + path = Path(value) + input_file = path if path.exists() else Path(config.CONFIG_LOCAL_DIR, path) + try: + with input_file.open() as f: + _config: dict[str, list[dict[str, Any]]] = load(f.read(), Loader=Loader) # noqa: S506 + ctx.default_map = _config + except Exception as e: + msg = f"Failed to load batch config file: {e}" + raise click.BadParameter(msg) from e + return value + + +class BatchCliTypedDict(TypedDict): + batch_config_file: Annotated[ + bool, + click.option( + "--batch-config-file", + type=click.Path(), + callback=click_get_defaults_from_file, + is_eager=True, + expose_value=False, + help="Read batch configuration from yaml file", + ), + ] + + +def build_sub_cmd_args(batch_config: MutableMapping[str, Any] | None): + bool_options = { + "drop_old": True, + "load": True, + "search_serial": True, + "search_concurrent": True, + "dry_run": False, + "custom_dataset_use_shuffled": True, + "custom_dataset_with_gt": True, + } + + def format_option(key: str, value: Any): + opt_name = key.replace("_", "-") + + if key in bool_options: + return format_bool_option(opt_name, value, skip=False) + + if key.startswith("skip_"): + raw_key = key[5:] + raw_opt = raw_key.replace("_", "-") + return format_bool_option(raw_opt, value, skip=True, raw_key=raw_key) + + return [f"--{opt_name}", str(value)] + + def format_bool_option(opt_name: str, value: Any, skip: bool = False, raw_key: str | None = None): + if isinstance(value, bool): + if skip: + if bool_options.get(raw_key, False): + return [f"--skip-{opt_name}"] if value else [f"--{opt_name}"] + return [f"--{opt_name}", str(value)] + if value: + return [f"--{opt_name}"] + if bool_options.get(opt_name.replace("-", "_"), False): + return [f"--skip-{opt_name}"] + return [] + return [f"--{opt_name}", str(value)] + + args_arr = [] + for sub_cmd_key, sub_cmd_config_list in batch_config.items(): + for sub_cmd_args in sub_cmd_config_list: + args = [sub_cmd_key] + for k, v in sub_cmd_args.items(): + args.extend(format_option(k, v)) + args_arr.append(args) + + return args_arr + + +@cli.command() +@click_parameter_decorators_from_typed_dict(BatchCliTypedDict) +def BatchCli(): + ctx = click.get_current_context() + batch_config = ctx.default_map + + runner = CliRunner() + + args_arr = build_sub_cmd_args(batch_config) + + for args in args_arr: + log.info(f"got batch config: {' '.join(args)}") + + for args in args_arr: + result = runner.invoke(cli, args) + time.sleep(5) + + from ..interface import global_result_future + + if global_result_future: + wait([global_result_future]) + + if result.exception: + log.exception(f"failed to run sub command: {args[0]}", exc_info=result.exception) diff --git a/vectordb_bench/cli/vectordbbench.py b/vectordb_bench/cli/vectordbbench.py index d4153bc1e..a63eacf5b 100644 --- a/vectordb_bench/cli/vectordbbench.py +++ b/vectordb_bench/cli/vectordbbench.py @@ -16,6 +16,7 @@ from ..backend.clients.vespa.cli import Vespa from ..backend.clients.weaviate_cloud.cli import Weaviate from ..backend.clients.zilliz_cloud.cli import ZillizAutoIndex +from .batch_cli import BatchCli from .cli import cli cli.add_command(PgVectorHNSW) @@ -37,6 +38,7 @@ cli.add_command(Vespa) cli.add_command(LanceDB) cli.add_command(QdrantCloud) +cli.add_command(BatchCli) if __name__ == "__main__": diff --git a/vectordb_bench/config-files/batch_sample_config.yml b/vectordb_bench/config-files/batch_sample_config.yml new file mode 100644 index 000000000..3edaab160 --- /dev/null +++ b/vectordb_bench/config-files/batch_sample_config.yml @@ -0,0 +1,17 @@ +pgvectorhnsw: + - db_label: pgConfigTest + user_name: vectordbbench + db_name: vectordbbench + host: localhost + m: 16 + ef_construction: 128 + ef_search: 128 +milvushnsw: + - skip_search_serial: True + case_type: Performance1536D50K + uri: http://localhost:19530 + m: 16 + ef_construction: 128 + ef_search: 128 + drop_old: False + load: False From 6c376269bf3b9793a31c4bc454ddaa7ba23907ac Mon Sep 17 00:00:00 2001 From: Navneet Verma Date: Sun, 8 Jun 2025 18:28:12 -0700 Subject: [PATCH 173/327] Fixing bugs in aws opensearch client and added fp16 support (#529) * Fixing bugs in aws opensearch client Signed-off-by: Navneet Verma * Added fp16 quantization type for aws_opensearch Signed-off-by: Navneet Verma --------- Signed-off-by: Navneet Verma --- README.md | 5 ++- .../clients/aws_opensearch/aws_opensearch.py | 34 +++++++++++---- .../backend/clients/aws_opensearch/cli.py | 43 +++++++++++++------ .../backend/clients/aws_opensearch/config.py | 32 ++++++++------ 4 files changed, 79 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index 6cb3cf7d8..19bfdaf61 100644 --- a/README.md +++ b/README.md @@ -201,9 +201,12 @@ Options: --force-merge-enabled BOOLEAN Whether to perform force merge operation --flush-threshold-size TEXT Size threshold for flushing the transaction log + --engine TEXT type of engine to use valid values [faiss, lucene] # Memory Management --cb-threshold TEXT k-NN Memory circuit breaker threshold - + + # Quantization Type + --quantization-type TEXT which type of quantization to use valid values [fp32, fp16] --help Show this message and exit. ``` diff --git a/vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py b/vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py index adb766300..42096f711 100644 --- a/vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py +++ b/vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py @@ -36,6 +36,7 @@ def __init__( self.vector_col_name = vector_col_name log.info(f"AWS_OpenSearch client config: {self.db_config}") + log.info(f"AWS_OpenSearch db case config : {self.case_config}") client = OpenSearch(**self.db_config) if drop_old: log.info(f"AWS_OpenSearch client drop old index: {self.index_name}") @@ -65,17 +66,14 @@ def _create_index(self, client: OpenSearch): "knn": True, "number_of_shards": self.case_config.number_of_shards, "number_of_replicas": 0, - "translog.flush_threshold_size": self.case_config.flush_threshold_size, # Setting trans log threshold to 5GB - **( - {"knn.algo_param.ef_search": self.case_config.ef_search} - if self.case_config.engine == AWSOS_Engine.nmslib - else {} - ), + "translog.flush_threshold_size": self.case_config.flush_threshold_size, + "knn.advanced.approximate_threshold": "-1", }, "refresh_interval": self.case_config.refresh_interval, } mappings = { + "_source": {"excludes": [self.vector_col_name], "recovery_source_excludes": [self.vector_col_name]}, "properties": { **{categoryCol: {"type": "keyword"} for categoryCol in self.category_col_names}, self.vector_col_name: { @@ -151,9 +149,18 @@ def search_embedding( body = { "size": k, - "query": {"knn": {self.vector_col_name: {"vector": query, "k": k}}}, + "query": { + "knn": { + self.vector_col_name: { + "vector": query, + "k": k, + "method_parameters": {"ef_search": self.case_config.efSearch}, + } + } + }, **({"filter": {"range": {self.id_col_name: {"gt": filters["id"]}}}} if filters else {}), } + try: resp = self.client.search( index=self.index_name, @@ -162,6 +169,7 @@ def search_embedding( _source=False, docvalue_fields=[self.id_col_name], stored_fields="_none_", + preference="_only_local" if self.case_config.number_of_shards == 1 else None, ) log.debug(f"Search took: {resp['took']}") log.debug(f"Search shards: {resp['_shards']}") @@ -200,7 +208,7 @@ def _wait_till_green(self): while True: res = self.client.cat.indices(index=self.index_name, h="health", format="json") health = res[0]["health"] - if health != "green": + if health == "green": break log.info(f"The index {self.index_name} has health : {health} and is not green. Retrying") time.sleep(SECONDS_WAITING_FOR_REPLICAS_TO_BE_ENABLED_SEC) @@ -228,8 +236,16 @@ def _do_force_merge(self): "persistent": {"knn.algo_param.index_thread_qty": self.case_config.index_thread_qty_during_force_merge} } self.client.cluster.put_settings(cluster_settings_body) + + log.info("Updating the graph threshold to ensure that during merge we can do graph creation.") + output = self.client.indices.put_settings( + index=self.index_name, body={"index.knn.advanced.approximate_threshold": "0"} + ) + log.info(f"response of updating setting is: {output}") + log.debug(f"Starting force merge for index {self.index_name}") - force_merge_endpoint = f"/{self.index_name}/_forcemerge?max_num_segments=1&wait_for_completion=false" + segments = self.case_config.number_of_segments + force_merge_endpoint = f"/{self.index_name}/_forcemerge?max_num_segments={segments}&wait_for_completion=false" force_merge_task_id = self.client.transport.perform_request("POST", force_merge_endpoint)["task"] while True: time.sleep(WAITING_FOR_FORCE_MERGE_SEC) diff --git a/vectordb_bench/backend/clients/aws_opensearch/cli.py b/vectordb_bench/backend/clients/aws_opensearch/cli.py index fa457154d..ae401aec1 100644 --- a/vectordb_bench/backend/clients/aws_opensearch/cli.py +++ b/vectordb_bench/backend/clients/aws_opensearch/cli.py @@ -11,12 +11,13 @@ run, ) from .. import DB +from .config import AWSOS_Engine, AWSOSQuantization class AWSOpenSearchTypedDict(TypedDict): host: Annotated[str, click.option("--host", type=str, help="Db host", required=True)] - port: Annotated[int, click.option("--port", type=int, default=443, help="Db Port")] - user: Annotated[str, click.option("--user", type=str, default="admin", help="Db User")] + port: Annotated[int, click.option("--port", type=int, default=80, help="Db Port")] + user: Annotated[str, click.option("--user", type=str, help="Db User")] password: Annotated[str, click.option("--password", type=str, help="Db password")] number_of_shards: Annotated[ int, @@ -48,16 +49,6 @@ class AWSOpenSearchTypedDict(TypedDict): ), ] - number_of_indexing_clients: Annotated[ - int, - click.option( - "--number-of-indexing-clients", - type=int, - help="Number of concurrent indexing clients", - default=1, - ), - ] - number_of_segments: Annotated[ int, click.option("--number-of-segments", type=int, help="Target number of segments after merging", default=1), @@ -92,6 +83,28 @@ class AWSOpenSearchTypedDict(TypedDict): ), ] + quantization_type: Annotated[ + str | None, + click.option( + "--quantization-type", + type=click.Choice(["fp32", "fp16"]), + help="quantization type for vectors (in index)", + default="fp32", + required=False, + ), + ] + + engine: Annotated[ + str | None, + click.option( + "--engine", + type=click.Choice(["faiss", "lucene"]), + help="quantization type for vectors (in index)", + default="faiss", + required=False, + ), + ] + class AWSOpenSearchHNSWTypedDict(CommonTypedDict, AWSOpenSearchTypedDict, HNSWFlavor2): ... @@ -117,9 +130,13 @@ def AWSOpenSearch(**parameters: Unpack[AWSOpenSearchHNSWTypedDict]): refresh_interval=parameters["refresh_interval"], force_merge_enabled=parameters["force_merge_enabled"], flush_threshold_size=parameters["flush_threshold_size"], - number_of_indexing_clients=parameters["number_of_indexing_clients"], index_thread_qty_during_force_merge=parameters["index_thread_qty_during_force_merge"], cb_threshold=parameters["cb_threshold"], + efConstruction=parameters["ef_construction"], + efSearch=parameters["ef_runtime"], + M=parameters["m"], + engine=AWSOS_Engine(parameters["engine"]), + quantization_type=AWSOSQuantization(parameters["quantization_type"]), ), **parameters, ) diff --git a/vectordb_bench/backend/clients/aws_opensearch/config.py b/vectordb_bench/backend/clients/aws_opensearch/config.py index dd51b266d..cf5192163 100644 --- a/vectordb_bench/backend/clients/aws_opensearch/config.py +++ b/vectordb_bench/backend/clients/aws_opensearch/config.py @@ -10,17 +10,21 @@ class AWSOpenSearchConfig(DBConfig, BaseModel): host: str = "" - port: int = 443 + port: int = 80 user: str = "" password: SecretStr = "" def to_dict(self) -> dict: + use_ssl = self.port == 443 + http_auth = ( + (self.user, self.password.get_secret_value()) if len(self.user) != 0 and len(self.password) != 0 else () + ) return { "hosts": [{"host": self.host, "port": self.port}], - "http_auth": (self.user, self.password.get_secret_value()), - "use_ssl": True, + "http_auth": http_auth, + "use_ssl": use_ssl, "http_compress": True, - "verify_certs": True, + "verify_certs": use_ssl, "ssl_assert_hostname": False, "ssl_show_warn": False, "timeout": 600, @@ -28,9 +32,13 @@ def to_dict(self) -> dict: class AWSOS_Engine(Enum): - nmslib = "nmslib" faiss = "faiss" - lucene = "Lucene" + lucene = "lucene" + + +class AWSOSQuantization(Enum): + fp32 = "fp32" + fp16 = "fp16" class AWSOpenSearchIndexConfig(BaseModel, DBCaseConfig): @@ -46,19 +54,14 @@ class AWSOpenSearchIndexConfig(BaseModel, DBCaseConfig): refresh_interval: str | None = "60s" force_merge_enabled: bool | None = True flush_threshold_size: str | None = "5120mb" - number_of_indexing_clients: int | None = 1 index_thread_qty_during_force_merge: int cb_threshold: str | None = "50%" + quantization_type: AWSOSQuantization = AWSOSQuantization.fp32 def parse_metric(self) -> str: if self.metric_type == MetricType.IP: return "innerproduct" if self.metric_type == MetricType.COSINE: - if self.engine == AWSOS_Engine.faiss: - log.info( - "Using innerproduct because faiss doesn't support cosine as metric type for Opensearch", - ) - return "innerproduct" return "cosinesimil" return "l2" @@ -71,6 +74,11 @@ def index_param(self) -> dict: "ef_construction": self.efConstruction, "m": self.M, "ef_search": self.efSearch, + **( + {"encoder": {"name": "sq", "parameters": {"type": self.quantization_type.fp16.value}}} + if self.quantization_type is not AWSOSQuantization.fp32 + else {} + ), }, } From 564b75ed3b8bc6094333e9121791fc4e10f4940a Mon Sep 17 00:00:00 2001 From: zhangzhe <760583490@qq.com> Date: Fri, 30 May 2025 14:51:29 +0800 Subject: [PATCH 174/327] Bugfix: add num_shards option to MilvusHNSW --- vectordb_bench/backend/clients/milvus/cli.py | 1 + 1 file changed, 1 insertion(+) diff --git a/vectordb_bench/backend/clients/milvus/cli.py b/vectordb_bench/backend/clients/milvus/cli.py index b9394cd06..4c80f6100 100644 --- a/vectordb_bench/backend/clients/milvus/cli.py +++ b/vectordb_bench/backend/clients/milvus/cli.py @@ -98,6 +98,7 @@ def MilvusHNSW(**parameters: Unpack[MilvusHNSWTypedDict]): uri=SecretStr(parameters["uri"]), user=parameters["user_name"], password=SecretStr(parameters["password"]) if parameters["password"] else None, + num_shards=int(parameters["num_shards"]), ), db_case_config=HNSWConfig( M=parameters["m"], From ed6a291d8da5b121789474b5c4c53f3a82ff2b18 Mon Sep 17 00:00:00 2001 From: zhangzhe <760583490@qq.com> Date: Fri, 30 May 2025 14:55:44 +0800 Subject: [PATCH 175/327] BugFix: An error occurs when the password option is not passed. --- vectordb_bench/backend/clients/milvus/cli.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/vectordb_bench/backend/clients/milvus/cli.py b/vectordb_bench/backend/clients/milvus/cli.py index 4c80f6100..8fa5ff584 100644 --- a/vectordb_bench/backend/clients/milvus/cli.py +++ b/vectordb_bench/backend/clients/milvus/cli.py @@ -56,7 +56,7 @@ def MilvusAutoIndex(**parameters: Unpack[MilvusAutoIndexTypedDict]): db_label=parameters["db_label"], uri=SecretStr(parameters["uri"]), user=parameters["user_name"], - password=SecretStr(parameters["password"]), + password=SecretStr(parameters["password"]) if parameters["password"] else None, num_shards=int(parameters["num_shards"]), ), db_case_config=AutoIndexConfig(), @@ -75,7 +75,7 @@ def MilvusFlat(**parameters: Unpack[MilvusAutoIndexTypedDict]): db_label=parameters["db_label"], uri=SecretStr(parameters["uri"]), user=parameters["user_name"], - password=SecretStr(parameters["password"]), + password=SecretStr(parameters["password"]) if parameters["password"] else None, num_shards=int(parameters["num_shards"]), ), db_case_config=FLATConfig(), @@ -123,7 +123,7 @@ def MilvusIVFFlat(**parameters: Unpack[MilvusIVFFlatTypedDict]): db_label=parameters["db_label"], uri=SecretStr(parameters["uri"]), user=parameters["user_name"], - password=SecretStr(parameters["password"]), + password=SecretStr(parameters["password"]) if parameters["password"] else None, num_shards=int(parameters["num_shards"]), ), db_case_config=IVFFlatConfig( @@ -145,7 +145,7 @@ def MilvusIVFSQ8(**parameters: Unpack[MilvusIVFFlatTypedDict]): db_label=parameters["db_label"], uri=SecretStr(parameters["uri"]), user=parameters["user_name"], - password=SecretStr(parameters["password"]), + password=SecretStr(parameters["password"]) if parameters["password"] else None, num_shards=int(parameters["num_shards"]), ), db_case_config=IVFSQ8Config( @@ -171,7 +171,7 @@ def MilvusDISKANN(**parameters: Unpack[MilvusDISKANNTypedDict]): db_label=parameters["db_label"], uri=SecretStr(parameters["uri"]), user=parameters["user_name"], - password=SecretStr(parameters["password"]), + password=SecretStr(parameters["password"]) if parameters["password"] else None, num_shards=int(parameters["num_shards"]), ), db_case_config=DISKANNConfig( @@ -200,7 +200,7 @@ def MilvusGPUIVFFlat(**parameters: Unpack[MilvusGPUIVFTypedDict]): db_label=parameters["db_label"], uri=SecretStr(parameters["uri"]), user=parameters["user_name"], - password=SecretStr(parameters["password"]), + password=SecretStr(parameters["password"]) if parameters["password"] else None, num_shards=int(parameters["num_shards"]), ), db_case_config=GPUIVFFlatConfig( @@ -235,7 +235,7 @@ def MilvusGPUBruteForce(**parameters: Unpack[MilvusGPUBruteForceTypedDict]): db_label=parameters["db_label"], uri=SecretStr(parameters["uri"]), user=parameters["user_name"], - password=SecretStr(parameters["password"]), + password=SecretStr(parameters["password"]) if parameters["password"] else None, num_shards=int(parameters["num_shards"]), ), db_case_config=GPUBruteForceConfig( @@ -267,7 +267,7 @@ def MilvusGPUIVFPQ(**parameters: Unpack[MilvusGPUIVFPQTypedDict]): db_label=parameters["db_label"], uri=SecretStr(parameters["uri"]), user=parameters["user_name"], - password=SecretStr(parameters["password"]), + password=SecretStr(parameters["password"]) if parameters["password"] else None, num_shards=int(parameters["num_shards"]), ), db_case_config=GPUIVFPQConfig( @@ -307,7 +307,7 @@ def MilvusGPUCAGRA(**parameters: Unpack[MilvusGPUCAGRATypedDict]): db_label=parameters["db_label"], uri=SecretStr(parameters["uri"]), user=parameters["user_name"], - password=SecretStr(parameters["password"]), + password=SecretStr(parameters["password"]) if parameters["password"] else None, num_shards=int(parameters["num_shards"]), ), db_case_config=GPUCAGRAConfig( From f0c88d13f005d950a05d39252f35e5b7ab6dd917 Mon Sep 17 00:00:00 2001 From: ZebinRen <80635329+ZebinRen@users.noreply.github.com> Date: Mon, 9 Jun 2025 03:37:55 +0200 Subject: [PATCH 176/327] Add support for Qdrant local setup (#533) * Add Qdrant local * Add support for local setup without authentication api-key for weaviate * Expose HSNW index parameters to cli for weaviate * Add hnsw-ef parameter to Qdrant local --------- Co-authored-by: Min Tian --- vectordb_bench/backend/clients/__init__.py | 15 ++ .../backend/clients/qdrant_local/cli.py | 65 +++++ .../backend/clients/qdrant_local/config.py | 46 ++++ .../clients/qdrant_local/qdrant_local.py | 231 ++++++++++++++++++ .../backend/clients/weaviate_cloud/cli.py | 33 ++- .../backend/clients/weaviate_cloud/config.py | 2 + .../clients/weaviate_cloud/weaviate_cloud.py | 5 + vectordb_bench/cli/vectordbbench.py | 2 + 8 files changed, 396 insertions(+), 3 deletions(-) create mode 100644 vectordb_bench/backend/clients/qdrant_local/cli.py create mode 100644 vectordb_bench/backend/clients/qdrant_local/config.py create mode 100644 vectordb_bench/backend/clients/qdrant_local/qdrant_local.py diff --git a/vectordb_bench/backend/clients/__init__.py b/vectordb_bench/backend/clients/__init__.py index f05913a06..21903594e 100644 --- a/vectordb_bench/backend/clients/__init__.py +++ b/vectordb_bench/backend/clients/__init__.py @@ -27,6 +27,7 @@ class DB(Enum): Pinecone = "Pinecone" ElasticCloud = "ElasticCloud" QdrantCloud = "QdrantCloud" + QdrantLocal = "QdrantLocal" WeaviateCloud = "WeaviateCloud" PgVector = "PgVector" PgVectoRS = "PgVectoRS" @@ -46,6 +47,7 @@ class DB(Enum): Clickhouse = "Clickhouse" Vespa = "Vespa" LanceDB = "LanceDB" + @property def init_cls(self) -> type[VectorDB]: # noqa: PLR0911, PLR0912, C901, PLR0915 @@ -74,6 +76,11 @@ def init_cls(self) -> type[VectorDB]: # noqa: PLR0911, PLR0912, C901, PLR0915 from .qdrant_cloud.qdrant_cloud import QdrantCloud return QdrantCloud + + if self == DB.QdrantLocal: + from .qdrant_local.qdrant_local import QdrantLocal + + return QdrantLocal if self == DB.WeaviateCloud: from .weaviate_cloud.weaviate_cloud import WeaviateCloud @@ -200,6 +207,9 @@ def config_cls(self) -> type[DBConfig]: # noqa: PLR0911, PLR0912, C901, PLR0915 from .qdrant_cloud.config import QdrantConfig return QdrantConfig + + if self == DB.QdrantLocal: + from .qdrant_local.config import QdrantLocalConfig if self == DB.WeaviateCloud: from .weaviate_cloud.config import WeaviateConfig @@ -322,6 +332,11 @@ def case_config_cls( # noqa: C901, PLR0911, PLR0912 from .qdrant_cloud.config import QdrantIndexConfig return QdrantIndexConfig + + if self == DB.QdrantLocal: + from .qdrant_local.config import QdrantLocalIndexConfig + + return QdrantLocalIndexConfig if self == DB.WeaviateCloud: from .weaviate_cloud.config import WeaviateIndexConfig diff --git a/vectordb_bench/backend/clients/qdrant_local/cli.py b/vectordb_bench/backend/clients/qdrant_local/cli.py new file mode 100644 index 000000000..c01f0afb7 --- /dev/null +++ b/vectordb_bench/backend/clients/qdrant_local/cli.py @@ -0,0 +1,65 @@ +from typing import Annotated, TypedDict, Unpack + +import click +from pydantic import SecretStr + +from vectordb_bench.backend.clients import DB +from vectordb_bench.cli.cli import ( + CommonTypedDict, + cli, + click_parameter_decorators_from_typed_dict, + run, +) + + +DBTYPE = DB.QdrantLocal + + +class QdrantLocalTypedDict(CommonTypedDict): + url: Annotated[ + str, + click.option("--url", type=str, help="Qdrant url", required=True), + ] + on_disk: Annotated[ + bool, + click.option( + "--on-disk", type=bool, default=False, help="Store the vectors and the HNSW index on disk" + ), + ] + m: Annotated[ + int, + click.option( + "--m", type=int, default=16, help="HNSW index parameter m, set 0 to disable the index" + ), + ] + ef_construct: Annotated[ + int, + click.option( + "--ef-construct", type=int, default=200, help="HNSW index parameter ef_construct" + ), + ] + hnsw_ef: Annotated[ + int, + click.option( + "--hnsw-ef", type=int, default=0, help="HNSW index parameter hnsw_ef, set 0 to use ef_construct for search", + ), + ] + +@cli.command() +@click_parameter_decorators_from_typed_dict(QdrantLocalTypedDict) +def QdrantLocal(**parameters: Unpack[QdrantLocalTypedDict]): + from .config import QdrantLocalConfig, QdrantLocalIndexConfig + + run( + db=DBTYPE, + db_config=QdrantLocalConfig( + url=SecretStr(parameters["url"]) + ), + db_case_config=QdrantLocalIndexConfig( + on_disk=parameters["on_disk"], + m=parameters["m"], + ef_construct=parameters["ef_construct"], + hnsw_ef=parameters["hnsw_ef"], + ), + **parameters, + ) diff --git a/vectordb_bench/backend/clients/qdrant_local/config.py b/vectordb_bench/backend/clients/qdrant_local/config.py new file mode 100644 index 000000000..b2949313f --- /dev/null +++ b/vectordb_bench/backend/clients/qdrant_local/config.py @@ -0,0 +1,46 @@ +from pydantic import BaseModel, SecretStr + +from ..api import DBCaseConfig, DBConfig, IndexType, MetricType + +class QdrantLocalConfig(DBConfig): + url: SecretStr + + def to_dict(self) -> dict: + return { + "url": self.url.get_secret_value(), + } + + +class QdrantLocalIndexConfig(BaseModel, DBCaseConfig): + metric_type: MetricType | None = None + m: int + ef_construct: int + hnsw_ef: int | None = 0 + on_disk: bool | None = False + + def parse_metric(self) -> str: + if self.metric_type == MetricType.L2: + return "Euclid" + + if self.metric_type == MetricType.IP: + return "Dot" + + return "Cosine" + + def index_param(self) -> dict: + return { + "distance": self.parse_metric(), + "m": self.m, + "ef_construct": self.ef_construct, + "on_disk": self.on_disk, + } + + def search_param(self) -> dict: + search_params = { + "exact": False, # Force to use ANNs + } + + if self.hnsw_ef != 0: + search_params["hnsw_ef"] = self.hnsw_ef + + return search_params \ No newline at end of file diff --git a/vectordb_bench/backend/clients/qdrant_local/qdrant_local.py b/vectordb_bench/backend/clients/qdrant_local/qdrant_local.py new file mode 100644 index 000000000..723808e8f --- /dev/null +++ b/vectordb_bench/backend/clients/qdrant_local/qdrant_local.py @@ -0,0 +1,231 @@ +"""Wrapper around the Qdrant over VectorDB""" + +import logging +import time +from collections.abc import Iterable +from contextlib import contextmanager + +from qdrant_client import QdrantClient +from qdrant_client.http.models import ( + Batch, + CollectionStatus, + FieldCondition, + Filter, + HnswConfigDiff, + OptimizersConfigDiff, + PayloadSchemaType, + Range, + SearchParams, + VectorParams, +) + +from ..api import VectorDB +from .config import QdrantLocalIndexConfig + +log = logging.getLogger(__name__) + +SECONDS_WAITING_FOR_INDEXING_API_CALL = 5 +QDRANT_BATCH_SIZE = 100 + + +def qdrant_collection_exists(client, collection_name: str) -> bool: + collection_exists = True + + try: + client.get_collection(collection_name) + except Exception as e: + collection_exists = False + + return collection_exists + +class QdrantLocal(VectorDB): + def __init__( + self, + dim: int, + db_config: dict, + db_case_config: dict, + collection_name: str = "QdrantLocalCollection", + drop_old: bool = False, + name: str = "QdrantLocal", + **kwargs, + ): + """Initialize wrapper around the qdrant.""" + self.name = name + self.db_config = db_config + self.case_config = db_case_config + self.search_parameter = self.case_config.search_param() + self.collection_name = collection_name + self.client = None + + self._primary_field = "pk" + self._vector_field = "vector" + + client = QdrantClient(**self.db_config) + + # Lets just print the parameters here for double check + log.info(f"Case config: {self.case_config.index_param()}") + log.info(f"Search parameter: {self.search_parameter}") + + if drop_old and qdrant_collection_exists(client, self.collection_name): + log.info(f"{self.name} client drop_old collection: {self.collection_name}") + client.delete_collection(self.collection_name) + + if not qdrant_collection_exists(client, self.collection_name): + log.info(f"{self.name} create collection: {self.collection_name}") + self._create_collection(dim, client) + + client = None + + @contextmanager + def init(self): + """ + Examples: + >>> with self.init(): + >>> self.insert_embeddings() + >>> self.search_embedding() + """ + # create connection + self.client = QdrantClient(**self.db_config) + yield + self.client = None + del self.client + + def _create_collection(self, dim: int, qdrant_client: QdrantClient): + log.info(f"Create collection: {self.collection_name}") + log.info(f"Index parameters: m={self.case_config.index_param()['m']}, ef_construct={self.case_config.index_param()['ef_construct']}, on_disk={self.case_config.index_param()['on_disk']}") + + # If the on_disk is true, we enable both on disk index and vectors. + try: + qdrant_client.create_collection( + collection_name=self.collection_name, + vectors_config=VectorParams( + size=dim, + distance=self.case_config.index_param()["distance"], + on_disk=self.case_config.index_param()["on_disk"], + ), + hnsw_config=HnswConfigDiff( + m = self.case_config.index_param()["m"], + ef_construct=self.case_config.index_param()["ef_construct"], + on_disk=self.case_config.index_param()["on_disk"], + ) + ) + + qdrant_client.create_payload_index( + collection_name=self.collection_name, + field_name=self._primary_field, + field_schema=PayloadSchemaType.INTEGER, + ) + + except Exception as e: + if "already exists!" in str(e): + return + log.warning(f"Failed to create collection: {self.collection_name} error: {e}") + raise e from None + + def optimize(self, data_size: int | None = None): + assert self.client, "Please call self.init() before" + # wait for vectors to be fully indexed + try: + while True: + info = self.client.get_collection(self.collection_name) + time.sleep(SECONDS_WAITING_FOR_INDEXING_API_CALL) + if info.status != CollectionStatus.GREEN: + continue + if info.status == CollectionStatus.GREEN: + log.info(f"Finishing building index for collection: {self.collection_name}") + msg = ( + f"Stored vectors: {info.vectors_count}, Indexed vectors: {info.indexed_vectors_count}, " + f"Collection status: {info.indexed_vectors_count}" + ) + log.info(msg) + return + + except Exception as e: + log.warning(f"QdrantCloud ready to search error: {e}") + raise e from None + + def insert_embeddings( + self, + embeddings: Iterable[list[float]], + metadata: list[int], + **kwargs, + ) -> tuple[int, Exception]: + """Insert embeddings into the database. + + Args: + embeddings(list[list[float]]): list of embeddings + metadata(list[int]): list of metadata + kwargs: other arguments + + Returns: + tuple[int, Exception]: number of embeddings inserted and exception if any + """ + assert self.client is not None + assert len(embeddings) == len(metadata) + insert_count = 0 + + # disable indexing for quick insertion + self.client.update_collection( + collection_name=self.collection_name, + optimizer_config=OptimizersConfigDiff(indexing_threshold=0), + ) + try: + for offset in range(0, len(embeddings), QDRANT_BATCH_SIZE): + vectors = embeddings[offset : offset + QDRANT_BATCH_SIZE] + ids = metadata[offset : offset + QDRANT_BATCH_SIZE] + payloads = [{self._primary_field: v} for v in ids] + _ = self.client.upsert( + collection_name=self.collection_name, + wait=True, + points=Batch(ids=ids, payloads=payloads, vectors=vectors), + ) + insert_count += QDRANT_BATCH_SIZE + # enable indexing after insertion + self.client.update_collection( + collection_name=self.collection_name, + optimizer_config=OptimizersConfigDiff(indexing_threshold=100), + ) + + except Exception as e: + log.info(f"Failed to insert data, {e}") + return insert_count, e + else: + return insert_count, None + + def search_embedding( + self, + query: list[float], + k: int = 100, + filters: dict | None = None, + timeout: int | None = None, + ) -> list[int]: + """Perform a search on a query embedding and return results with score. + Should call self.init() first. + """ + assert self.client is not None + + f = None + if filters: + f = Filter( + must=[ + FieldCondition( + key=self._primary_field, + range=Range( + gt=filters.get("id"), + ), + ), + ], + ) + res = ( + self.client.query_points( + collection_name=self.collection_name, + query=query, + limit=k, + query_filter=f, + search_params=SearchParams(**self.search_parameter), + + ).points + ) + + return [result.id for result in res] + diff --git a/vectordb_bench/backend/clients/weaviate_cloud/cli.py b/vectordb_bench/backend/clients/weaviate_cloud/cli.py index 181898c74..9faf768a6 100644 --- a/vectordb_bench/backend/clients/weaviate_cloud/cli.py +++ b/vectordb_bench/backend/clients/weaviate_cloud/cli.py @@ -15,12 +15,34 @@ class WeaviateTypedDict(CommonTypedDict): api_key: Annotated[ str, - click.option("--api-key", type=str, help="Weaviate api key", required=True), + click.option("--api-key", type=str, help="Weaviate api key", required=False, default=''), ] url: Annotated[ str, click.option("--url", type=str, help="Weaviate url", required=True), ] + no_auth: Annotated[ + bool, + click.option("--no-auth", is_flag=True, help="Do not use api-key, set it to true if you are using a local setup. Default is False.", default=False), + ] + m: Annotated[ + int, + click.option( + "--m", type=int, default=16, help="HNSW index parameter m." + ), + ] + ef_construct: Annotated[ + int, + click.option( + "--ef-construction", type=int, default=256, help="HNSW index parameter ef_construction" + ), + ] + ef: Annotated[ + int, + click.option( + "--ef", type=int, default=256, help="HNSW index parameter ef for search" + ), + ] @cli.command() @@ -32,9 +54,14 @@ def Weaviate(**parameters: Unpack[WeaviateTypedDict]): db=DB.WeaviateCloud, db_config=WeaviateConfig( db_label=parameters["db_label"], - api_key=SecretStr(parameters["api_key"]), + api_key=SecretStr(parameters["api_key"]) if parameters["api_key"] != '' else SecretStr("-"), url=SecretStr(parameters["url"]), + no_auth=parameters["no_auth"], + ), + db_case_config=WeaviateIndexConfig( + efConstruction=parameters["ef_construction"], + maxConnections=parameters["m"], + ef=parameters["ef"], ), - db_case_config=WeaviateIndexConfig(ef=256, efConstruction=256, maxConnections=16), **parameters, ) diff --git a/vectordb_bench/backend/clients/weaviate_cloud/config.py b/vectordb_bench/backend/clients/weaviate_cloud/config.py index 4c58167d4..f29a307a3 100644 --- a/vectordb_bench/backend/clients/weaviate_cloud/config.py +++ b/vectordb_bench/backend/clients/weaviate_cloud/config.py @@ -6,11 +6,13 @@ class WeaviateConfig(DBConfig): url: SecretStr api_key: SecretStr + no_auth: bool | None = False def to_dict(self) -> dict: return { "url": self.url.get_secret_value(), "auth_client_secret": self.api_key.get_secret_value(), + "no_auth": self.no_auth, } diff --git a/vectordb_bench/backend/clients/weaviate_cloud/weaviate_cloud.py b/vectordb_bench/backend/clients/weaviate_cloud/weaviate_cloud.py index c31104d8b..18a17a661 100644 --- a/vectordb_bench/backend/clients/weaviate_cloud/weaviate_cloud.py +++ b/vectordb_bench/backend/clients/weaviate_cloud/weaviate_cloud.py @@ -37,6 +37,11 @@ def __init__( self._scalar_field = "key" self._vector_field = "vector" self._index_name = "vector_idx" + + # If local setup is used, we + if db_config['no_auth']: + del db_config['auth_client_secret'] + del db_config['no_auth'] from weaviate import Client diff --git a/vectordb_bench/cli/vectordbbench.py b/vectordb_bench/cli/vectordbbench.py index a63eacf5b..27bec6f26 100644 --- a/vectordb_bench/cli/vectordbbench.py +++ b/vectordb_bench/cli/vectordbbench.py @@ -10,6 +10,7 @@ from ..backend.clients.pgvector.cli import PgVectorHNSW from ..backend.clients.pgvectorscale.cli import PgVectorScaleDiskAnn from ..backend.clients.qdrant_cloud.cli import QdrantCloud +from ..backend.clients.qdrant_local.cli import QdrantLocal from ..backend.clients.redis.cli import Redis from ..backend.clients.test.cli import Test from ..backend.clients.tidb.cli import TiDB @@ -38,6 +39,7 @@ cli.add_command(Vespa) cli.add_command(LanceDB) cli.add_command(QdrantCloud) +cli.add_command(QdrantLocal) cli.add_command(BatchCli) From 19439318508d0339e247805b6f95a3f87b319995 Mon Sep 17 00:00:00 2001 From: Christoph Koerner Date: Thu, 5 Jun 2025 07:41:37 +0100 Subject: [PATCH 177/327] Fix python import in MemoryDB client The incorrect import is a result of previous duplicate import of the same class name and recent refactoring. https://github.com/zilliztech/VectorDBBench/commit/f9dec0854e2fd597d86e2a1c0a9faaf6289f1eea#diff-1638e273a6d7eee7b4412bf79bda9112da86d53e18d1b650b8b80a2e1df4f7e6R15 --- vectordb_bench/backend/clients/memorydb/memorydb.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vectordb_bench/backend/clients/memorydb/memorydb.py b/vectordb_bench/backend/clients/memorydb/memorydb.py index 9d077f5df..7e7a8650b 100644 --- a/vectordb_bench/backend/clients/memorydb/memorydb.py +++ b/vectordb_bench/backend/clients/memorydb/memorydb.py @@ -9,10 +9,10 @@ from redis import Redis from redis.cluster import RedisCluster from redis.commands.search.field import NumericField, TagField, VectorField -from redis.commands.search.indexDefinition import IndexDefinition +from redis.commands.search.indexDefinition import IndexDefinition, IndexType from redis.commands.search.query import Query -from ..api import IndexType, VectorDB +from ..api import VectorDB from .config import MemoryDBIndexConfig log = logging.getLogger(__name__) From 0bda245876522ad13d95d17f8b3c41776e7dac49 Mon Sep 17 00:00:00 2001 From: "min.tian" Date: Mon, 9 Jun 2025 09:47:59 +0800 Subject: [PATCH 178/327] upgrade ruff / black, reformat all Signed-off-by: min.tian --- vectordb_bench/backend/clients/__init__.py | 13 ++-- .../backend/clients/qdrant_local/cli.py | 25 +++---- .../backend/clients/qdrant_local/config.py | 19 ++--- .../clients/qdrant_local/qdrant_local.py | 75 ++++++++++--------- .../backend/clients/weaviate_cloud/cli.py | 23 +++--- .../clients/weaviate_cloud/weaviate_cloud.py | 10 +-- 6 files changed, 81 insertions(+), 84 deletions(-) diff --git a/vectordb_bench/backend/clients/__init__.py b/vectordb_bench/backend/clients/__init__.py index 21903594e..c096a14c8 100644 --- a/vectordb_bench/backend/clients/__init__.py +++ b/vectordb_bench/backend/clients/__init__.py @@ -47,7 +47,6 @@ class DB(Enum): Clickhouse = "Clickhouse" Vespa = "Vespa" LanceDB = "LanceDB" - @property def init_cls(self) -> type[VectorDB]: # noqa: PLR0911, PLR0912, C901, PLR0915 @@ -76,10 +75,10 @@ def init_cls(self) -> type[VectorDB]: # noqa: PLR0911, PLR0912, C901, PLR0915 from .qdrant_cloud.qdrant_cloud import QdrantCloud return QdrantCloud - + if self == DB.QdrantLocal: from .qdrant_local.qdrant_local import QdrantLocal - + return QdrantLocal if self == DB.WeaviateCloud: @@ -207,10 +206,12 @@ def config_cls(self) -> type[DBConfig]: # noqa: PLR0911, PLR0912, C901, PLR0915 from .qdrant_cloud.config import QdrantConfig return QdrantConfig - + if self == DB.QdrantLocal: from .qdrant_local.config import QdrantLocalConfig + return QdrantLocalConfig + if self == DB.WeaviateCloud: from .weaviate_cloud.config import WeaviateConfig @@ -332,10 +333,10 @@ def case_config_cls( # noqa: C901, PLR0911, PLR0912 from .qdrant_cloud.config import QdrantIndexConfig return QdrantIndexConfig - + if self == DB.QdrantLocal: from .qdrant_local.config import QdrantLocalIndexConfig - + return QdrantLocalIndexConfig if self == DB.WeaviateCloud: diff --git a/vectordb_bench/backend/clients/qdrant_local/cli.py b/vectordb_bench/backend/clients/qdrant_local/cli.py index c01f0afb7..7995b99b3 100644 --- a/vectordb_bench/backend/clients/qdrant_local/cli.py +++ b/vectordb_bench/backend/clients/qdrant_local/cli.py @@ -1,4 +1,4 @@ -from typing import Annotated, TypedDict, Unpack +from typing import Annotated, Unpack import click from pydantic import SecretStr @@ -11,7 +11,6 @@ run, ) - DBTYPE = DB.QdrantLocal @@ -22,29 +21,27 @@ class QdrantLocalTypedDict(CommonTypedDict): ] on_disk: Annotated[ bool, - click.option( - "--on-disk", type=bool, default=False, help="Store the vectors and the HNSW index on disk" - ), + click.option("--on-disk", type=bool, default=False, help="Store the vectors and the HNSW index on disk"), ] m: Annotated[ int, - click.option( - "--m", type=int, default=16, help="HNSW index parameter m, set 0 to disable the index" - ), + click.option("--m", type=int, default=16, help="HNSW index parameter m, set 0 to disable the index"), ] ef_construct: Annotated[ int, - click.option( - "--ef-construct", type=int, default=200, help="HNSW index parameter ef_construct" - ), + click.option("--ef-construct", type=int, default=200, help="HNSW index parameter ef_construct"), ] hnsw_ef: Annotated[ int, click.option( - "--hnsw-ef", type=int, default=0, help="HNSW index parameter hnsw_ef, set 0 to use ef_construct for search", + "--hnsw-ef", + type=int, + default=0, + help="HNSW index parameter hnsw_ef, set 0 to use ef_construct for search", ), ] + @cli.command() @click_parameter_decorators_from_typed_dict(QdrantLocalTypedDict) def QdrantLocal(**parameters: Unpack[QdrantLocalTypedDict]): @@ -52,9 +49,7 @@ def QdrantLocal(**parameters: Unpack[QdrantLocalTypedDict]): run( db=DBTYPE, - db_config=QdrantLocalConfig( - url=SecretStr(parameters["url"]) - ), + db_config=QdrantLocalConfig(url=SecretStr(parameters["url"])), db_case_config=QdrantLocalIndexConfig( on_disk=parameters["on_disk"], m=parameters["m"], diff --git a/vectordb_bench/backend/clients/qdrant_local/config.py b/vectordb_bench/backend/clients/qdrant_local/config.py index b2949313f..ebdf99dc4 100644 --- a/vectordb_bench/backend/clients/qdrant_local/config.py +++ b/vectordb_bench/backend/clients/qdrant_local/config.py @@ -1,10 +1,11 @@ from pydantic import BaseModel, SecretStr -from ..api import DBCaseConfig, DBConfig, IndexType, MetricType +from ..api import DBCaseConfig, DBConfig, MetricType + class QdrantLocalConfig(DBConfig): url: SecretStr - + def to_dict(self) -> dict: return { "url": self.url.get_secret_value(), @@ -17,7 +18,7 @@ class QdrantLocalIndexConfig(BaseModel, DBCaseConfig): ef_construct: int hnsw_ef: int | None = 0 on_disk: bool | None = False - + def parse_metric(self) -> str: if self.metric_type == MetricType.L2: return "Euclid" @@ -26,7 +27,7 @@ def parse_metric(self) -> str: return "Dot" return "Cosine" - + def index_param(self) -> dict: return { "distance": self.parse_metric(), @@ -34,13 +35,13 @@ def index_param(self) -> dict: "ef_construct": self.ef_construct, "on_disk": self.on_disk, } - + def search_param(self) -> dict: search_params = { - "exact": False, # Force to use ANNs + "exact": False, # Force to use ANNs } - + if self.hnsw_ef != 0: search_params["hnsw_ef"] = self.hnsw_ef - - return search_params \ No newline at end of file + + return search_params diff --git a/vectordb_bench/backend/clients/qdrant_local/qdrant_local.py b/vectordb_bench/backend/clients/qdrant_local/qdrant_local.py index 723808e8f..1340be614 100644 --- a/vectordb_bench/backend/clients/qdrant_local/qdrant_local.py +++ b/vectordb_bench/backend/clients/qdrant_local/qdrant_local.py @@ -28,22 +28,23 @@ QDRANT_BATCH_SIZE = 100 -def qdrant_collection_exists(client, collection_name: str) -> bool: +def qdrant_collection_exists(client: QdrantClient, collection_name: str) -> bool: collection_exists = True - + try: client.get_collection(collection_name) - except Exception as e: + except Exception: collection_exists = False - + return collection_exists - + + class QdrantLocal(VectorDB): def __init__( self, dim: int, db_config: dict, - db_case_config: dict, + db_case_config: QdrantLocalIndexConfig, collection_name: str = "QdrantLocalCollection", drop_old: bool = False, name: str = "QdrantLocal", @@ -56,26 +57,26 @@ def __init__( self.search_parameter = self.case_config.search_param() self.collection_name = collection_name self.client = None - + self._primary_field = "pk" self._vector_field = "vector" - + client = QdrantClient(**self.db_config) - + # Lets just print the parameters here for double check log.info(f"Case config: {self.case_config.index_param()}") log.info(f"Search parameter: {self.search_parameter}") - + if drop_old and qdrant_collection_exists(client, self.collection_name): log.info(f"{self.name} client drop_old collection: {self.collection_name}") client.delete_collection(self.collection_name) - + if not qdrant_collection_exists(client, self.collection_name): log.info(f"{self.name} create collection: {self.collection_name}") self._create_collection(dim, client) client = None - + @contextmanager def init(self): """ @@ -89,11 +90,15 @@ def init(self): yield self.client = None del self.client - + def _create_collection(self, dim: int, qdrant_client: QdrantClient): log.info(f"Create collection: {self.collection_name}") - log.info(f"Index parameters: m={self.case_config.index_param()['m']}, ef_construct={self.case_config.index_param()['ef_construct']}, on_disk={self.case_config.index_param()['on_disk']}") - + log.info( + f"Index parameters: m={self.case_config.index_param()['m']}, " + f"ef_construct={self.case_config.index_param()['ef_construct']}, " + f"on_disk={self.case_config.index_param()['on_disk']}" + ) + # If the on_disk is true, we enable both on disk index and vectors. try: qdrant_client.create_collection( @@ -104,10 +109,10 @@ def _create_collection(self, dim: int, qdrant_client: QdrantClient): on_disk=self.case_config.index_param()["on_disk"], ), hnsw_config=HnswConfigDiff( - m = self.case_config.index_param()["m"], + m=self.case_config.index_param()["m"], ef_construct=self.case_config.index_param()["ef_construct"], on_disk=self.case_config.index_param()["on_disk"], - ) + ), ) qdrant_client.create_payload_index( @@ -121,7 +126,7 @@ def _create_collection(self, dim: int, qdrant_client: QdrantClient): return log.warning(f"Failed to create collection: {self.collection_name} error: {e}") raise e from None - + def optimize(self, data_size: int | None = None): assert self.client, "Please call self.init() before" # wait for vectors to be fully indexed @@ -139,11 +144,11 @@ def optimize(self, data_size: int | None = None): ) log.info(msg) return - + except Exception as e: log.warning(f"QdrantCloud ready to search error: {e}") raise e from None - + def insert_embeddings( self, embeddings: Iterable[list[float]], @@ -163,7 +168,7 @@ def insert_embeddings( assert self.client is not None assert len(embeddings) == len(metadata) insert_count = 0 - + # disable indexing for quick insertion self.client.update_collection( collection_name=self.collection_name, @@ -185,13 +190,13 @@ def insert_embeddings( collection_name=self.collection_name, optimizer_config=OptimizersConfigDiff(indexing_threshold=100), ) - + except Exception as e: log.info(f"Failed to insert data, {e}") return insert_count, e else: return insert_count, None - + def search_embedding( self, query: list[float], @@ -203,7 +208,7 @@ def search_embedding( Should call self.init() first. """ assert self.client is not None - + f = None if filters: f = Filter( @@ -215,17 +220,13 @@ def search_embedding( ), ), ], - ) - res = ( - self.client.query_points( - collection_name=self.collection_name, - query=query, - limit=k, - query_filter=f, - search_params=SearchParams(**self.search_parameter), - - ).points - ) - - return [result.id for result in res] + ) + res = self.client.query_points( + collection_name=self.collection_name, + query=query, + limit=k, + query_filter=f, + search_params=SearchParams(**self.search_parameter), + ).points + return [result.id for result in res] diff --git a/vectordb_bench/backend/clients/weaviate_cloud/cli.py b/vectordb_bench/backend/clients/weaviate_cloud/cli.py index 9faf768a6..cba3c2377 100644 --- a/vectordb_bench/backend/clients/weaviate_cloud/cli.py +++ b/vectordb_bench/backend/clients/weaviate_cloud/cli.py @@ -15,7 +15,7 @@ class WeaviateTypedDict(CommonTypedDict): api_key: Annotated[ str, - click.option("--api-key", type=str, help="Weaviate api key", required=False, default=''), + click.option("--api-key", type=str, help="Weaviate api key", required=False, default=""), ] url: Annotated[ str, @@ -23,25 +23,24 @@ class WeaviateTypedDict(CommonTypedDict): ] no_auth: Annotated[ bool, - click.option("--no-auth", is_flag=True, help="Do not use api-key, set it to true if you are using a local setup. Default is False.", default=False), + click.option( + "--no-auth", + is_flag=True, + help="Do not use api-key, set it to true if you are using a local setup. Default is False.", + default=False, + ), ] m: Annotated[ int, - click.option( - "--m", type=int, default=16, help="HNSW index parameter m." - ), + click.option("--m", type=int, default=16, help="HNSW index parameter m."), ] ef_construct: Annotated[ int, - click.option( - "--ef-construction", type=int, default=256, help="HNSW index parameter ef_construction" - ), + click.option("--ef-construction", type=int, default=256, help="HNSW index parameter ef_construction"), ] ef: Annotated[ int, - click.option( - "--ef", type=int, default=256, help="HNSW index parameter ef for search" - ), + click.option("--ef", type=int, default=256, help="HNSW index parameter ef for search"), ] @@ -54,7 +53,7 @@ def Weaviate(**parameters: Unpack[WeaviateTypedDict]): db=DB.WeaviateCloud, db_config=WeaviateConfig( db_label=parameters["db_label"], - api_key=SecretStr(parameters["api_key"]) if parameters["api_key"] != '' else SecretStr("-"), + api_key=SecretStr(parameters["api_key"]) if parameters["api_key"] != "" else SecretStr("-"), url=SecretStr(parameters["url"]), no_auth=parameters["no_auth"], ), diff --git a/vectordb_bench/backend/clients/weaviate_cloud/weaviate_cloud.py b/vectordb_bench/backend/clients/weaviate_cloud/weaviate_cloud.py index 18a17a661..d6111c8da 100644 --- a/vectordb_bench/backend/clients/weaviate_cloud/weaviate_cloud.py +++ b/vectordb_bench/backend/clients/weaviate_cloud/weaviate_cloud.py @@ -37,11 +37,11 @@ def __init__( self._scalar_field = "key" self._vector_field = "vector" self._index_name = "vector_idx" - - # If local setup is used, we - if db_config['no_auth']: - del db_config['auth_client_secret'] - del db_config['no_auth'] + + # If local setup is used, we + if db_config["no_auth"]: + del db_config["auth_client_secret"] + del db_config["no_auth"] from weaviate import Client From 2884761a4069f92d66ee31b175746eb55ca674b6 Mon Sep 17 00:00:00 2001 From: Zebin Ren Date: Tue, 10 Jun 2025 01:13:31 +0200 Subject: [PATCH 179/327] change lancedb vector type to float32 --- vectordb_bench/backend/clients/lancedb/lancedb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vectordb_bench/backend/clients/lancedb/lancedb.py b/vectordb_bench/backend/clients/lancedb/lancedb.py index 334a7ccf8..eb10ecadd 100644 --- a/vectordb_bench/backend/clients/lancedb/lancedb.py +++ b/vectordb_bench/backend/clients/lancedb/lancedb.py @@ -45,7 +45,7 @@ def __init__( db.open_table(self.table_name) except Exception: schema = pa.schema( - [pa.field("id", pa.int64()), pa.field("vector", pa.list_(pa.float64(), list_size=self.dim))] + [pa.field("id", pa.int64()), pa.field("vector", pa.list_(pa.float32(), list_size=self.dim))] ) db.create_table(self.table_name, schema=schema, mode="overwrite") From af8cc1a854ef26e6aedc6678782c74655162600c Mon Sep 17 00:00:00 2001 From: Zebin Ren Date: Tue, 10 Jun 2025 16:04:27 +0200 Subject: [PATCH 180/327] add num_shards to MilvusConfig.to_dict() --- vectordb_bench/backend/clients/milvus/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/vectordb_bench/backend/clients/milvus/config.py b/vectordb_bench/backend/clients/milvus/config.py index bf5920817..a0e04fbf8 100644 --- a/vectordb_bench/backend/clients/milvus/config.py +++ b/vectordb_bench/backend/clients/milvus/config.py @@ -14,6 +14,7 @@ def to_dict(self) -> dict: "uri": self.uri.get_secret_value(), "user": self.user if self.user else None, "password": self.password.get_secret_value() if self.password else None, + "num_shards": self.num_shards, } @validator("*") From 4d6d8dfaf36d2b70defd1d9e1fdea533d9f5ae65 Mon Sep 17 00:00:00 2001 From: ZebinRen <80635329+ZebinRen@users.noreply.github.com> Date: Wed, 11 Jun 2025 08:38:40 +0200 Subject: [PATCH 181/327] expose lancedb index parameters to the cli interface (#537) * expose lancedb index parameters to the cli interface * fix formatting issues --- vectordb_bench/backend/clients/lancedb/cli.py | 70 ++++++++++++++++--- .../backend/clients/lancedb/config.py | 15 +++- .../backend/clients/lancedb/lancedb.py | 28 +++++--- 3 files changed, 96 insertions(+), 17 deletions(-) diff --git a/vectordb_bench/backend/clients/lancedb/cli.py b/vectordb_bench/backend/clients/lancedb/cli.py index 573c64d05..219ae114e 100644 --- a/vectordb_bench/backend/clients/lancedb/cli.py +++ b/vectordb_bench/backend/clients/lancedb/cli.py @@ -58,10 +58,46 @@ def LanceDBAutoIndex(**parameters: Unpack[LanceDBTypedDict]): ) +class LanceDBIVFPQTypedDict(CommonTypedDict, LanceDBTypedDict): + num_partitions: Annotated[ + int, + click.option( + "--num-partitions", + type=int, + default=0, + help="Number of partitions for IVFPQ index, unset = use LanceDB default", + ), + ] + num_sub_vectors: Annotated[ + int, + click.option( + "--num-sub-vectors", + type=int, + default=0, + help="Number of sub-vectors for IVFPQ index, unset = use LanceDB default", + ), + ] + nbits: Annotated[ + int, + click.option( + "--nbits", + type=int, + default=8, + help="Number of bits for IVFPQ index (must be 4 or 8), unset = use LanceDB default", + ), + ] + nprobes: Annotated[ + int, + click.option( + "--nprobes", type=int, default=0, help="Number of probes for IVFPQ search, unset = use LanceDB default" + ), + ] + + @cli.command() -@click_parameter_decorators_from_typed_dict(LanceDBTypedDict) -def LanceDBIVFPQ(**parameters: Unpack[LanceDBTypedDict]): - from .config import LanceDBConfig, _lancedb_case_config +@click_parameter_decorators_from_typed_dict(LanceDBIVFPQTypedDict) +def LanceDBIVFPQ(**parameters: Unpack[LanceDBIVFPQTypedDict]): + from .config import LanceDBConfig, LanceDBIndexConfig run( db=DB.LanceDB, @@ -70,15 +106,29 @@ def LanceDBIVFPQ(**parameters: Unpack[LanceDBTypedDict]): uri=parameters["uri"], token=SecretStr(parameters["token"]) if parameters.get("token") else None, ), - db_case_config=_lancedb_case_config.get(IndexType.IVFPQ)(), + db_case_config=LanceDBIndexConfig( + index=IndexType.IVFPQ, + num_partitions=parameters["num_partitions"], + num_sub_vectors=parameters["num_sub_vectors"], + nbits=parameters["nbits"], + nprobes=parameters["nprobes"], + ), **parameters, ) +class LanceDBHNSWTypedDict(CommonTypedDict, LanceDBTypedDict): + m: Annotated[int, click.option("--m", type=int, default=0, help="HNSW parameter m")] + ef_construction: Annotated[ + int, click.option("--ef-construction", type=int, default=0, help="HNSW parameter ef_construction") + ] + ef: Annotated[int, click.option("--ef", type=int, default=0, help="HNSW search parameter ef")] + + @cli.command() -@click_parameter_decorators_from_typed_dict(LanceDBTypedDict) -def LanceDBHNSW(**parameters: Unpack[LanceDBTypedDict]): - from .config import LanceDBConfig, _lancedb_case_config +@click_parameter_decorators_from_typed_dict(LanceDBHNSWTypedDict) +def LanceDBHNSW(**parameters: Unpack[LanceDBHNSWTypedDict]): + from .config import LanceDBConfig, LanceDBHNSWIndexConfig run( db=DB.LanceDB, @@ -87,6 +137,10 @@ def LanceDBHNSW(**parameters: Unpack[LanceDBTypedDict]): uri=parameters["uri"], token=SecretStr(parameters["token"]) if parameters.get("token") else None, ), - db_case_config=_lancedb_case_config.get(IndexType.HNSW)(), + db_case_config=LanceDBHNSWIndexConfig( + m=parameters["m"], + ef_construction=parameters["ef_construction"], + ef=parameters["ef"], + ), **parameters, ) diff --git a/vectordb_bench/backend/clients/lancedb/config.py b/vectordb_bench/backend/clients/lancedb/config.py index 0bbdfc4c9..b0621c22e 100644 --- a/vectordb_bench/backend/clients/lancedb/config.py +++ b/vectordb_bench/backend/clients/lancedb/config.py @@ -25,6 +25,7 @@ class LanceDBIndexConfig(BaseModel, DBCaseConfig): nbits: int = 8 # Must be 4 or 8 sample_rate: int = 256 max_iterations: int = 50 + nprobes: int = 0 def index_param(self) -> dict: if self.index not in [ @@ -52,7 +53,11 @@ def index_param(self) -> dict: return params def search_param(self) -> dict: - pass + params = {} + if self.nprobes > 0: + params["nprobes"] = self.nprobes + + return params def parse_metric(self) -> str: if self.metric_type in [MetricType.L2, MetricType.COSINE]: @@ -81,6 +86,7 @@ class LanceDBHNSWIndexConfig(LanceDBIndexConfig): index: IndexType = IndexType.HNSW m: int = 0 ef_construction: int = 0 + ef: int = 0 def index_param(self) -> dict: params = LanceDBIndexConfig.index_param(self) @@ -94,6 +100,13 @@ def index_param(self) -> dict: return params + def search_param(self) -> dict: + params = {} + if self.ef != 0: + params = {"ef": self.ef} + + return params + _lancedb_case_config = { IndexType.IVFPQ: LanceDBIndexConfig, diff --git a/vectordb_bench/backend/clients/lancedb/lancedb.py b/vectordb_bench/backend/clients/lancedb/lancedb.py index eb10ecadd..ad6b4d288 100644 --- a/vectordb_bench/backend/clients/lancedb/lancedb.py +++ b/vectordb_bench/backend/clients/lancedb/lancedb.py @@ -32,6 +32,10 @@ def __init__( self.table_name = collection_name self.dim = dim self.uri = db_config["uri"] + # avoid the search_param being called every time during the search process + self.search_config = db_case_config.search_param() + + log.info(f"Search config: {self.search_config}") db = lancedb.connect(self.uri) @@ -77,20 +81,28 @@ def search_embedding( filters: dict | None = None, ) -> list[int]: if filters: - results = ( - self.table.search(query) - .select(["id"]) - .where(f"id >= {filters['id']}", prefilter=True) - .limit(k) - .to_list() - ) + results = self.table.search(query).select(["id"]).where(f"id >= {filters['id']}", prefilter=True).limit(k) + if self.case_config.index == IndexType.IVFPQ and "nprobes" in self.search_config: + results = results.nprobes(self.search_config["nprobes"]).to_list() + elif self.case_config.index == IndexType.HNSW and "ef" in self.search_config: + results = results.ef(self.search_config["ef"]).to_list() + else: + results = results.to_list() else: - results = self.table.search(query).select(["id"]).limit(k).to_list() + results = self.table.search(query).select(["id"]).limit(k) + if self.case_config.index == IndexType.IVFPQ and "nprobes" in self.search_config: + results = results.nprobes(self.search_config["nprobes"]).to_list() + elif self.case_config.index == IndexType.HNSW and "ef" in self.search_config: + results = results.ef(self.search_config["ef"]).to_list() + else: + results = results.to_list() + return [int(result["id"]) for result in results] def optimize(self, data_size: int | None = None): if self.table and hasattr(self, "case_config") and self.case_config.index != IndexType.NONE: log.info(f"Creating index for LanceDB table ({self.table_name})") + log.info(f"Index parameters: {self.case_config.index_param()}") self.table.create_index(**self.case_config.index_param()) # Better recall with IVF_PQ (though still bad) but breaks HNSW: https://github.com/lancedb/lancedb/issues/2369 if self.case_config.index in (IndexType.IVFPQ, IndexType.AUTOINDEX): From 40da33c7a7601515cab1a1f613d2104eb1498813 Mon Sep 17 00:00:00 2001 From: Norris <33706975+norrishuang@users.noreply.github.com> Date: Wed, 11 Jun 2025 14:39:07 +0800 Subject: [PATCH 182/327] Add parameters of aws opensearch, support hnsw engine options, support multi-client load data (#527) * fix error of OpenSearch frontend:Validation error for AWSOpenSearchIndexConfig index_thread_qty_during_force_merge * add parameters of aws opensearch in frontend. * fiexd issue of opensearch ui and add multi-client indexing, add parameters of hnsw engine and metric-type. * fiexd issue of opensearch ui and add multi-client indexing, add parameters of hnsw engine and metric-type. * fiexd issue of opensearch ui and add multi-client indexing, add parameters of hnsw engine and metric-type. * clean logs * clean comments * clean logs * clean comments * fixed some issue, add warmup before search test only. * fixed some issue of create index and mulit-client * support faiss fp16 * clear comment * recover some code format * recover some code format * recover some code format * recover some code format * recover some code format * resolved conflict with pr: #529 --- .../clients/aws_opensearch/aws_opensearch.py | 163 +++++++++++++++++- .../backend/clients/aws_opensearch/cli.py | 37 ++-- .../backend/clients/aws_opensearch/config.py | 19 +- .../frontend/config/dbCaseConfigs.py | 114 +++++++++++- vectordb_bench/models.py | 7 + 5 files changed, 318 insertions(+), 22 deletions(-) diff --git a/vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py b/vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py index 42096f711..ae049bea4 100644 --- a/vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py +++ b/vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py @@ -44,6 +44,14 @@ def __init__( if is_existed: client.indices.delete(index=self.index_name) self._create_index(client) + else: + is_existed = client.indices.exists(index=self.index_name) + if not is_existed: + self._create_index(client) + log.info(f"AWS_OpenSearch client create index: {self.index_name}") + + self._update_ef_search_before_search(client) + self._load_graphs_to_memory(client) @classmethod def config_cls(cls) -> AWSOpenSearchConfig: @@ -53,7 +61,17 @@ def config_cls(cls) -> AWSOpenSearchConfig: def case_config_cls(cls, index_type: IndexType | None = None) -> AWSOpenSearchIndexConfig: return AWSOpenSearchIndexConfig - def _create_index(self, client: OpenSearch): + def _create_index(self, client: OpenSearch) -> None: + ef_search_value = ( + self.case_config.ef_search if self.case_config.ef_search is not None else self.case_config.efSearch + ) + log.info(f"Creating index with ef_search: {ef_search_value}") + log.info(f"Creating index with number_of_replicas: {self.case_config.number_of_replicas}") + + log.info(f"Creating index with engine: {self.case_config.engine}") + log.info(f"Creating index with metric type: {self.case_config.metric_type_name}") + log.info(f"All case_config parameters: {self.case_config.__dict__}") + cluster_settings_body = { "persistent": { "knn.algo_param.index_thread_qty": self.case_config.index_thread_qty, @@ -65,13 +83,13 @@ def _create_index(self, client: OpenSearch): "index": { "knn": True, "number_of_shards": self.case_config.number_of_shards, - "number_of_replicas": 0, - # Setting trans log threshold to 5GB + "number_of_replicas": self.case_config.number_of_replicas, "translog.flush_threshold_size": self.case_config.flush_threshold_size, "knn.advanced.approximate_threshold": "-1", }, "refresh_interval": self.case_config.refresh_interval, } + settings["index"]["knn.algo_param.ef_search"] = ef_search_value mappings = { "_source": {"excludes": [self.vector_col_name], "recovery_source_excludes": [self.vector_col_name]}, "properties": { @@ -84,6 +102,8 @@ def _create_index(self, client: OpenSearch): }, } try: + log.info(f"Creating index with settings: {settings}") + log.info(f"Creating index with mappings: {mappings}") client.indices.create( index=self.index_name, body={"settings": settings, "mappings": mappings}, @@ -110,6 +130,18 @@ def insert_embeddings( """Insert the embeddings to the opensearch.""" assert self.client is not None, "should self.init() first" + num_clients = self.case_config.number_of_indexing_clients or 1 + log.info(f"Number of indexing clients from case_config: {num_clients}") + + if num_clients <= 1: + log.info("Using single client for data insertion") + return self._insert_with_single_client(embeddings, metadata) + log.info(f"Using {num_clients} parallel clients for data insertion") + return self._insert_with_multiple_clients(embeddings, metadata, num_clients) + + def _insert_with_single_client( + self, embeddings: Iterable[list[float]], metadata: list[int] + ) -> tuple[int, Exception]: insert_data = [] for i in range(len(embeddings)): insert_data.append( @@ -127,7 +159,108 @@ def insert_embeddings( except Exception as e: log.warning(f"Failed to insert data: {self.index_name} error: {e!s}") time.sleep(10) - return self.insert_embeddings(embeddings, metadata) + return self._insert_with_single_client(embeddings, metadata) + + def _insert_with_multiple_clients( + self, embeddings: Iterable[list[float]], metadata: list[int], num_clients: int + ) -> tuple[int, Exception]: + import concurrent.futures + from concurrent.futures import ThreadPoolExecutor + + embeddings_list = list(embeddings) + chunk_size = max(1, len(embeddings_list) // num_clients) + chunks = [] + + for i in range(0, len(embeddings_list), chunk_size): + end = min(i + chunk_size, len(embeddings_list)) + chunks.append((embeddings_list[i:end], metadata[i:end])) + + clients = [] + for _ in range(min(num_clients, len(chunks))): + client = OpenSearch(**self.db_config) + clients.append(client) + + log.info(f"AWS_OpenSearch using {len(clients)} parallel clients for data insertion") + + def insert_chunk(client_idx: int, chunk_idx: int): + chunk_embeddings, chunk_metadata = chunks[chunk_idx] + client = clients[client_idx] + + insert_data = [] + for i in range(len(chunk_embeddings)): + insert_data.append( + {"index": {"_index": self.index_name, self.id_col_name: chunk_metadata[i]}}, + ) + insert_data.append({self.vector_col_name: chunk_embeddings[i]}) + + try: + resp = client.bulk(insert_data) + log.info(f"Client {client_idx} added {len(resp['items'])} documents") + return len(chunk_embeddings), None + except Exception as e: + log.warning(f"Client {client_idx} failed to insert data: {e!s}") + return 0, e + + results = [] + with ThreadPoolExecutor(max_workers=len(clients)) as executor: + futures = [] + + for chunk_idx in range(len(chunks)): + client_idx = chunk_idx % len(clients) + futures.append(executor.submit(insert_chunk, client_idx, chunk_idx)) + + for future in concurrent.futures.as_completed(futures): + count, error = future.result() + results.append((count, error)) + + from contextlib import suppress + + for client in clients: + with suppress(Exception): + client.close() + + total_count = sum(count for count, _ in results) + errors = [error for _, error in results if error is not None] + + if errors: + log.warning("Some clients failed to insert data, retrying with single client") + time.sleep(10) + return self._insert_with_single_client(embeddings, metadata) + + resp = self.client.indices.stats(self.index_name) + log.info( + f"""Total document count in index after parallel insertion: + {resp['_all']['primaries']['indexing']['index_total']}""", + ) + + return (total_count, None) + + def _update_ef_search_before_search(self, client: OpenSearch): + ef_search_value = ( + self.case_config.ef_search if self.case_config.ef_search is not None else self.case_config.efSearch + ) + + try: + index_settings = client.indices.get_settings(index=self.index_name) + current_ef_search = ( + index_settings.get(self.index_name, {}) + .get("settings", {}) + .get("index", {}) + .get("knn.algo_param", {}) + .get("ef_search") + ) + + if current_ef_search != str(ef_search_value): + log.info(f"Updating ef_search before search from {current_ef_search} to {ef_search_value}") + settings_body = {"index": {"knn.algo_param.ef_search": ef_search_value}} + client.indices.put_settings(index=self.index_name, body=settings_body) + log.info(f"Successfully updated ef_search to {ef_search_value} before search") + + log.info(f"Current engine: {self.case_config.engine}") + log.info(f"Current metric_type: {self.case_config.metric_type_name}") + + except Exception as e: + log.warning(f"Failed to update ef_search parameter before search: {e}") def search_embedding( self, @@ -181,6 +314,7 @@ def search_embedding( def optimize(self, data_size: int | None = None): """optimize will be called between insertion and search in performance cases.""" + self._update_ef_search() # Call refresh first to ensure that all segments are created self._refresh_index() if self.case_config.force_merge_enabled: @@ -190,7 +324,22 @@ def optimize(self, data_size: int | None = None): # Call refresh again to ensure that the index is ready after force merge. self._refresh_index() # ensure that all graphs are loaded in memory and ready for search - self._load_graphs_to_memory() + self._load_graphs_to_memory(self.client) + + def _update_ef_search(self): + ef_search_value = ( + self.case_config.ef_search if self.case_config.ef_search is not None else self.case_config.efSearch + ) + log.info(f"Updating ef_search parameter to: {ef_search_value}") + + settings_body = {"index": {"knn.algo_param.ef_search": ef_search_value}} + try: + self.client.indices.put_settings(index=self.index_name, body=settings_body) + log.info(f"Successfully updated ef_search to {ef_search_value}") + log.info(f"Current engine: {self.case_config.engine}") + log.info(f"Current metric_type: {self.case_config.metric_type}") + except Exception as e: + log.warning(f"Failed to update ef_search parameter: {e}") def _update_replicas(self): index_settings = self.client.indices.get_settings(index=self.index_name) @@ -254,8 +403,8 @@ def _do_force_merge(self): break log.debug(f"Completed force merge for index {self.index_name}") - def _load_graphs_to_memory(self): + def _load_graphs_to_memory(self, client: OpenSearch): if self.case_config.engine != AWSOS_Engine.lucene: log.info("Calling warmup API to load graphs into memory") warmup_endpoint = f"/_plugins/_knn/warmup/{self.index_name}" - self.client.transport.perform_request("GET", warmup_endpoint) + client.transport.perform_request("GET", warmup_endpoint) diff --git a/vectordb_bench/backend/clients/aws_opensearch/cli.py b/vectordb_bench/backend/clients/aws_opensearch/cli.py index ae401aec1..01df7625a 100644 --- a/vectordb_bench/backend/clients/aws_opensearch/cli.py +++ b/vectordb_bench/backend/clients/aws_opensearch/cli.py @@ -1,3 +1,4 @@ +import logging from typing import Annotated, TypedDict, Unpack import click @@ -5,7 +6,7 @@ from ....cli.cli import ( CommonTypedDict, - HNSWFlavor2, + HNSWFlavor1, cli, click_parameter_decorators_from_typed_dict, run, @@ -13,6 +14,8 @@ from .. import DB from .config import AWSOS_Engine, AWSOSQuantization +log = logging.getLogger(__name__) + class AWSOpenSearchTypedDict(TypedDict): host: Annotated[str, click.option("--host", type=str, help="Db host", required=True)] @@ -39,13 +42,23 @@ class AWSOpenSearchTypedDict(TypedDict): ), ] - index_thread_qty_during_force_merge: Annotated[ - int, + engine: Annotated[ + str, click.option( - "--index-thread-qty-during-force-merge", - type=int, - help="Thread count during force merge operations", - default=4, + "--engine", + type=click.Choice(["nmslib", "faiss", "lucene"], case_sensitive=False), + help="HNSW algorithm implementation to use", + default="faiss", + ), + ] + + metric_type: Annotated[ + str, + click.option( + "--metric-type", + type=click.Choice(["l2", "cosine", "ip"], case_sensitive=False), + help="Distance metric type for vector similarity", + default="l2", ), ] @@ -55,26 +68,26 @@ class AWSOpenSearchTypedDict(TypedDict): ] refresh_interval: Annotated[ - int, + str, click.option( "--refresh-interval", type=str, help="How often to make new data available for search", default="60s" ), ] force_merge_enabled: Annotated[ - int, + bool, click.option("--force-merge-enabled", type=bool, help="Whether to perform force merge operation", default=True), ] flush_threshold_size: Annotated[ - int, + str, click.option( "--flush-threshold-size", type=str, help="Size threshold for flushing the transaction log", default="5120mb" ), ] cb_threshold: Annotated[ - int, + str, click.option( "--cb-threshold", type=str, @@ -106,7 +119,7 @@ class AWSOpenSearchTypedDict(TypedDict): ] -class AWSOpenSearchHNSWTypedDict(CommonTypedDict, AWSOpenSearchTypedDict, HNSWFlavor2): ... +class AWSOpenSearchHNSWTypedDict(CommonTypedDict, AWSOpenSearchTypedDict, HNSWFlavor1): ... @cli.command() diff --git a/vectordb_bench/backend/clients/aws_opensearch/config.py b/vectordb_bench/backend/clients/aws_opensearch/config.py index cf5192163..2329507fc 100644 --- a/vectordb_bench/backend/clients/aws_opensearch/config.py +++ b/vectordb_bench/backend/clients/aws_opensearch/config.py @@ -45,7 +45,9 @@ class AWSOpenSearchIndexConfig(BaseModel, DBCaseConfig): metric_type: MetricType = MetricType.L2 engine: AWSOS_Engine = AWSOS_Engine.faiss efConstruction: int = 256 - efSearch: int = 256 + ef_search: int = 200 + engine_name: str | None = None + metric_type_name: str | None = None M: int = 16 index_thread_qty: int | None = 4 number_of_shards: int | None = 1 @@ -59,16 +61,29 @@ class AWSOpenSearchIndexConfig(BaseModel, DBCaseConfig): quantization_type: AWSOSQuantization = AWSOSQuantization.fp32 def parse_metric(self) -> str: + log.info(f"User specified metric_type: {self.metric_type_name}") + self.metric_type = MetricType[self.metric_type_name.upper()] if self.metric_type == MetricType.IP: return "innerproduct" if self.metric_type == MetricType.COSINE: return "cosinesimil" + if self.metric_type == MetricType.L2: + log.info("Using l2 as specified by user") + return "l2" return "l2" def index_param(self) -> dict: + log.info(f"Using engine: {self.engine} for index creation") + log.info(f"Using metric_type: {self.metric_type_name} for index creation") + log.info(f"Resulting space_type: {self.parse_metric()} for index creation") + + parameters = {"ef_construction": self.efConstruction, "m": self.M} + + if self.engine == AWSOS_Engine.faiss and self.faiss_use_fp16: + parameters["encoder"] = {"name": "sq", "parameters": {"type": "fp16"}} + return { "name": "hnsw", - "space_type": self.parse_metric(), "engine": self.engine.value, "parameters": { "ef_construction": self.efConstruction, diff --git a/vectordb_bench/frontend/config/dbCaseConfigs.py b/vectordb_bench/frontend/config/dbCaseConfigs.py index 3c9430b2b..0ccc74af9 100644 --- a/vectordb_bench/frontend/config/dbCaseConfigs.py +++ b/vectordb_bench/frontend/config/dbCaseConfigs.py @@ -146,6 +146,7 @@ class InputType(IntEnum): Option = 20003 Float = 20004 Bool = 20005 + Select = 20006 class CaseConfigInput(BaseModel): @@ -482,7 +483,7 @@ class CaseConfigInput(BaseModel): label=CaseConfigParamType.ef_search, inputType=InputType.Number, inputConfig={ - "min": 100, + "min": 1, "max": 1024, "value": 256, }, @@ -1264,6 +1265,87 @@ class CaseConfigInput(BaseModel): isDisplayed=lambda config: config[CaseConfigParamType.IndexType] == IndexType.HNSW.value, ) +CaseConfigParamInput_INDEX_THREAD_QTY_DURING_FORCE_MERGE_AWSOpensearch = CaseConfigInput( + label=CaseConfigParamType.index_thread_qty_during_force_merge, + displayLabel="Index Thread Qty During Force Merge", + inputHelp="Thread count during force merge operations", + inputType=InputType.Number, + inputConfig={ + "min": 1, + "max": 32, + "value": 4, + }, +) + +CaseConfigParamInput_NUMBER_OF_INDEXING_CLIENTS_AWSOpensearch = CaseConfigInput( + label=CaseConfigParamType.number_of_indexing_clients, + displayLabel="Number of Indexing Clients", + inputHelp="Number of concurrent clients for data insertion", + inputType=InputType.Number, + inputConfig={ + "min": 1, + "max": 32, + "value": 1, + }, +) + +CaseConfigParamInput_NUMBER_OF_SHARDS_AWSOpensearch = CaseConfigInput( + label=CaseConfigParamType.number_of_shards, + displayLabel="Number of Shards", + inputHelp="Number of primary shards for the index", + inputType=InputType.Number, + inputConfig={ + "min": 1, + "max": 32, + "value": 1, + }, +) + +CaseConfigParamInput_NUMBER_OF_REPLICAS_AWSOpensearch = CaseConfigInput( + label=CaseConfigParamType.number_of_replicas, + displayLabel="Number of Replicas", + inputHelp="Number of replica copies for each primary shard", + inputType=InputType.Number, + inputConfig={ + "min": 0, + "max": 10, + "value": 1, + }, +) + +CaseConfigParamInput_INDEX_THREAD_QTY_AWSOpensearch = CaseConfigInput( + label=CaseConfigParamType.index_thread_qty, + displayLabel="Index Thread Qty", + inputHelp="Thread count for native engine indexing", + inputType=InputType.Number, + inputConfig={ + "min": 1, + "max": 32, + "value": 4, + }, +) + +CaseConfigParamInput_ENGINE_NAME_AWSOpensearch = CaseConfigInput( + label=CaseConfigParamType.engine_name, + displayLabel="Engine", + inputHelp="HNSW algorithm implementation to use", + inputType=InputType.Option, + inputConfig={ + "options": ["faiss", "nmslib", "lucene"], + "default": "faiss", + }, +) + +CaseConfigParamInput_METRIC_TYPE_NAME_AWSOpensearch = CaseConfigInput( + label=CaseConfigParamType.metric_type_name, + displayLabel="Metric Type", + inputHelp="Distance metric type for vector similarity", + inputType=InputType.Option, + inputConfig={ + "options": ["l2", "cosine", "ip"], + "default": "l2", + }, +) MilvusLoadConfig = [ CaseConfigParamInput_IndexType, @@ -1612,6 +1694,32 @@ class CaseConfigInput(BaseModel): LanceDBPerformanceConfig = LanceDBLoadConfig +AWSOpensearchLoadingConfig = [ + CaseConfigParamInput_EFConstruction_AWSOpensearch, + CaseConfigParamInput_M_AWSOpensearch, + CaseConfigParamInput_ENGINE_NAME_AWSOpensearch, + CaseConfigParamInput_METRIC_TYPE_NAME_AWSOpensearch, + CaseConfigParamInput_INDEX_THREAD_QTY_DURING_FORCE_MERGE_AWSOpensearch, + CaseConfigParamInput_NUMBER_OF_INDEXING_CLIENTS_AWSOpensearch, + CaseConfigParamInput_NUMBER_OF_SHARDS_AWSOpensearch, + CaseConfigParamInput_NUMBER_OF_REPLICAS_AWSOpensearch, + CaseConfigParamInput_INDEX_THREAD_QTY_AWSOpensearch, +] + +AWSOpenSearchPerformanceConfig = [ + CaseConfigParamInput_EFConstruction_AWSOpensearch, + CaseConfigParamInput_M_AWSOpensearch, + CaseConfigParamInput_EF_SEARCH_AWSOpensearch, + CaseConfigParamInput_ENGINE_NAME_AWSOpensearch, + CaseConfigParamInput_METRIC_TYPE_NAME_AWSOpensearch, + CaseConfigParamInput_INDEX_THREAD_QTY_DURING_FORCE_MERGE_AWSOpensearch, + CaseConfigParamInput_NUMBER_OF_INDEXING_CLIENTS_AWSOpensearch, + CaseConfigParamInput_NUMBER_OF_SHARDS_AWSOpensearch, + CaseConfigParamInput_NUMBER_OF_REPLICAS_AWSOpensearch, + CaseConfigParamInput_INDEX_THREAD_QTY_AWSOpensearch, +] + +# Map DB to config CASE_CONFIG_MAP = { DB.Milvus: { CaseLabel.Load: MilvusLoadConfig, @@ -1676,4 +1784,8 @@ class CaseConfigInput(BaseModel): CaseLabel.Load: LanceDBLoadConfig, CaseLabel.Performance: LanceDBPerformanceConfig, }, + DB.AWSOpenSearch: { + CaseLabel.Load: AWSOpensearchLoadingConfig, + CaseLabel.Performance: AWSOpenSearchPerformanceConfig, + }, } diff --git a/vectordb_bench/models.py b/vectordb_bench/models.py index c35c21755..66cc9deaa 100644 --- a/vectordb_bench/models.py +++ b/vectordb_bench/models.py @@ -105,6 +105,13 @@ class CaseConfigParamType(Enum): num_partitions = "num_partitions" num_sub_vectors = "num_sub_vectors" sample_rate = "sample_rate" + index_thread_qty_during_force_merge = "index_thread_qty_during_force_merge" + number_of_indexing_clients = "number_of_indexing_clients" + number_of_shards = "number_of_shards" + number_of_replicas = "number_of_replicas" + index_thread_qty = "index_thread_qty" + engine_name = "engine_name" + metric_type_name = "metric_type_name" # mongodb params mongodb_quantization_type = "quantization" From 34b3b256ce0125c3667d1704eff63918173be63e Mon Sep 17 00:00:00 2001 From: fan <37357096+wyfanxiao@users.noreply.github.com> Date: Sat, 14 Jun 2025 10:07:37 +0800 Subject: [PATCH 183/327] Add OceanBase Database Support to VectorDBBench (#540) * support oceanbase * fix bug * fix bug * support IP * support multiple zone * support filter case * support ivf * support hnsw_bq and extra info * optimized formatting * optimized formatting * Revert dbCaseConfigs.py to match remote main branch * resolve review comments * Update __init__.py * Addressed review comments * Fixed code style issues --- README.md | 67 ++++++ install/requirements_py3.11.txt | 1 + pyproject.toml | 2 + vectordb_bench/backend/clients/__init__.py | 16 ++ vectordb_bench/backend/clients/api.py | 1 + .../backend/clients/oceanbase/cli.py | 100 ++++++++ .../backend/clients/oceanbase/config.py | 125 ++++++++++ .../backend/clients/oceanbase/oceanbase.py | 215 ++++++++++++++++++ vectordb_bench/cli/cli.py | 58 +++++ vectordb_bench/cli/vectordbbench.py | 3 + 10 files changed, 588 insertions(+) create mode 100644 vectordb_bench/backend/clients/oceanbase/cli.py create mode 100644 vectordb_bench/backend/clients/oceanbase/config.py create mode 100644 vectordb_bench/backend/clients/oceanbase/oceanbase.py diff --git a/README.md b/README.md index 19bfdaf61..b3c9e53f3 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,7 @@ All the database client supported | mongodb | `pip install vectordb-bench[mongodb]` | | tidb | `pip install vectordb-bench[tidb]` | | vespa | `pip install vectordb-bench[vespa]` | +| oceanbase | `pip install vectordb-bench[oceanbase]` | ### Run @@ -209,6 +210,72 @@ Options: --quantization-type TEXT which type of quantization to use valid values [fp32, fp16] --help Show this message and exit. ``` +### Run OceanBase from command line + +Execute tests for the index types: HNSW, HNSW_SQ, or HNSW_BQ. + +```shell +vectordbbench oceanbasehnsw --host xxx --port xxx --user root@mysql_tenant --database test \ +--m 16 --ef-construction 200 --case-type Performance1536D50K \ +--index-type HNSW --ef-search 100 +``` + +To list the options for oceanbase, execute `vectordbbench oceanbasehnsw --help`, The following are some OceanBase-specific command-line options. + +```text +$ vectordbbench oceanbasehnsw --help +Usage: vectordbbench oceanbasehnsw [OPTIONS] + +Options: + [...] + --host TEXT OceanBase host + --user TEXT OceanBase username [required] + --password TEXT OceanBase database password + --database TEXT DataBase name [required] + --port INTEGER OceanBase port [required] + --m INTEGER hnsw m [required] + --ef-construction INTEGER hnsw ef-construction [required] + --ef-search INTEGER hnsw ef-search [required] + --index-type [HNSW|HNSW_SQ|HNSW_BQ] + Type of index to use. Supported values: + HNSW, HNSW_SQ, HNSW_BQ [required] + --help Show this message and exit. + ``` + +Execute tests for the index types: IVF_FLAT, IVF_SQ8, or IVF_PQ. + +```shell +vectordbbench oceanbaseivf --host xxx --port xxx --user root@mysql_tenant --database test \ +--nlist 1000 --sample_per_nlist 256 --case-type Performance768D1M \ +--index-type IVF_FLAT --ivf_nprobes 100 +``` + +To list the options for oceanbase, execute `vectordbbench oceanbaseivf --help`, The following are some OceanBase-specific command-line options. + +```text +$ vectordbbench oceanbaseivf --help +Usage: vectordbbench oceanbaseivf [OPTIONS] + +Options: + [...] + --host TEXT OceanBase host + --user TEXT OceanBase username [required] + --password TEXT OceanBase database password + --database TEXT DataBase name [required] + --port INTEGER OceanBase port [required] + --index-type [IVF_FLAT|IVF_SQ8|IVF_PQ] + Type of index to use. Supported values: + IVF_FLAT, IVF_SQ8, IVF_PQ [required] + --nlist INTEGER Number of cluster centers [required] + --sample_per_nlist INTEGER The cluster centers are calculated by total + sampling sample_per_nlist * nlist vectors + [required] + --ivf_nprobes TEXT How many clustering centers to search during + the query [required] + --m INTEGER The number of sub-vectors that each data + vector is divided into during IVF-PQ + --help Show this message and exit. Show this message and exit. + ``` #### Using a configuration file. diff --git a/install/requirements_py3.11.txt b/install/requirements_py3.11.txt index 86958ada2..0ae328a6f 100644 --- a/install/requirements_py3.11.txt +++ b/install/requirements_py3.11.txt @@ -24,3 +24,4 @@ scikit-learn pymilvus clickhouse_connect pyvespa +mysql-connector-python \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 8cab39194..d452e14ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,6 +73,7 @@ all = [ "clickhouse-connect", "pyvespa", "lancedb", + "mysql-connector-python", ] qdrant = [ "qdrant-client" ] @@ -96,6 +97,7 @@ tidb = [ "PyMySQL" ] clickhouse = [ "clickhouse-connect" ] vespa = [ "pyvespa" ] lancedb = [ "lancedb" ] +oceanbase = [ "mysql-connector-python" ] [project.urls] "repository" = "https://github.com/zilliztech/VectorDBBench" diff --git a/vectordb_bench/backend/clients/__init__.py b/vectordb_bench/backend/clients/__init__.py index c096a14c8..0d79f5a04 100644 --- a/vectordb_bench/backend/clients/__init__.py +++ b/vectordb_bench/backend/clients/__init__.py @@ -47,6 +47,7 @@ class DB(Enum): Clickhouse = "Clickhouse" Vespa = "Vespa" LanceDB = "LanceDB" + OceanBase = "OceanBase" @property def init_cls(self) -> type[VectorDB]: # noqa: PLR0911, PLR0912, C901, PLR0915 @@ -151,6 +152,11 @@ def init_cls(self) -> type[VectorDB]: # noqa: PLR0911, PLR0912, C901, PLR0915 return MongoDB + if self == DB.OceanBase: + from .oceanbase.oceanbase import OceanBase + + return OceanBase + if self == DB.MariaDB: from .mariadb.mariadb import MariaDB @@ -282,6 +288,11 @@ def config_cls(self) -> type[DBConfig]: # noqa: PLR0911, PLR0912, C901, PLR0915 return MongoDBConfig + if self == DB.OceanBase: + from .oceanbase.config import OceanBaseConfig + + return OceanBaseConfig + if self == DB.MariaDB: from .mariadb.config import MariaDBConfig @@ -394,6 +405,11 @@ def case_config_cls( # noqa: C901, PLR0911, PLR0912 return MongoDBIndexConfig + if self == DB.OceanBase: + from .oceanbase.config import _oceanbase_case_config + + return _oceanbase_case_config.get(index_type) + if self == DB.MariaDB: from .mariadb.config import _mariadb_case_config diff --git a/vectordb_bench/backend/clients/api.py b/vectordb_bench/backend/clients/api.py index ff7b378a7..6909b8ee1 100644 --- a/vectordb_bench/backend/clients/api.py +++ b/vectordb_bench/backend/clients/api.py @@ -17,6 +17,7 @@ class MetricType(str, Enum): class IndexType(str, Enum): HNSW = "HNSW" HNSW_SQ = "HNSW_SQ" + HNSW_BQ = "HNSW_BQ" HNSW_PQ = "HNSW_PQ" HNSW_PRQ = "HNSW_PRQ" DISKANN = "DISKANN" diff --git a/vectordb_bench/backend/clients/oceanbase/cli.py b/vectordb_bench/backend/clients/oceanbase/cli.py new file mode 100644 index 000000000..425bf9dd0 --- /dev/null +++ b/vectordb_bench/backend/clients/oceanbase/cli.py @@ -0,0 +1,100 @@ +import os +from typing import Annotated, Unpack + +import click +from pydantic import SecretStr + +from vectordb_bench.backend.clients import DB +from vectordb_bench.cli.cli import ( + CommonTypedDict, + HNSWFlavor4, + OceanBaseIVFTypedDict, + cli, + click_parameter_decorators_from_typed_dict, + run, +) + +from ..api import IndexType + + +class OceanBaseTypedDict(CommonTypedDict): + host: Annotated[str, click.option("--host", type=str, help="OceanBase host", default="")] + user: Annotated[str, click.option("--user", type=str, help="OceanBase username", required=True)] + password: Annotated[ + str, + click.option( + "--password", + type=str, + help="OceanBase database password", + default=lambda: os.environ.get("OB_PASSWORD", ""), + ), + ] + database: Annotated[str, click.option("--database", type=str, help="DataBase name", required=True)] + port: Annotated[int, click.option("--port", type=int, help="OceanBase port", required=True)] + + +class OceanBaseHNSWTypedDict(CommonTypedDict, OceanBaseTypedDict, HNSWFlavor4): ... + + +@cli.command() +@click_parameter_decorators_from_typed_dict(OceanBaseHNSWTypedDict) +def OceanBaseHNSW(**parameters: Unpack[OceanBaseHNSWTypedDict]): + from .config import OceanBaseConfig, OceanBaseHNSWConfig + + run( + db=DB.OceanBase, + db_config=OceanBaseConfig( + db_label=parameters["db_label"], + user=SecretStr(parameters["user"]), + password=SecretStr(parameters["password"]), + host=parameters["host"], + port=parameters["port"], + database=parameters["database"], + ), + db_case_config=OceanBaseHNSWConfig( + m=parameters["m"], + efConstruction=parameters["ef_construction"], + ef_search=parameters["ef_search"], + index=parameters["index_type"], + ), + **parameters, + ) + + +class OceanBaseIVFTypedDict(CommonTypedDict, OceanBaseTypedDict, OceanBaseIVFTypedDict): ... + + +@cli.command() +@click_parameter_decorators_from_typed_dict(OceanBaseIVFTypedDict) +def OceanBaseIVF(**parameters: Unpack[OceanBaseIVFTypedDict]): + from .config import OceanBaseConfig, OceanBaseIVFConfig + + type_str = parameters["index_type"] + if type_str == "IVF_FLAT": + input_index_type = IndexType.IVFFlat + elif type_str == "IVF_PQ": + input_index_type = IndexType.IVFPQ + elif type_str == "IVF_SQ8": + input_index_type = IndexType.IVFSQ8 + + input_m = 0 if parameters["m"] is None else parameters["m"] + + run( + db=DB.OceanBase, + db_config=OceanBaseConfig( + db_label=parameters["db_label"], + user=SecretStr(parameters["user"]), + password=SecretStr(parameters["password"]), + host=parameters["host"], + port=parameters["port"], + database=parameters["database"], + ), + db_case_config=OceanBaseIVFConfig( + m=input_m, + nlist=parameters["nlist"], + sample_per_nlist=parameters["sample_per_nlist"], + index=input_index_type, + ivf_nprobes=parameters["ivf_nprobes"], + ), + **parameters, + ) diff --git a/vectordb_bench/backend/clients/oceanbase/config.py b/vectordb_bench/backend/clients/oceanbase/config.py new file mode 100644 index 000000000..ce649f573 --- /dev/null +++ b/vectordb_bench/backend/clients/oceanbase/config.py @@ -0,0 +1,125 @@ +from typing import TypedDict + +from pydantic import BaseModel, SecretStr, validator + +from ..api import DBCaseConfig, DBConfig, IndexType, MetricType + + +class OceanBaseConfigDict(TypedDict): + user: str + host: str + port: str + password: str + database: str + + +class OceanBaseConfig(DBConfig): + user: SecretStr = SecretStr("root@perf") + password: SecretStr + host: str + port: int + database: str + + def to_dict(self) -> OceanBaseConfigDict: + user_str = self.user.get_secret_value() + pwd_str = self.password.get_secret_value() + return { + "user": user_str, + "host": self.host, + "port": self.port, + "password": pwd_str, + "database": self.database, + } + + @validator("*") + def not_empty_field(cls, v: any, field: any): + if field.name in ["password", "host", "db_label"]: + return v + if isinstance(v, str | SecretStr) and len(v) == 0: + raise ValueError("Empty string!") + return v + + +class OceanBaseIndexConfig(BaseModel): + index: IndexType + metric_type: MetricType | None = None + lib: str = "vsag" + + def parse_metric(self) -> str: + if self.metric_type == MetricType.L2 or ( + self.index == IndexType.HNSW_BQ and self.metric_type == MetricType.COSINE + ): + return "l2" + if self.metric_type == MetricType.IP: + return "inner_product" + return "cosine" + + def parse_metric_func_str(self) -> str: + if self.metric_type == MetricType.L2 or ( + self.index == IndexType.HNSW_BQ and self.metric_type == MetricType.COSINE + ): + return "l2_distance" + if self.metric_type == MetricType.IP: + return "negative_inner_product" + return "cosine_distance" + + +class OceanBaseHNSWConfig(OceanBaseIndexConfig, DBCaseConfig): + m: int + efConstruction: int + ef_search: int | None = None + index: IndexType + + def index_param(self) -> dict: + return { + "lib": self.lib, + "metric_type": self.parse_metric(), + "index_type": self.index.value, + "params": {"m": self.m, "ef_construction": self.efConstruction}, + } + + def search_param(self) -> dict: + return {"metric_type": self.parse_metric_func_str(), "params": {"ef_search": self.ef_search}} + + +class OceanBaseIVFConfig(OceanBaseIndexConfig, DBCaseConfig): + m: int + sample_per_nlist: int + nlist: int + index: IndexType + ivf_nprobes: int | None = None + + def index_param(self) -> dict: + if self.index == IndexType.IVFPQ: + return { + "lib": "OB", + "metric_type": self.parse_metric(), + "index_type": self.index.value, + "params": { + "m": self.M, + "sample_per_nlist": self.sample_per_nlist, + "nlist": self.nlist, + }, + } + return { + "lib": "OB", + "metric_type": self.parse_metric(), + "index_type": self.index.value, + "params": { + "sample_per_nlist": self.sample_per_nlist, + "nlist": self.nlist, + }, + } + + def search_param(self) -> dict: + return {"metric_type": self.metric_type, "params": {"ivf_nprobes": self.ivf_nprobes}} + + +_oceanbase_case_config = { + IndexType.HNSW_SQ: OceanBaseHNSWConfig, + IndexType.HNSW: OceanBaseHNSWConfig, + IndexType.HNSW_BQ: OceanBaseHNSWConfig, + IndexType.IVFFlat: OceanBaseIVFConfig, + IndexType.IVFPQ: OceanBaseIVFConfig, + IndexType.IVFSQ8: OceanBaseIVFConfig, +} diff --git a/vectordb_bench/backend/clients/oceanbase/oceanbase.py b/vectordb_bench/backend/clients/oceanbase/oceanbase.py new file mode 100644 index 000000000..dfaf5b75a --- /dev/null +++ b/vectordb_bench/backend/clients/oceanbase/oceanbase.py @@ -0,0 +1,215 @@ +import logging +import struct +import time +from collections.abc import Generator +from contextlib import contextmanager +from typing import Any + +import mysql.connector as mysql + +from ..api import IndexType, VectorDB +from .config import OceanBaseConfigDict, OceanBaseHNSWConfig + +log = logging.getLogger(__name__) + +OCEANBASE_DEFAULT_LOAD_BATCH_SIZE = 256 + + +class OceanBase(VectorDB): + def __init__( + self, + dim: int, + db_config: OceanBaseConfigDict, + db_case_config: OceanBaseHNSWConfig, + collection_name: str = "items", + drop_old: bool = False, + **kwargs, + ): + self.name = "OceanBase" + self.dim = dim + self.db_config = db_config + self.db_case_config = db_case_config + self.table_name = collection_name + self.load_batch_size = OCEANBASE_DEFAULT_LOAD_BATCH_SIZE + self._index_name = "vidx" + self._primary_field = "id" + self._vector_field = "embedding" + + log.info( + f"{self.name} initialized with config:\nDatabase: {self.db_config}\nCase Config: {self.db_case_config}" + ) + + self._conn = None + self._cursor = None + + try: + self._connect() + if drop_old: + self._drop_table() + self._create_table() + finally: + self._disconnect() + + def _connect(self): + try: + self._conn = mysql.connect( + host=self.db_config["host"], + user=self.db_config["user"], + port=self.db_config["port"], + password=self.db_config["password"], + database=self.db_config["database"], + ) + self._cursor = self._conn.cursor() + except mysql.Error: + log.exception("Failed to connect to the database") + raise + + def _disconnect(self): + if self._cursor: + self._cursor.close() + self._cursor = None + if self._conn: + self._conn.close() + self._conn = None + + @contextmanager + def init(self) -> Generator[None, None, None]: + try: + self._connect() + self._cursor.execute("SET autocommit=1") + + if self.db_case_config.index in {IndexType.HNSW, IndexType.HNSW_SQ, IndexType.HNSW_BQ}: + self._cursor.execute( + f"SET ob_hnsw_ef_search={(self.db_case_config.search_param())['params']['ef_search']}" + ) + else: + self._cursor.execute( + f"SET ob_ivf_nprobes={(self.db_case_config.search_param())['params']['ivf_nprobes']}" + ) + yield + finally: + self._disconnect() + + def _drop_table(self): + if not self._cursor: + raise ValueError("Cursor is not initialized") + + log.info(f"Dropping table {self.table_name}") + self._cursor.execute(f"DROP TABLE IF EXISTS {self.table_name}") + + def _create_table(self): + if not self._cursor: + raise ValueError("Cursor is not initialized") + + log.info(f"Creating table {self.table_name}") + create_table_query = f""" + CREATE TABLE {self.table_name} ( + id INT PRIMARY KEY, + embedding VECTOR({self.dim}) + ); + """ + self._cursor.execute(create_table_query) + + def optimize(self, data_size: int): + index_params = self.db_case_config.index_param() + index_args = ", ".join(f"{k}={v}" for k, v in index_params["params"].items()) + index_query = ( + f"CREATE /*+ PARALLEL(18) */ VECTOR INDEX idx1 " + f"ON {self.table_name}(embedding) " + f"WITH (distance={self.db_case_config.parse_metric()}, " + f"type={index_params['index_type']}, lib={index_params['lib']}, {index_args}" + ) + + if self.db_case_config.index in {IndexType.HNSW, IndexType.HNSW_SQ, IndexType.HNSW_BQ}: + index_query += ", extra_info_max_size=32" + + index_query += ")" + + log.info("Create index query: %s", index_query) + + try: + log.info("Creating index...") + start_time = time.time() + self._cursor.execute(index_query) + log.info(f"Index created in {time.time() - start_time:.2f} seconds") + + log.info("Performing major freeze...") + self._cursor.execute("ALTER SYSTEM MAJOR FREEZE;") + time.sleep(10) + self._wait_for_major_compaction() + + log.info("Gathering schema statistics...") + self._cursor.execute("CALL dbms_stats.gather_schema_stats('test', degree => 96);") + except mysql.Error: + log.exception("Failed to optimize index") + raise + + def need_normalize_cosine(self) -> bool: + if self.db_case_config.index == IndexType.HNSW_BQ: + log.info("current HNSW_BQ only supports L2, cosine dataset need normalize.") + return True + + return False + + def _wait_for_major_compaction(self): + while True: + self._cursor.execute( + "SELECT IF(COUNT(*) = COUNT(STATUS = 'IDLE' OR NULL), 'TRUE', 'FALSE') " + "AS all_status_idle FROM oceanbase.DBA_OB_ZONE_MAJOR_COMPACTION;" + ) + all_status_idle = self._cursor.fetchone()[0] + if all_status_idle == "TRUE": + break + time.sleep(10) + + def insert_embeddings( + self, + embeddings: list[list[float]], + metadata: list[int], + **kwargs: Any, + ) -> tuple[int, Exception | None]: + if not self._cursor: + raise ValueError("Cursor is not initialized") + + insert_count = 0 + try: + for batch_start in range(0, len(embeddings), self.load_batch_size): + batch_end = min(batch_start + self.load_batch_size, len(embeddings)) + batch = [(metadata[i], embeddings[i]) for i in range(batch_start, batch_end)] + values = ", ".join(f"({item_id}, '[{','.join(map(str, embedding))}]')" for item_id, embedding in batch) + self._cursor.execute( + f"INSERT /*+ ENABLE_PARALLEL_DML PARALLEL(32) */ INTO {self.table_name} VALUES {values}" # noqa: S608 + ) + insert_count += len(batch) + except mysql.Error: + log.exception("Failed to insert embeddings") + raise + + return insert_count, None + + def search_embedding( + self, + query: list[float], + k: int = 100, + filters: dict[str, Any] | None = None, + timeout: int | None = None, + ) -> list[int]: + if not self._cursor: + raise ValueError("Cursor is not initialized") + + packed = struct.pack(f"<{len(query)}f", *query) + hex_vec = packed.hex() + filter_clause = f"WHERE id >= {filters['id']}" if filters else "" + query_str = ( + f"SELECT id FROM {self.table_name} " # noqa: S608 + f"{filter_clause} ORDER BY " + f"{self.db_case_config.parse_metric_func_str()}(embedding, X'{hex_vec}') " + f"APPROXIMATE LIMIT {k}" + ) + + try: + self._cursor.execute(query_str) + return [row[0] for row in self._cursor.fetchall()] + except mysql.Error: + log.exception("Failed to execute search query") + raise diff --git a/vectordb_bench/cli/cli.py b/vectordb_bench/cli/cli.py index 1b0eb295b..ca30c590a 100644 --- a/vectordb_bench/cli/cli.py +++ b/vectordb_bench/cli/cli.py @@ -455,6 +455,22 @@ class HNSWFlavor3(HNSWBaseRequiredTypedDict): ] +class HNSWFlavor4(HNSWBaseRequiredTypedDict): + ef_search: Annotated[ + int | None, + click.option("--ef-search", type=int, help="hnsw ef-search", required=True), + ] + index_type: Annotated[ + str | None, + click.option( + "--index-type", + type=click.Choice(["HNSW", "HNSW_SQ", "HNSW_BQ"], case_sensitive=False), + help="Type of index to use. Supported values: HNSW, HNSW_SQ, HNSW_BQ", + required=True, + ), + ] + + class IVFFlatTypedDict(TypedDict): lists: Annotated[int | None, click.option("--lists", type=int, help="ivfflat lists")] probes: Annotated[int | None, click.option("--probes", type=int, help="ivfflat probes")] @@ -471,6 +487,48 @@ class IVFFlatTypedDictN(TypedDict): ] +class OceanBaseIVFTypedDict(TypedDict): + index_type: Annotated[ + str | None, + click.option( + "--index-type", + type=click.Choice(["IVF_FLAT", "IVF_SQ8", "IVF_PQ"], case_sensitive=False), + help="Type of index to use. Supported values: IVF_FLAT, IVF_SQ8, IVF_PQ", + required=True, + ), + ] + nlist: Annotated[ + int | None, + click.option("--nlist", "nlist", type=int, help="Number of cluster centers", required=True), + ] + sample_per_nlist: Annotated[ + int | None, + click.option( + "--sample_per_nlist", + "sample_per_nlist", + type=int, + help="The cluster centers are calculated by total sampling sample_per_nlist * nlist vectors", + required=True, + ), + ] + ivf_nprobes: Annotated[ + int | None, + click.option( + "--ivf_nprobes", + "ivf_nprobes", + type=str, + help="How many clustering centers to search during the query", + required=True, + ), + ] + m: Annotated[ + int | None, + click.option( + "--m", "m", type=int, help="The number of sub-vectors that each data vector is divided into during IVF-PQ" + ), + ] + + @click.group() def cli(): ... diff --git a/vectordb_bench/cli/vectordbbench.py b/vectordb_bench/cli/vectordbbench.py index 27bec6f26..0ef7c1b06 100644 --- a/vectordb_bench/cli/vectordbbench.py +++ b/vectordb_bench/cli/vectordbbench.py @@ -5,6 +5,7 @@ from ..backend.clients.mariadb.cli import MariaDBHNSW from ..backend.clients.memorydb.cli import MemoryDB from ..backend.clients.milvus.cli import MilvusAutoIndex +from ..backend.clients.oceanbase.cli import OceanBaseHNSW, OceanBaseIVF from ..backend.clients.pgdiskann.cli import PgDiskAnn from ..backend.clients.pgvecto_rs.cli import PgVectoRSHNSW, PgVectoRSIVFFlat from ..backend.clients.pgvector.cli import PgVectorHNSW @@ -33,6 +34,8 @@ cli.add_command(PgVectorScaleDiskAnn) cli.add_command(PgDiskAnn) cli.add_command(AlloyDBScaNN) +cli.add_command(OceanBaseHNSW) +cli.add_command(OceanBaseIVF) cli.add_command(MariaDBHNSW) cli.add_command(TiDB) cli.add_command(Clickhouse) From ef2585972b1b7671ec54e46baedb884f5973bc0c Mon Sep 17 00:00:00 2001 From: Min Tian Date: Mon, 16 Jun 2025 23:48:25 +0800 Subject: [PATCH 184/327] VectorDBBench 1.0 (#543) This commit marks the milestone release of VectorDBBench 1.0, introducing a wide range of new features, major enhancements, and updated benchmarks. Key changes include: - UI: Introduce a brand new homepage and navigation bar. The new design integrates powerful front-end pages for intuitive test result analysis and visualization. - Cases: Add new label-filter test cases. This allows testing search performance with metadata filters using expressions like color == "red". Initial support includes Milvus, Zilliz Cloud, Elasticsearch Cloud, Qdrant Cloud, Pinecone, and OpenSearch (AWS). - Cases: Implement new streaming test cases. These cases are designed to measure search performance while data is actively being inserted, simulating real-world "read-while-writing" scenarios. - Dataset: Add the new BioASQ dataset. This dataset is 1024-dimensional and comes in 1M and 10M sizes, enriching the diversity of our test data. - Custom Dataset: Enhance the custom dataset functionality. Users now have more flexible configuration options to simulate their own data distributions and schemas better. - New Results: Re-run and update all benchmark results for `Milvus`, `ZillizCloud`, `ElasticCloud`, `QdrantCloud`, `Pinecone`, and `OpenSearch(AWS)` to reflect their latest performance on the new test cases. --- .github/workflows/pull_request.yml | 1 + README.md | 43 +- fig/homepage/bar-chart.png | Bin 0 -> 81190 bytes fig/homepage/concurrent.png | Bin 0 -> 207344 bytes fig/homepage/custom.png | Bin 0 -> 75547 bytes fig/homepage/label_filter.png | Bin 0 -> 122594 bytes fig/homepage/qp$.png | Bin 0 -> 73754 bytes fig/homepage/run_test.png | Bin 0 -> 558438 bytes fig/homepage/streaming.png | Bin 0 -> 43722 bytes fig/homepage/table.png | Bin 0 -> 171580 bytes fig/run_test_select_case.png | Bin 0 -> 255926 bytes fig/run_test_select_db.png | Bin 0 -> 254473 bytes fig/run_test_submit.png | Bin 0 -> 50047 bytes pyproject.toml | 2 +- vectordb_bench/__init__.py | 41 +- vectordb_bench/backend/assembler.py | 25 +- vectordb_bench/backend/cases.py | 209 +- vectordb_bench/backend/clients/api.py | 22 +- .../clients/aws_opensearch/aws_opensearch.py | 123 +- .../backend/clients/aws_opensearch/config.py | 27 +- .../backend/clients/chroma/chroma.py | 8 +- .../backend/clients/elastic_cloud/config.py | 20 +- .../clients/elastic_cloud/elastic_cloud.py | 178 +- .../backend/clients/milvus/config.py | 1 + .../backend/clients/milvus/milvus.py | 96 +- .../backend/clients/pinecone/pinecone.py | 64 +- .../backend/clients/qdrant_cloud/config.py | 62 +- .../clients/qdrant_cloud/qdrant_cloud.py | 133 +- vectordb_bench/backend/dataset.py | 170 +- vectordb_bench/backend/filter.py | 76 + vectordb_bench/backend/runner/__init__.py | 6 +- vectordb_bench/backend/runner/mp_runner.py | 91 +- vectordb_bench/backend/runner/rate_runner.py | 120 +- .../backend/runner/read_write_runner.py | 193 +- .../backend/runner/serial_runner.py | 79 +- vectordb_bench/backend/task_runner.py | 68 +- vectordb_bench/cli/cli.py | 2 +- .../frontend/components/check_results/data.py | 27 +- .../components/check_results/filters.py | 78 +- .../components/check_results/headerIcon.py | 29 +- .../frontend/components/check_results/nav.py | 20 + .../components/custom/displayCustomCase.py | 51 +- .../components/custom/displaypPrams.py | 15 +- .../components/custom/getCustomConfig.py | 10 + .../components/label_filter/charts.py | 60 + .../components/run_test/caseSelector.py | 100 +- .../components/run_test/dbSelector.py | 14 +- .../components/run_test/inputWidget.py | 48 + .../components/run_test/submitTask.py | 4 +- .../frontend/components/streaming/charts.py | 253 + .../frontend/components/streaming/data.py | 62 + .../frontend/components/tables/data.py | 2 +- .../components/welcome/explainPrams.py | 66 + .../frontend/components/welcome/pagestyle.py | 106 + .../components/welcome/welcomePrams.py | 147 + .../frontend/config/dbCaseConfigs.py | 347 +- vectordb_bench/frontend/config/styles.py | 34 +- vectordb_bench/frontend/pages/concurrent.py | 6 +- vectordb_bench/frontend/pages/custom.py | 4 + vectordb_bench/frontend/pages/label_filter.py | 56 + .../frontend/pages/quries_per_dollar.py | 6 +- vectordb_bench/frontend/pages/results.py | 60 + vectordb_bench/frontend/pages/run_test.py | 6 +- vectordb_bench/frontend/pages/streaming.py | 135 + vectordb_bench/frontend/pages/tables.py | 4 + vectordb_bench/frontend/vdb_benchmark.py | 57 +- vectordb_bench/interface.py | 8 +- vectordb_bench/metric.py | 16 +- vectordb_bench/models.py | 42 +- ...result_20250318_standard_elasticcloud.json | 5890 +++++++++++++ .../result_20250509_standard_milvus.json | 6138 ++++++++++++++ .../result_20250224_standard_opensearch.json | 7319 +++++++++++++++++ .../result_20250124_standard_pinecone.json | 2365 ++++++ .../result_20250602_standard_qdrantcloud.json | 3556 ++++++++ .../result_20230727_standard_zillizcloud.json | 791 -- .../result_20230808_standard_zillizcloud.json | 679 -- ..._20240105_standard_202401_zillizcloud.json | 1352 --- .../result_20250613_standard_zillizcloud.json | 6290 ++++++++++++++ vectordb_bench/results/dbPrices.json | 16 +- 79 files changed, 34583 insertions(+), 3546 deletions(-) create mode 100644 fig/homepage/bar-chart.png create mode 100644 fig/homepage/concurrent.png create mode 100644 fig/homepage/custom.png create mode 100644 fig/homepage/label_filter.png create mode 100644 fig/homepage/qp$.png create mode 100644 fig/homepage/run_test.png create mode 100644 fig/homepage/streaming.png create mode 100644 fig/homepage/table.png create mode 100644 fig/run_test_select_case.png create mode 100644 fig/run_test_select_db.png create mode 100644 fig/run_test_submit.png create mode 100644 vectordb_bench/backend/filter.py create mode 100644 vectordb_bench/frontend/components/label_filter/charts.py create mode 100644 vectordb_bench/frontend/components/run_test/inputWidget.py create mode 100644 vectordb_bench/frontend/components/streaming/charts.py create mode 100644 vectordb_bench/frontend/components/streaming/data.py create mode 100644 vectordb_bench/frontend/components/welcome/explainPrams.py create mode 100644 vectordb_bench/frontend/components/welcome/pagestyle.py create mode 100644 vectordb_bench/frontend/components/welcome/welcomePrams.py create mode 100644 vectordb_bench/frontend/pages/label_filter.py create mode 100644 vectordb_bench/frontend/pages/results.py create mode 100644 vectordb_bench/frontend/pages/streaming.py create mode 100644 vectordb_bench/results/ElasticCloud/result_20250318_standard_elasticcloud.json create mode 100644 vectordb_bench/results/Milvus/result_20250509_standard_milvus.json create mode 100644 vectordb_bench/results/OpenSearch/result_20250224_standard_opensearch.json create mode 100644 vectordb_bench/results/Pinecone/result_20250124_standard_pinecone.json create mode 100644 vectordb_bench/results/QdrantCloud/result_20250602_standard_qdrantcloud.json delete mode 100644 vectordb_bench/results/ZillizCloud/result_20230727_standard_zillizcloud.json delete mode 100644 vectordb_bench/results/ZillizCloud/result_20230808_standard_zillizcloud.json delete mode 100644 vectordb_bench/results/ZillizCloud/result_20240105_standard_202401_zillizcloud.json create mode 100644 vectordb_bench/results/ZillizCloud/result_20250613_standard_zillizcloud.json diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index ea346dcd0..e9de2c3b9 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -4,6 +4,7 @@ on: pull_request: branches: - main + - vdbbench_* jobs: build: diff --git a/README.md b/README.md index b3c9e53f3..270ac32cb 100644 --- a/README.md +++ b/README.md @@ -426,52 +426,35 @@ The standard benchmark results displayed here include all 15 cases that we curre All standard benchmark results are generated by a client running on an 8 core, 32 GB host, which is located in the same region as the server being tested. The client host is equipped with an `Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz` processor. Also all the servers for the open-source systems tested in our benchmarks run on hosts with the same type of processor. ### Run Test Page -![image](https://github.com/zilliztech/VectorDBBench/assets/105927039/f3135a29-8f12-4aac-bbb3-f2f55e2a2ff0) -This is the page to run a test: 1. Initially, you select the systems to be tested - multiple selections are allowed. Once selected, corresponding forms will pop up to gather necessary information for using the chosen databases. The db_label is used to differentiate different instances of the same system. We recommend filling in the host size or instance type here (as we do in our standard results). 2. The next step is to select the test cases you want to perform. You can select multiple cases at once, and a form to collect corresponding parameters will appear. 3. Finally, you'll need to provide a task label to distinguish different test results. Using the same label for different tests will result in the previous results being overwritten. Now we can only run one task at the same time. +![image](fig/run_test_select_db.png) +![image](fig/run_test_select_case.png) +![image](fig/run_test_submit.png) + ## Module ### Code Structure ![image](https://github.com/zilliztech/VectorDBBench/assets/105927039/8c06512e-5419-4381-b084-9c93aed59639) ### Client -Our client module is designed with flexibility and extensibility in mind, aiming to integrate APIs from different systems seamlessly. As of now, it supports Milvus, Zilliz Cloud, Elastic Search, Pinecone, Qdrant Cloud, Weaviate Cloud, PgVector, Redis, and Chroma. Stay tuned for more options, as we are consistently working on extending our reach to other systems. +Our client module is designed with flexibility and extensibility in mind, aiming to integrate APIs from different systems seamlessly. As of now, it supports Milvus, Zilliz Cloud, Elastic Search, Pinecone, Qdrant Cloud, Weaviate Cloud, PgVector, Redis, Chroma, etc. Stay tuned for more options, as we are consistently working on extending our reach to other systems. ### Benchmark Cases -We've developed an array of 15 comprehensive benchmark cases to test vector databases' various capabilities, each designed to give you a different piece of the puzzle. These cases are categorized into three main types: +We've developed lots of comprehensive benchmark cases to test vector databases' various capabilities, each designed to give you a different piece of the puzzle. These cases are categorized into four main types: #### Capacity Case - **Large Dim:** Tests the database's loading capacity by inserting large-dimension vectors (GIST 100K vectors, 960 dimensions) until fully loaded. The final number of inserted vectors is reported. - **Small Dim:** Similar to the Large Dim case but uses small-dimension vectors (SIFT 500K vectors, 128 dimensions). #### Search Performance Case - **XLarge Dataset:** Measures search performance with a massive dataset (LAION 100M vectors, 768 dimensions) at varying parallel levels. The results include index building time, recall, latency, and maximum QPS. -- **Large Dataset:** Similar to the XLarge Dataset case, but uses a slightly smaller dataset (10M-768dim, 5M-1536dim). -- **Medium Dataset:** A case using a medium dataset (1M-768dim, 500K-1536dim). +- **Large Dataset:** Similar to the XLarge Dataset case, but uses a slightly smaller dataset (10M-1024dim, 10M-768dim, 5M-1536dim). +- **Medium Dataset:** A case using a medium dataset (1M-1024dim, 1M-768dim, 500K-1536dim). +- **Small Dataset:** For development (100K-768dim, 50K-1536dim). #### Filtering Search Performance Case -- **Large Dataset, Low Filtering Rate:** Evaluates search performance with a large dataset (10M-768dim, 5M-1536dim) under a low filtering rate (1% vectors) at different parallel levels. -- **Medium Dataset, Low Filtering Rate:** This case uses a medium dataset (1M-768dim, 500K-1536dim) with a similar low filtering rate. -- **Large Dataset, High Filtering Rate:** It tests with a large dataset (10M-768dim, 5M-1536dim) but under a high filtering rate (99% vectors). -- **Medium Dataset, High Filtering Rate:** This case uses a medium dataset (1M-768dim, 500K-1536dim) with a high filtering rate. -For a quick reference, here is a table summarizing the key aspects of each case: - -Case No. | Case Type | Dataset Size | Filtering Rate | Results | -|----------|-----------|--------------|----------------|---------| -1 | Capacity Case | SIFT 500K vectors, 128 dimensions | N/A | Number of inserted vectors | -2 | Capacity Case | GIST 100K vectors, 960 dimensions | N/A | Number of inserted vectors | -3 | Search Performance Case | LAION 100M vectors, 768 dimensions | N/A | Index building time, recall, latency, maximum QPS | -4 | Search Performance Case | Cohere 10M vectors, 768 dimensions | N/A | Index building time, recall, latency, maximum QPS | -5 | Search Performance Case | Cohere 1M vectors, 768 dimensions | N/A | Index building time, recall, latency, maximum QPS | -6 | Filtering Search Performance Case | Cohere 10M vectors, 768 dimensions | 1% vectors | Index building time, recall, latency, maximum QPS | -7 | Filtering Search Performance Case | Cohere 1M vectors, 768 dimensions | 1% vectors | Index building time, recall, latency, maximum QPS | -8 | Filtering Search Performance Case | Cohere 10M vectors, 768 dimensions | 99% vectors | Index building time, recall, latency, maximum QPS | -9 | Filtering Search Performance Case | Cohere 1M vectors, 768 dimensions | 99% vectors | Index building time, recall, latency, maximum QPS | -10 | Search Performance Case | OpenAI generated 500K vectors, 1536 dimensions | N/A | Index building time, recall, latency, maximum QPS | -11 | Search Performance Case | OpenAI generated 5M vectors, 1536 dimensions | N/A | Index building time, recall, latency, maximum QPS | -12 | Filtering Search Performance Case | OpenAI generated 500K vectors, 1536 dimensions | 1% vectors | Index building time, recall, latency, maximum QPS | -13 | Filtering Search Performance Case | OpenAI generated 5M vectors, 1536 dimensions | 1% vectors | Index building time, recall, latency, maximum QPS | -14 | Filtering Search Performance Case | OpenAI generated 500K vectors, 1536 dimensions | 99% vectors | Index building time, recall, latency, maximum QPS | -15 | Filtering Search Performance Case | OpenAI generated 5M vectors, 1536 dimensions | 99% vectors | Index building time, recall, latency, maximum QPS | - +- **Int-Filter Cases:** Evaluates search performance with int-based filter expression (e.g. "id >= 2,000"). +- **Label-Filter Cases:** Evaluates search performance with label-based filter expressions (e.g., "color == 'red'"). The test includes randomly generated labels to simulate real-world filtering scenarios. +#### Streaming Cases +- **Insertion-Under-Load Case:** Evaluates search performance while maintaining a constant insertion workload. VectorDBBench applies a steady stream of insert requests at a fixed rate to simulate real-world scenarios where search operations must perform reliably under continuous data ingestion. Each case provides an in-depth examination of a vector database's abilities, providing you a comprehensive view of the database's performance. diff --git a/fig/homepage/bar-chart.png b/fig/homepage/bar-chart.png new file mode 100644 index 0000000000000000000000000000000000000000..a78c67ecab89748fcda863991d43e0cedb54e791 GIT binary patch literal 81190 zcmeFZWmuG7^fro(ARq=R2&j~_NVkfD#L!(MAl<^yVIZiKbc2KpFfc>Mkb;DS!~jDh z9mCKJ4QG%3{_lI8&*wegUOsU>Jj}EAepcM;UiX^Ms>*T{WVB=?BqS8_FQ2QEkX*qpI(_fxLsxvGhgej~3r@5^%k=P%>Pvyez-%iqaM%MK*BYPYvtwY0QgFyVg>f7J*_h&vVS9pF$q z`wMBDTwF^0mcRRFe?<)x7*rrv2VbyAN*axp+hcxwxHM4V2!$j(ejyV?S^KJ$myREdD$Tz~C2kZ8s`T_s<>lpV zk4~Owmz9;t(!Cr02$u7`mC@f&8!?c6{O%G%&)&ui?@LAozde&6+b5ANA$Kw_G5V~( zzk6y*7|?-Q&$Y);+Vj?KM>aM#qVrY$eYlg`(?B;1b93v-I^XtRztkKYkl;BtXlSzA z+Fr6SGh3|;6`l5HhSnc#D#E|L0Uv5Km=@eyG3kf9p{>}5G^*y&ePy4Wk&)36=F68L z2Fuef(4TXn@Y(#uHpcbu8S9TyXmuGy-#kzAB_#Cq^(DueTUgMd;&_bmK3`|EIXT)V zBO_bxu2)OuHsS*%Ue zDkv+vStapXhMhfkPP`y(f%D`L+D)jC`AT^g@nzm6kN{9DL^tNyFw zxj3lJK%SOEfo++7neU;S@9FXG0p8IcQCV5}dwK9eh4-!lJfar-FRu8+FrT=R0fdft zS$DE%iQ>VGwcp;?C5+_bLT$5n2S#1RzF2MpS6o4b zn_k+#2mfj7yCZH?=TkLmQdYaU`9|{eXiGswWi7-3_8F2SBzkoB=FRT`0RcR??&ds| zG^v9wN59OerXYktg@b~UQooV=IP58#(e`UmQBnH|R$<`^je|bDXV0DqnEoUUDz_gk z7lj1;^PRUy%oJ08ID=#!w?TRGE7oYk0)fJbN z%+1YxudEdL7#Ns{)PX`#U^(DYE;vr&tzO%Uy~yqvAJ=Wc-sEG^0whw~#wO2BC<&Oc zVQNxr@|tHyM~5xO`Acx{=$J4+I1!?6n45nG+q_9l-Me~Ox1+s1IwB%{WJI61Bq5kWC&z;3?Kf(VHao;z z=0h+CNkaC0V^uEn@5^^5yj3RsPir?ynoWCydFygAG9G~UsKln6?38nh{@S$+YfSbM za1qC%Nx87Qy+MqAIVG0eV%))>Klgr*GtR@I>g&3mj%>`d()&@8k*WLm)GVR9CALDE zJ`Mc*`CRg3Z-#?|qt7zUHygZlL`}_@E(1M%L9Pbe2KctRneHj@k7dTwy&xM~TY7?| zN#pr(mu|^T0>PoE-qX`lzrrEKZFOv^lq$G6!A;p5pxR**lBNPxML*I zUzWdcBAsm3@@+Fj{r&yv#od$?<9J}Sb9X^FN^-2zn`sHPg-I{COb9uQ<)nza=jG>j zx|VJe4?#t0|L30PNU1f)!-uNi*&PiBlc&d%^w0J{3{-6jq;Sz~wJ5P}4WmD3l|C^M zEB0>tdOuOvG4)`3$;>Nceln)c_i(vaj#10gvr203&pG^L{iMk;Tbhsh@!@V9ZMS=x zlna&&yW^YgmOVo@uD3)wJu=dgoqYl_>C z8mu^Zc-+W-PWm0%(@S{tY*zw%(y#R@Q&(5t?vbJTg~eh`s=6#N)6K!KKMiN%*4V|x z4Tgt@59X5`OJDss3rQ6CHHYrnO^J=2@0sU#8PLC|LbaccE z{0S?1zDFCaWTd1$@qA|6;596Pd$JEF%53BMFqoZfj7s%gmRgquWp2IVpGz)QrXU1@ zE#QZWWXNyv^n=WkGMI}(xgYqbYiPh$hWKhX+c}jFSAoxHNKg9i8ewJ-8;QDAZ@#s ztEoktY)PMD1Uwx5qO2O;o}p~z<>m(O$}yYDXcu>bqi`4SuA|p}>2-^G+>c z#5Vh|oN8UzabnDL(S{ei#M52>(}h8_{Xhct=RPpbsB(vK8+50}?<8?zHV2ki)C`ss z$)e~w+ZLgXMdauAXS@hjYFhsG?H+tn)MBy;16QdY_@6BzDarGVAPuqc@j0J9eWIy= z-PBErF%yA$6IPsa;=tz1Oqv3P@G}u|49?z0C8|j7cQAuWj;6k`IH=!Je=La5H#xOS z_v4|!R}uHCm~Y;=Arf=TIx;@~p=M*xf^L!NM@$#F zim2nN*H)MCsD#>#_*%`(U}$u7^bHIlAmBW_+_=_Dw2Xy~ts)ot?`JoaQ!A67A4Gm$GJ}IFwyGQCu*jCxCh;w6U_8WWq z0>auP3IG^dd%UD0h%iy%;o)Dtd~pSC13zYFW)_LY+!nC2$B#>4;;X8zuBFPU61nCbv7+buL4Xhg*(8bWg_4qzro8;; zi#lO z7oz32F6&;-Kl9hNBxHg@3*EHfBjaj)7O;5l?M3H_<2%5b39I9{eq&wT901(FJIsaF z`(>L80T7TW=}maZ!J&YK`q#FE(c^BEmbY*f3X#tln3ntPu8yaSB+7=2xWHCd9Jyu} z7Lt%I`*U&PWudf!3Y-DsQuv5WWHyE_o2vv+=+ptPqDPvRzifhhg2ZHZE z+}iLFYw|e>(Z{%YJ`vgYz`gx*<#e7>fUo~*)j}$boY_3UC@Jm}y=(rb-ZyDz%9r*a zOjY9gTM_BV%Ijg(2X-rRI>MJJDC{v(D}n=gQj>TCbuFy{Bt8k(X`8{_+1^^|Qu4!y z-giR3Z(?gUPcyg{AbREMcmD$7TXJ~QwbVa{l`;1m-!uq@h!UnzBXZp8qWA=(r|?&#d)zimR{p zuP%2)@}oYNKcb7UJ3Zbk-5QJ}5C+DJpV^y8>l^XZq5eI)a$PK+`4er2ssJNI&(JX6 ztToJf{m&1ciqS%o57ypFN}Zm&&2`!?TT&x9km`*+rY_(|D^$|_PpUf9$p;1oh^cF} za)z=Z%u(oBd2w-Z6D#QoDe~Fy^t36QdH^Z=tDI&f)sZnHGjn{TP7?cxGX(}IePL~# ztC8eO;i@9xh>Q;%(wXUEa!00}Id{?3BQ0_Asz};pwSVT;vjLED-SijEQj0EHIHAlI zi@uV-wYdqhA~nW*LA=Nagj?lg5fgN0etAh;TpY2uq`sP-&0e z)QyXY@zi{U&s*ICRvsN6zu4opw>~9=Z!$uj?Pz82&L21N@OEtxoI-+vJwB|koXSsp zv!;sUy9sorN^*4X@UX+QNF=nB0aP^%KI=b(cFLIgu~_k)kJN+zjx#k~Cb-VncV{5` z=^y~wWv_o-gQI@~qxG8lcuQ$l(4zBpg{rbL ztg8)~XmzS575!4ez(AC$c9<*y+roZ!xEiE2t>yIREDDfLcE0>{6 z6(hmT-Bo%VsmSAOGg|GA>Y!7!VR_^jzOrHm7g9k-od3HiCq7&c_29dA@6Z78m3eMi zz|BW19CbihBpXWm{nIDX6bVoDYB&62idYsRzK7xBL1hV0ch-DoLMGq}=%svg#obp0 zjNGSxer~Z%>dJl*Ov4V)pSasfPrhD>0Ep8svPtIHO&A_4Rh3_FD23Q6#&QkLZ8X~0K9d}P_S{SAV+51scY;3k=s;6v;MVC52 zg&{S)5x@QPD;8EN0@Sj$t~0QdPd4psob?r??pg-~;jK zW)^mESOBJn_dG$YBUWM7OYd3FU7~aqmk16IcOQupzpiR*RWad(0G6o*CnP7|tvx+C z;NU?c761)0B-z@b|26|}sOvc)1}n7dNMb%-&ZnVpB{x9e$4Obq4P9I!i0VpR*+ z2uulcliRWh+fV^D8;4lmaLgr_W$}v+&vkEt@Jdv$l(ARq!(v?Q6B@?F=BB{ zKh7<=-f(Vtupo(wXaekcSVc4(qDrzeGmnmyx)qwfM)$22f)Me?&MnkM8tyq>R45QgZJjF{0vN0 z^((7Ve9y8FTsb9@cCslmTxw*#cQFlQEP2C`s^#4aUu3x2R&_xveY+yWD zzqA8@CS(+F9T4CEuY=-9DrdsP?=GJP{=;%s8Xj&Kh;UW1NMFo6%$t*wGbS~4BHP!H zbZem-3wF9dIN-PFTwMQr(Xsy6t)inrAQyqa70y6&Y#UQiC=>=lQ36Z#T4sl>O^myG z%B55Qi{g-ys%yrt$fYUvQ3#LqEbrTp)M3T>3-U2E`Xs{0I`aBZRd2r041mn;Nn8favRuyH9#Bs zM8yAcXmQbM+;bsCIXwfMArVe9wT#lV^|!lA_*%M9vV8Z)h!KT+MsnuEH_WnBA|}Cl7v;jF}do`uevvPg4B{SB8H|B z`*)nb(}-lX(Wi4MlHS_>M>C9&4$f_`5l~w?A;RbCK@0`>{gVvfo5r`}>W^Lk@T(v% zucn~z1t4sIIkC&1)Vu+Ks{ny$85^h8)Yjg-b0;?^=i!A*mnx==#JuMdEbrQ!lY8-E z39!qKSSfJ>kW4`hucfMbkC&Gh5L+C;|K?_WQqpT`42dASqa$j-M$5`7XL@Fa*NRaL zm-t8n0CnImbMx~YA|kq=Gt6@q2nyK|n6OU9PBZVUhjD#RV*tC%D;E>HJ9QXD)-yWdv$@eX6OY8$=`wC#vQF zFE2`2l;PhIjm*ElFldwf^S4&YAAq<+@9@ih{`?sbY8t>lBcq}O9mYgJK2YR2YYU1o zV)1+{>vX{kP&~xfDKiTEEJ$1NDaoQPCeC)y%= z0Z5LCh?pwPFD_OGFWZGf?$68&W}db-^)*OF%I|hSwUQ24qRNhg^M)aVNF!W7F|F_{ zk^lRsPFQl;+P!T#;YCkl7Cx zH>cb8=cs{VZ@F0NAA)HZzCZ!ok)JSmf}b2qU0eZJ8~s*GQLGsddnq=jLTMq?PZowj z9jbAB1k%;_j)Q-d1&PZoiNaWB5Z*wYt8%&nJ^c=7wbDulP$mKniC1@0>_dHh{rv9~ zPWk2D%%J)1G*prUIu}uW!B8L@)BqpJZ_zU4!Ics+044Chn+8?iqIl4zCBsCLb!^P= zI~>k=@7@!Tpoj!0-&6)OF=2zEiE`8pHwXb~arwP*1q3(=V5J0JHM~Wxk~ygXl2ct* zHy$us)o9NDj=hEHf3;mm&{@o+|1Oh|bW#6TE0K^$oP8(w-{l6T|0$vV@8$n{Ccr5E zUk34BMaLanD_wn}ptJkeVtAzD*xX=LJW}TY*giV%Q)=J^3o1iY|Mjw4p=_wqtGe!I z6yE=5UjK(o?e97k?FI$}0i_%Xc%1hi&e6xl#AxM!KF66eC(DCfsEz}xY$Vdy+2}Z8|e$j1FvUL0%5uF@4 z)#EpMG)()r5*oMjFT{OgIX}4m@Rwy@*<8}vm_A^f<=n?b0q^E6>|0_G84VLs2|t}r z6{5##L)X?_ymuEAHWOU!$IEr9JU`~;=S84U71yRS1)ZKShSG^Q&betL%4EX)?+@7- zFVxyYAO*j(UO!}E(fIM>9SHI2rfZdy_{nC|-BcM$69Bb9B(oK8da|Z0ZUI_3$|=ek zHu~Amyj^vCe4?jkG7QFf>c72gRY_K}w@DVafe&GYuPNe`IAK4jr&wmN{Wh@-AR95X=!PItyKV)w%Zs2$%ZG+fQtq+rD&xs zEiBqCN~#9J0TSBZGaMpKXrN_a5foI+UD9(Xz|FyUl7VGlJ@9zg$vbOHJSsa|d(F2{ zT~Duy5t>5`(b_Ikkx20Ed1=b zqn(Z*ns~ok+Q^X{M#{A4u6d8^a06lKala0`9XzJPI!SoqM7NCl{2)zmMe4r4y*I}_ zc2P*0Px;drEiKv63ULA;tukzpsacn2cAHuNZ@_6;u#rlNwuj~>E&z^+@Qr!8oDSb$SP+4 z-gA7&`cSBR>XwtAU&MsoX4R+Ql-cn*RJ*Fn(PcG`yB6=?S>YPjE>4hP@A+o zoJmghsReGQoCY;Asx^kzjFBSFojbaD?HaRuUQT|Wt-nF`=B9$#@$xb)x7Umey@acw zP!hmGG=P%gXqcSTJvk{Bj8_G*;zgE3QdE$D9BJpd~0M=zs?e4#?>y?`cLz z$dwBhFD|2TNPYsbJAHh#AgyC!l8$i$j6%Wcp-mp1SDrV!)sbjC{`9fx$=Fb{RjRB_JId=2KEPI7_K?+(R!#UWk zNe9?M=yLV=QD2$yqyEor>qaub8QTw|XEL~ufM!;A!nA?QK~e=An#fROdC|- z2j&aBUzh0ZG%rKY3rITpgCPN0@0RV)w8AY_+sXzE894<77`8MN&{)JYPZxN)z-@ds|x%=-};CPagj^sh*w= z{PC({+%1X0dxa00WV~Qd=j=c$ziiY zOD%4}tM!(t8oJ!_8m!@i(Z4dgt5McVdSafBwR(H0(XU@I?jI!jmvM3;H1+j)*q_e<@4g<41yOecgs~=9jHaGV?Nhu+~@lHW^u2N`N0FE2LY( zA;-qf>eX<6vAPmOP8JNC{&`0W+n=Xmn(WmqCtkzL-`xCN%x^m6OR(Zi`f-iVwPuhG ztNy{k+`_`4^fZvQfV2R)w`XPa_>r2q`Au2wjl4wKx!GC8NUv>DzS@(-=-609aj}Bm z_|XlUH+F)t8xJ{bDqZM6giTuLiUt*ROV}M`{96 zN`+9ShNMOtd0I)E96u3PLlE7?JQgddb;EBRS0Ah3k(Ks^I=C z@&^uUCYksVTUNTKZgP56Hkp*i9tzrs8Ka`MPmjjEcSFg@-dwS8UpX?KTpi<;KfubZ zxsW-yukAm@39o(y3tt~sPB*uVaFtL)jetG%%@TDd)0ZiV8B&mi(6 zXx}s$2n%8$uW>rCnA-Q+4E>=#6eT=)qWg>ej;t>IjT^5*zIC;=_nxeUqsPOy3L=c4 zj<#bHdCiUmj~bV=T{2tLdL8Swa=$43?rqni18EzOvqeP_vPScal#Ez8M*q1&Kg>Qm zH#^2)&CeoWKuT)jOL#D`Z;25cc1`i}t~la6-RuYyV8}FX4P#3}Vq76|J6GIs@`2ff z<0qd#8CJt@1YTKMD1~U~QOmb?aK6!BoqnlqR&~$Yd~N}$Q$mfiS2H%YHawMK-~Wp! zt$v_kxDf1zOBKe`r9gt>VPtH)S}MWJ{PUsc8)C=8zNxYCcXd4=$cb`0<|QD6f!+BD zB@HPIY&9rx3#M*|)W1pga&?3*_A|VDcgAb}^tdti7=XLmQNOt>cMQK7>kU6*PYMsO z(6}5$&^_I8W10;KA;okvRyLFR)=yMwjJngSdcOF6axgFFbtJs&+Qv6|a%kCNHQ?v# zwG^9#;+xRG@O5MNisEcW3BK*t$qj42N0A!9@TKiRKdCw1?T(JZs$weL zQMbq4I_%5@c2*^&9|Dx9ffPA?%(;LoR=i~UyG&zl2D6TXHM5=e^v=i^59g1p?@$Jf z;F2m_ zp6sKfxwU~M9d>rQ!BpEuR} z`0;Prcg7x8?vT2S9p0gj<+i&|neyva`hHL{y-Ttg4-6tWRrwm|6=>s6km>!ODk{Pm z4)*O^+HB+#op+v_r*&bLW(B}Y1zW%$R=O0SH^ zC2v4k=xltLCS_j(SkRsEx+Wrso34=7e)YDVg99-;XWnbSWhox;_D+H&;0A5C7gj$C zizR>9nBlQ6@_BBVAUHz?xh>t7jxRT`z%`$)K?XoQk~5xT4!qmZT`1rPcqOGt8FN>a zTOLRn&p?xtm`sesZNFePOwE{Jt@z(_kl#8L4t~ow84I>UN)(iTMGx4(9+DRWIH*4J z$=;o{!SFELz6JOB`tOuD)9Q$Mq^z~Kw|6d5nZuc>aH4MJ?S@g3$nT9J_QS|~O9_^t z*ze!x#NKep%Y~=)HePV)uTR^5jUu$S|HgAl*T3{XeH9}kn))Fq%yuT$NZ$!pwmb@t z3$7&wSCW%El7}lL`qgd)GNZ(r2K41C1=n=!?DBz}6cqO9p#NkA8U|#fR(nZRIXN%u zkLGw|qN2=>#(w&}Cb&=fTGmLpt;C1Xi)RDX#u5=53R)_6?Eh#9rDKtgVpDN@XxVr3 z;pMa8Ko(GXG~)u@zEEL={eH}+p{bd%Lq|t8eMt~8XA2$4u-eqO zsbAZ7T|a*Mq>3S!$k4XS4cTcky``ubN2RD;Jq`;e>_k!4%I1s;q$4PBiMrHeVBJymM;)K~B zKJ1$>pg6tfzQ&NR7Dwf*n;L%`-az6Hh-;gZngp-S&zH8_P&Bk_7>knS4|9l=<*+37 z3QpeQ<&G|Z9dFXn!Oagq)v0Y;yGi~>|q1CkGTo9|O)WrC)LwpaSNI zBlzYak!tet2`5J*hhE!Q_dA)+`0l4^Lfi0yIuPH3lvrg#LRW{g=^m=NVI7S9A^_A? z^E;`kPDfxJ21j7O)4e@CF_}^<*@yNx_l&%}k-4n&t%WYlwE$fT`aWnHYJ?e4Hj&Qt z$yf9%U#48CM=}861^anZE>vV>Jrr2zT%iQBNSlWd6zb@8E-P2aL-i|8Y`Clp=Cykq z10ww9X7S=gwSy~JLuTfKZATS}KRq;8|DcAkO4?3N-vHcY5i~|{Lte7WYgdac0EjSN z;)HCveltmEZLpmFP}9HwPA^`M?PoXBAN%j^w(@sNWXdu6Y69WaNRe64$;ocL?Z$NT z0ci8=09C8)pN4k@t1~Thj+&s|ak3d#|B#7EhS6^?otgd7BUNJCM(Xs)h1go8EXzAS zI@%s{=@!M?ZqEK9>bM+giqe%9iRBt-E?{Ox!aIeM);FG+1qe#08~dfZt)3X$+^zEI z3XwM&XJIoa_k+KGAK2sESQ!DRxHtN(*oym~5czp;Zaa@N5TY2|));2rNHV4cqTFHM zz1=tUBa=o|IY$6v(j%l~sH3lsjEsB?Kiwn`)-o7XjXT103+Dg^2w*|4J?N??I3!j` zjiHLv## z=-`BeYd|LpkqEUmdoAJ{N(RqjdQzdwg?{~{iJ1~&{`9pD^Ghu06pVljTY75T5R}ap z-KpI2BSaa(4v^_{$rG#Y_a9Og645=kO(?>*r{oCVpCIvt4am&dc^WHR#bH(pZ9DSp zPG$VdOr(qKydKN;V@B$68p0WSK)wR<5Q4QOlRZ8+Dm`q5;RDzJM3cK`hhneX8B;Ma z8HdDyuHO1ydqRKU7XNEi>7d8gfwoI{+@p+)>`U~7ZV5U!`;Dus3Sip+VW4GiZ$!sd zRZ>-?{2_s#)p|^KL4U}edqe%tBaXWC&YIb>BPAK$pgAM zZvjNr(zTM}_}gf6a=tC*nls;(1SjSo(53?FDAiZ5zR?*dOifKGXQl&bL8v_Sr7O4p zxJjFu(G}-_re>w#Lc_a2(Q*kB*D})lsn2dJ65ozm-a5{Hdb&KI+lti~vJ+&Em)r^4VP!Jw~Xp{i9%neDsvkm>?ded*4EEQwq|N;S>` zimS@6dG_PS>TyOXsw+7%ac=ND%!X^BrwvCO(a@T?#g4)9Kida5`nD;Wqt`G<6l-zy0B++#C5B7Bb{1Eh1Nfz<1;V5X@ik6 z#pz(h-zwQkE7Gf3T)Jv~1C9eFa?_{9_#D#lsy_$XlTn;nq2|hlkP|E3be#}2n*~t0 zVY>5|+~4^=q#^=`N|E799nHP-S% zKpw=IS={XCmlYXV++xq$+ey>DcmDhyXI)Xi?+7z^ zO8eZ3bdM8OSVTmXk1y=HgWpNBtl4I3A;bNH;o~i%iAQ;WvAfiv%~}7g&2^uUaw&v% z8|aBAZUe5Wr)cI^@$f(a0jRQ-G@C3v)_X3`SSUulZ!~2pWbBddo%@epaTzMNPnz(U zp}@eMSgaR*r$hqv@k6t$657)+;ta-8KnR#FQnm-;BK;bVB4+F5{@mq>W5c=Z(T9SAgADvO@$Ky#$tSjCe?Ww!8FMQrIu4gpu^tYB zF54;BfngIsZEoWU$6m1BH*>dWaF`esWGIF5fq`z5JEG5r9&c}o+O=w`lIW+-8_iP-{WwmH4KmfahngiABNR4^8cERx2eP3I|kspa~@I3&pc_Bnxc3$<3A8FCNH4 zp@0AblUO!+TSYg7Zbo7T1J2l@5`X07sGhKYVyuqw0wB}qu0nv9_x0xz+Fw0rH@tcj(x6+MpBX^^^0x%mb z?%9A^WaKh3&lg^dlv;pI{3QXTDL_}s>tG4K1JKGMU~+qb4n33votLNJe>gf>YSMI3 zz7Gi2LhXTYv+ow(*mtV{_kKNcdt0O4>7k2>#go$s*|wv46jv{b0S%d&qJTC4vEAJ4 z&76#4h11=*dfaEZReNVBr<9Rp4G$`(pLifkUb~YnQG6U#?$7ca>YTHspk->H^}(hB zU(EV--VuP9t^`S~GiNB6it!y(`NoH#%eaCTt0ho00&z=Ox!s}i+|3v-JgVb$kw=G( zq`Mq%o@M1~{Vr>e*>l-I&IkEHMD(Pm^&{3PgPR?l1O8}&${aB6SX!)8(_}5oj1O2z}yUK`&j+<0_uhppq1A0kR zbRQ6R{inxsYhZSXj};e{1({qKe0xH1om4sHu{I+Y6yOlFFRI)@+TC6-blX8}Vjt8e zWzHQu_G85zOIwwMM0VxOsMb*W^yNfa`;lS;;#Owx;d3)H+>VZpK}nacw}#s}Gy7XE z-l5L^`t@E+Y%JZH*^$VCSTE?VJZ81iH1pkUcJ#MHcc1uPcg3bq2$lv>E}=k%cjaJ* z00!&;C0)xdI4B=S?UPo0cnI1Qdt(c{XYqI@#DLA+RsaDd$rr?;TSd2Ls@Uj@@eU7Osy{>NhSvGy5a>lRj*cGG zZc96|KWc!+XbHAREf@=T;F!ePEWHe>o?oHovz=S0oi%`FA z$3flUD!(5R2(E3^Zr!+Hg+&^n0r3a`DXKa679u0#S^Y8IF?91jfeD%By(+r1;&nkM$Je&E4jqLQk_G57;FROl znp`myI@K_TL~=^Xex`b~;<-b%cGC@^_F&L}Aa+s#M=p8PA)dk-SE|e9wSKZs>mv^} zyY@4?kXSYJ#!(G)uhtHypK73-kQ+BZZ&%K*OtqGFcMU(jN%1w-kZFA<1DXgl-oVHn!WNZ~ z0g8Xv$X`k4dUrEW_7Mljs@LieOBgtvU`A5oGEqF0CJhBM9)U8sp+Nag93*V%hN}oi z*KQLCgJ3Z48yIAahl63DMpDEjDu#R@8MX$L3#Bh96&1FFw?3aZdqOMeRa#`Ep(_d0 zr9_<=vHb7YHDR>#R(ORsFX8REd~{t=v0K~31dPcn+J~4r^l2j8dRTl%B-u@EK@-$f zhzQ{ zGefm`T~_|;-!%aoO+uv30PRMs@qsdyEnnzIR`7p+bUI@YNRE3oA&AFJrr{F-=Jqodxu5IFdBXRzk5hj zul3{PwBqR*)8fgE!&h4t|6AE99zc!+dqz&ynyboQgksIbm!x z4%9wxgU=Cn^r=*$1_aRT=WW#hc&cBq);XIesfKo8{6#6|Rl~)4JEuHAV%XzfT zEwZ3lZHVv%qvi_TZG%f$uJY=Jc3B5IkjigNBkpUWvsbk*UAT}_Tr3NAUiZqkL7ECo zK!Ec|5sTpfT6W?mIHv8ng%9%buxpvWhyf>e<1U!#12cBd0Eg{>?~rbQrb|8>8DJ!7 zP|Vgx?M5@l>m`|7kBqZ0g;S+Ux9FUU(fwx2zJu{&!;;5`Y!gN3f&q(mxJ@f z?tyDHM&4SV(5tUk5H<>qd3i9&()!6M`Ssh3dncbZ4gg2;z2`Ko_Atz0xt(z7M61ZI z!)P^DJbuDp@v0Z^{dDFh&#pa=x)N0*&+4e)$GJ7($Xfp^3vxf;s#tn}vCLxj2gu?p zx)ARQyWwKHVeP7eYd2-dIwFd_m$EwpnE738tRPzkU;y@<9MfY9Xij`lpHw^0&R{PzU znq<}olU$5!$nOVyg(v-Z)c`csPGAZf#IBpPp%=k4G@}!36c|dH;}zd z7E&{ndHvU0F_jkA*LQYzFOUG;+54F*B3{#H0e{z9BF8r2l|Z#w2s;d4*==i6eOX)c zHO1h3fp$>ADAi2uWX=1xtzR$XsbkNQfPD2(7c$m2l+!#C>OB<7{O-x8Z})?_dWE54N3<2q$o0lFqknTW`K5l`sr=YynsM>7`qatly z-!b>*{Q2|uxVeetmu~Uv%an{#SD?^IIU5<gPL|UzKSI z!R%2C1%u_!r+!CKGVXj(IY7&c5TZz@qDi5c+HWi-Ix!FEK|^jAeGSz z`Q6Y*k}>zLl5U(-2kU+Q@Z}3fzHS)>_;Y3}@2l0|D264EWRpknVSLYNhUO0+bY~wX zcyj4V+Brc(7z97Xl!#7Efw z0r@>yx5f5)?#7dcH%#WY;k)zfz-4xSwzOoVr=KBl0^fEbH==E9%s{TaiyyrOCWEL! zN97C&0PK61w37gU_P5>-td^$>jzWN`UhShDfi)a7gu{n%cQoWiFCfl*vyFS+nIK46 zO<*`ja#tcc#66CCnmbeV;r$zzsqH9Z*%6wR{?fNX_2~g;dzG7N;X)9-T5f-(yYwxN z7}}|+`=5eCUE`bKm?br>@9!%rZUN&NLNDDJw(4I7x61|nyj{I>gN(&hOp@fA<*u?D zo|^1qyqM<=K%foSi3fk__!?z!-aeD^ji<=8Dbv*4-0L0MadiBpG6_v{(VNH1Pgq%F}bPc#i^$z&>$BeV-S|bRGH=OiXjS(8g%uGd9$VX+f7Lf*RBQhsNMlv{?f&Am7&G??&fB21lVJQ>Zka4(!GOq+k^Jk z@3;ql%!^Q#mC@b~&EBneiZCzx$$i`wYwXzYA>Zf5&A4gtpcx&cG+FWWL34GLB(=JYpr zcg=;L7Z;O%je~~#46ypy`peVY_R7U`ef-60#>jxNqdR@-2Fb@!(1FZGTO># zNM3CQOi|jt0^!iJ{CJAW{}oF5XkCVxjjg)2Acj-tEXnJ~cTRRt{y$n^8mb=)S-{VIl&Y;3sIuF2px?k0x#S=aaxn^BXq~94G)Z`0Y z-AD7aBA4mvCM_%K!JC=zv8P-Ej8s2uKi9*D&eK;m1chvh+>RX{av(e~7tY?#R=WG{ z%yIQr_9WVuu~pgl)lQUsv#gxyR0jrsmLDgy5?GAB$P~CEXZ7+bfnLd?ETW^GlW*<& zueViFTk{2%(({o;qi*$5r-RS8fG~bAmJ}bhN0b>= zr~W30Iv+iX8-y*yr`)_w|2cl~Wzd$_&Pqt*^y7vj1`)DI(L8qsI{GsopIxpsb`)M~ z>H2?Yd+V^Oy02|?0}|38f;6HcBHdjIC;|d1-6;*y-2&1|N(dqd(%p@Mq)0c2ba%@) zw$Jl>-}k$|bDeYkIA>p8)@HA@_FQw#ImUh8;~uzF$2$(8*sFzmaJgJy4X{wv;8oRz@l zFd;?1I_@>zW@u57-ny7keH?T9#W#ymqU+?Z4ezr4dbP?jeKvRN;9Js@MSkYOU#q6R zr6c7+7az;}7VQ1qcm&uuM6naqPhMIM)|B-Om$*l+WSJQmWlsq(1Q1GU_5b+cbM3AI z0)YgDb&B5q-5cU(`=ui+9*`g9p%LJ(&t!aEz$7Tc7j)*^KykNXzkz~Wm|s{p70nh$ zM)CznK70K^;dA?{W35uY@`RQ1jPE;UHI`@>I!;cKX`QUu-F%^=BG09#bqfN$B;8QH z_3e#~x87MP9t9JLY_eca;kBjtTgDSC%&e`O_3hLc*Qoo~@YKrPgm91{CLKLJi9sRl9u}?XFsp2K|m6`4>c#u9_cO&^p(BTlK~?2MoQsbr>1Z{%XvAkBqS6D8s1^ zX=P2umaOhgBvX^q(5x%xYdA+%^Ndb2yH=T4OD5lYnP4YI3G2voU9#{qLiIn2Dju?{ zEbC$pZ_w{#EX?KUYCO74v%MeF)c$z^S2?5&4Hf0mP@0ZV5`Nw0lpk5ySF46^rV{$l z0BWe=e<5h-sQTdFqq$M~gJtA>O%|!e@Rq)WQnA_<*O-IFfu}NkbIP4~q1$hEJI5g)u(orZF#bYi zRIRbUGKBu@Ze)O(rQkkwZ3Nwg4sVz=e*06cwkP&^{^SfF6u$OMWSur&oH@0Z-5Y*I z6@>qxd1%eqj*W}#dh83VtN6b*+TW)0yC~WSik1^~y6=A0kfPm6k^_Tvy zWBpO*?c0c11r3IuoF)C0&SYUU0!3r(T_aZG{;*M^`Vy9!hDr6&Wc|DcRwouCThgw9 z#4=B{#AigcZxUj$7kLvT3pk@+wR)+} zZU|T+K%U*|2a-v=zc-GRfc7UrhBUs98Z$_joBsHJ$HKd6Uh+G*GM-E)Q7qP2+j`oK zP+b6hmLIKP=23f^uSk+GX1>HTn=e!Ggm*F9?K(PHo2JSzf2=q_y|`2%P0D&_lkh5s z*q5YAPJ}%nKlAf%Nqi2EbcH)uePDlTY?uxk8ZY;`76tl8bo z4Gs0R5Vb9#*l#<|FM2XEbH-|ndT+_@IJ&=<}688D?6`Q(K zRLP#Ddu)w2WFk}A&#Rvy_S{dV##3OUih3kmWVB=XFdzuAIJ@?RM<{Jz#cp|ab9c3Ywcb!R4bxDEH?0mpc@ZZmDC7Cw4OW}LD{=CUD0fgfFSC&=2mB) zGu7r+VB$PG>>?Ff&y8*n|N4lqN37`AyE}d>`<|h0!rD%5dvL1lak7OsTIMPigI_`v z`lM4m&8+Q*7qd9J{kQqLc!uH&<0X=k74JV=l|_2f&viUs<_uEa3c^ElC-yA7kqv4` zw^v*{>GyY$$bS-VQ!>9ge22)e9oJ+@<2LbJp!z(!fs1$01P=Cgt9uveuIv@LB*FLf zD$f~{ET32u^~{oDUpZbCqb!E(qssM+2A&bSRR`=p^#(YsIt|U5TS^fpO0jWFY64N8 z@Hw3B$UxB&$4K&q8T)dv#TJLh=~oJ>CF@u-hI&}3FDy@EpQ#B61ho)&b@@U;X zmkK;xshqEq#HCH1PeUbai5^dPb3dP@?9ybN=ggd=p}KQ(T~3`|`34U&$(!v(3ws#Z zMT%LQdVzTArA(4F_iEc_Zc;sTA3b%tZxOsh+CHfVcB2Z51zSoNkA z2r8#M5X)n&4V$mj5_qoF6(|_;0Y_zheBoXHMzp@BsEG&h`}cMI6FyCs1sufeD$m;d z#jV#zt*u5S;-uTp7^y0+Ev~QbhgrtNYCchYt|I8ArOQmJKiGldG_W;U->1`;EL!p1 zQ{ZhBp`ogU0DDtptax^goyWD0<%7c-Yhk)o9@LJG2T|y!8M+^=m-|_+qWb=b3H08j zfAgestR(VPcckAuzvZ7fLOuL%3W>r$t;y@BeKZ&EUSVcF@C=kBs&SJ4-IH6Cogqq(5*$SJ+cm;)?jr6HsG>-=Af=_C)-V{!nuN!YlOtazxCvE zg`DNKjKzIEw2n_|*Qi+-2r<$A#`o5>sdX9~{uvn>Qc}y>(we`Qo14GkGT7$_Pc(9) z5srtI>}{V83IevlkGeAjk!2#P+VxIk?55jy9mCx7&LBM5wfs`RI~CR2-Q5Njo?%5r z?94iKH$dogQ#yj%b4kd45jT!a_fLSP?B8sQciieR;l%sSuU{9i2rL?yAK4Z21TU|x zjW^w{$Rq(=o``TZs8iNr@mF)}cYcrOMwL3+DfiZjY7&WW6TBNYy32cA?db5Z1y%xN zEUBeX9cWKhSc2OSdXI0G<)CFdJw@$#!LU+Lwb%l=Jz2K zj?{0Teim0!M~~o|tt}TSK8_{NQ?Rp?yN%OrxC_>8MZ)3L8#F=DG_`Mh+)r^h`< zIS08Xt}jxC59=r^r9af8jP)YK4qW%NeaLiRREUSMM)x*ySKY@1c}oL@mVTzJX?z|q zyf`}3<9-Fvmy6?^T&?n5E0+0&mG8+ecTaC1pGn+_#&;k!Z1euJM|*sd*x=lq#7BTs zE&7A;t3J5mA@PcjAD3;APq1Vv%LcEW6{XL8CRUjl(H_&D>tk2)wDE3_#bFi4ye^{t z)JuHBQdSfFQJVqBYl5|)^arQ2%R4xEq+5yhs%lL&z317Y$sHc$R*UJapY4WBws~lC z_~=_6%x-La#Yl9BHTHQjQG#nFp_K06k|&sdhQ@U|j!QTCS(U7olLnGxVUar!Iq!1z z(T;FT$cmpKek(Xi7U!VC4!PykBYe`xyO3I8}x9 zC6V{>tbFiSdaOXnwvJW^RNt)e4G@u022R%tKFG*ve=UEehA6invoNX9&>k^r99VH+ z!N_B|P-P?$z>P!oY3%9AjVqASqzFG44-S4=jo+YSgGX4IEcM65|6{N1QEA$p&3zo_ z-b|s9gVU{}lD^f)^Fb1%B(vwf<$ga)u$>J15|ij*9D}(~(d3^7%4RC(sTHFt-}7cp zf&%*4A2$cyIe|MUNf9y|=-~Q1g^_1PE5gP*vSOv66z!O(@VG!E&MK*#oYIhKPd{ny z;wR2EsLA|gtBr_qDYG)kS7dDa9dZlIBz@0Yh?fsAvY(rSs-?M4=FO|;5Z)S$P2jwr ziijNCU;dLbaF1CH!fHRWXELQ4-@e{iu*S^0IRB)WGx+0WG67SK;dxd14ep}&sePI6 zwh-^Ck=|<;V3i!;;2?D#piOIL*gLnQE|pr1o#1zG=0>CrPT;ULu=mfsa_-qm|L}o; z-cnW$(F7rvlJ1Ppz*2#DAs-!IAr`ezLP2Hx($m(Z%SnX4M5HI?Q-jNNBb}_RycI+B zt5^yCF%l9H{s#o{CAMupy#;y^_0zNLO;(zmErp5AGPEX|1lpNjq~sj-y4+oY(}rF} zeMjBtJkRN2olqGYifl>gLp7})XPXp=FVGp8sJQGfcXE~sGux9;r~cl{s#kdJ*FvP{ zC2}&$b0)5Q=LnZZ3u*gv_Po(+_@^$E(ngkB!JAk&BOlb_sbRw4ji|mE*T9qQWQt0S z?&+E>jXj|wK$ps5vq()P)=7D3`g>6W^H$7zG=U07`J7P%m}>mDS9@;Mm22n>IMa zBc73Cx>Qmgcm?z3?Wz;EwoNST-5_Q4@H#((pgoqved3S<5jU-NT*OGh=lRcuOWju6 zO@;Bj7Muy6*+*VP7GE77U!5rwQux3%biu;NXpFLEF*2aXzI>B3l7t)x0`cpjTUh5Fj*N$= zXWDa>j4bK!CPX6xSHr}8XHlHw1@lJRC9RrQ+FO&JBuBKrQ4sVkemlG8X4W)g89C42 zTRUN2k*uYBfSmlbZii*l0X-cZ;$ZpfvS)Q!03I&BfA@U#fr49{75>zl)6RaXS*;%% z8c>kfWn(1O@QgJ%&P4kl%bPb{ipnMwFMm|TIUhH#!Jt!2>k8oXM?hy*4={oaz_f5~UYzEEqs~ol|=`})KW&Rx0T=*S(4d$-r%3s8NPfTIRYB+c77 z;62wSE`~^h`T^QzmWYo|VUUiExSKS-ibpM;TfG@waT-Dlm-=Vr_l=Elcibg5Ekm?e zfT!9|<`)^_H#L#_)O%Q7AdUhe0w6{zL!D0beDQERj#ShX@A`!eJ> zTE29nK>I0XH#s?d^t}hifBZeN&E*zIDJaYN9gey2&zJw~z_Q=bDLyIh?c<55uxsvT zGjnoH^ul1u@ZN`XKx*@qOKB(Um9@27vouff5n(rmKcpnFh*3$ZSM#_ux^Hnf4SalJ zYEe-y)NVP!I+4T<_yX_LA;x13H_}5I{${TaqQ)d}K1N37;r~A7&vfYi8(Le!sdsLHx0uB=)%vVHq-@@+R0!9Rt zP+#v|1PN)Z!NoTbfPHC;ibQM=N}H}19EMg1#zPkW4S*g50>BzB$i2T2v$4D$ZJWC>^ zT!Y1?Cf|OkS0DV$68LOrfFWwm!U>dQ)~vuq*q$HTS&kb~H&vyjA;5r70FeeJLfc1` zaVGYk@iy}w{EY{PuSmXd;Cs+V2a_p!Eaji%J#R*m7hVG-gr{9HTcb8x=P!~4 z4acXqylQX!`P1MdFyOvMASP*eP4Crh#wLO%GcMmZ`S~pGZ)3LnM+-3Ce&6_Jv`f=V z?vmM@pr9Mb+z){ea;BBgkmlV=V3v>ofzGzT$;imegG}XHaFi`p&EG|yUnqFsQ3&5S zn%h8Y%AP}GcZ*qBY`XN?#Y3b)#GfGQST=R|=WxXt((&aeLjORvr#6R^qcv_UctKu* zQ@>uJojSG;e$N=)f;#8X>Ltdrsexyx^{M{NdV;AC#&aBfeQZr@57VjPc;u9H;yM+H zkMUL=YIqjShfA;sXJPhy0|^97>HA4^ByR{w^j0l8&3SUggpGea!llyl>~??&VOb=d zP+H9rAlan48ooL8jve3|ot|HD9=nk*eyrg=3Vvs}(~?-{DxEj(v*lOTXE;_mRP$9bW&DY2rd z$E$yQY1eujZGGImJIDAAF~RvtD}-<5Yt9)zgz^34NP}k@E;uze2iS)B5BBArU94?= zD15Dw`#>!i*XGi^;hB{?$z_K{)7^z|s|H$F=?=eOToE7aMic3ZXKM#|Omngqu=f7U zP_w3W@OIFX(p9!oddbS_xIlJlbm5TNC#CaEEmVGz0E2uYr z8D?w)`=(Zrrnc@DvWM^%J74Xoj*@vT!C02cmEd0m>AFF)xoMdrx&A~NC&FQE9EGhf zsdL&N7))5-A$M_e+A<=`lcuENLM-=;Wy|9s#z>xElt2XO=lYD})hNQnCAirRboi27B7vfJ!YTqNTJ?#7UZh}rR zMYl&LK)yQp^fiW#5BOU}TUTb53v}`$XN>yhe(11h{J}vlrn!{lnrm^s_H=%T9pPl= zKw^Xxlhq#^0=QiU=>zJGV$in&y|_5*?i4o#-j^?VPiaCP(w|>!dAI+}d@|~TQFJGQBvOH_)>O!+CRt7OW#zQqc(k=pS?d?AG0cz=-K{ zQ}mB!2P&9~!{G{aYP(WV6NF5a-=U?8VgpDm^>yKqT=472Nc)Rvbv(RqKtFweGyu8- zA8nZwt}iBrDQ##K;zK%N=HncS;QrV?zr`}?Mwg#UC7+(7q$os=CUmq{^i_zH4+A3b zyDUb3qX2v#!)UDBI@x)6hUR8u!lnIz2Yp$tar@4l=84*kKbZeKM+90=(>?vJprS}V zFwQ%Foy(kl0{g`kbo8n^8`tOjlLziT* z|C58R3La@60k2hHSzEg=^|qR!>=qf@jJP(}K{z$Jkjb_o&S1^1`j6z{WipnOt_o6e zqSLlRBw@_G1g7jWeM%Y9JSWr=OtH!W_{e#3{!#b^iiPB(iF<5?FYOY~&xsFfRQnWq zSkLl=>^I*$F;@Bu2}eCjO=Y%HY@~xVWIRBpupR6(!oi^L0aNU+6zBIek^X-J-uTz3 zFpvmwtz@dqg;+(#n9;feI=7=$Km3sXDA{@?CH24oQ)N4V!4ZU%)MExc$EH*3rj6ze)xtc(i8m@?{fpvCG%wLps(32irEBQ~eF?N+bTBp(&xi?3xJoPUqEcX03(%76VuJx|5yqlH%%|2`vS z_n!`>g>Gi6zlQQ{A! z0y8v6@Pk9LaMN@tR>?Nd@M<1m-F&K``S(w__H~gx7*4?Z)Wwm1y1ru6`%BmF9t&${ zcoQLq{O`HpH&+v-roWunfBxrDt++oc5=(yS+dD03`DV}SXAg3~|9)niWaw`m{a*+8 z$*TDHKZp9~pc|;@c~}4Y_y9@c{F8hC&(HiHr~G@l|2Yc;*#AEryK~$ zYefDT7IbX2zuKMu422olza##iq5hxe`LHQvmRl5Y!!Rv>=c0MKTb%Wv60}1_m zkUzJy_(czLkV@X1;=A?Vr)MB3a%%0LC$qgU5k$MUAz%4{g*=Ced98) z)SDPkpiTTit@giezvQE{-B5fr%5Z7Ri#ftF4bRXng=F^i5Oi;m4w7)ubHRbKEB(3=QDiyQVdA(U&>%!cj z)u<%ZfVw&Z*dmCoUw>USnyq3*ivaG$cFT(u@KHi$oQLiB99X#fHclyiEy_ls9sY%p zBq>7ex0)_@OzJ;>nqEZD_4KGy(k+xyZXazQ$KPGg&L5(?UyH~zz7Eu#q?XogNk4xY zvm^*xS9$QZ4-V{(_vN%xFd)5rHnF>V=*E=F3U}JTfc*x5M%aj`t_9Y@K07L~AA7IO zz|Kx6Dk|zU;3!B9irA>c#E_^c!ZwJNcqS_|Z`!NVykf7qo`s2tX)t$g#)1)((riZs zO1OyH6?LaQ!0(yWxw}|9vJ@GM-7Z*{9fp%Q#i)gXsuCFMYF;v2$sd%_cy%F7;eIh~ zLt4>!Lf0rq)joH|f*F;hYspt=vGlNx-9os{VCho5u+XNd{;c}Z$q9V9>j)nm5kbSl z?qa8|R5=Cxrz2&zIfbtPc`~x%1doI(=kD%q)cf~uU~SLH?r^Xdfn9K6s|<)Qa2~UC zN+jnpnFYr&lST_}3J<8g;rBQ$nksq6Am0LFa-TH1c0R+ALfuFRP`9(b8*w6i_?HKB z-aWCpi=6UiHeUTG7MhUN1V_to3y9t3-!88Bd=Fddddv_{c8+860vStW6Ch_wfPDL2 z)Qau?fooQ)_afl8Uo3`g6fXWI;}6=ecd(6279_ys*xHTM!0;pcOT9js>7{{12=uFx z-L4@ZnTR4JLzjPmCM!#lotO8mM&5m=c81mK(`*r_#VReF#M2c^%%*j8eylh`4xz}D zTWr384}6e?WrJjT+n4fMJ@y2@ArurZ|CqCnchKBAmW75)baa}_Yl9Ij#b?HqJ5o$0o`!w8v zO4Ng|B{ihoAoj2thFrMkXeb8D=N$PR|C-dE7n0ljjP? zxc5>q%ZMG=#30{gcsEztC;9+-3nBKLv%oG775tytpl5|0^bQIm=X#y5s-)EHH#FZK z*$(Y?P}3EIp^H`>*g+{W#U6u$+zu4()Oa%gUD2OXpyy2l1I%Woj=sD1L{U{WvZI5& ztz8N*f!yzo*w{}LpMMg!vZAKH^#G9iud>3hV!z}*fEmhp$tv#ZdPkEFP*s3p5#pdM zDbt|#@XYMNumTAhjGXvw;+r>hB07QCy zl!$D0cY6c`VP-s276x&ba~J4nRm1>;G8rL_m{(NN7Zig`0H1aNh;PW-8Sm!?6c&<% zTeG$4Y4k(-g&&N+h*?V&zhN%gaDVXC?tP#Nz0L6OmZ(DS+B?Xl zLG{SUrYgybR6Qn;PAEUudYwE;|B})%s;d=l+*LNVK86VcSF!(1yQp*Tw)2^kMM@U{ zFGVALku+^deCEH!jh0AJ5!l#WKdPS%I3BQo_Rj0F_Rjjz<;CwL(h;-Oy`1bSTy=pg z;%D#f;ySCLfrj?c87K55=uY53Lli-GC}a$5ZPUf8M@>aFX~41q&66OE)r0qg-;_kS z(i4XiL50`)Wc4+K?NTqbPSbdWM~Cj8U9v+@K2A>J#>S z@ZfL!G^tp=4gcV4=C`wHu z{)Q@dS>DO>-T13EI`iBOAYyJknYqu4h5&Q0*DD}Kwc0~UMqVB8;*@DTs;>tX0?EUq zCmCAx*7-u`-*VZ?G>Uo+K+09+d4$q6TY-g#-;x_@(z28;KKk_upY5Nw_iWFa-3!Oi z1C*6*=0?AVB7BbGSV}0jnfD}UoYqARoF_r|HY~#(&Po#?uh>74M2ky;9f-AmL~O-4 zLP{KwGO-cgg(@Iyc|X#(+OD@rX(`lybr$N#8m+LOPhXEV$}J^cwEV@tkdSy3R%7LV_S`);Iw=fOybRb)FS8MFs|_>lS*sFeM^u&}cT<=I#Wc4(RB;<|H9 zA6)J_bv_8Y6626E+pekojW1KnbQiH)t(Ir*H^^v6lY%S@E)1kI%s;y9FxE#sV~kle z^Dd_;@(#BNZwWF{O)s*gA(!{0Lxh)?GS-#z$|rP$CA6k8S1wnU1+-+Vn^B#KLog@e zvl>v^pC+I-05~I?nIo@k!=~c(v3z~nor=$92CbbOq}y{_TV1{HK9*qLZ=8sD{aQM{ z!ff_E{wGkO6j`-DU9SSA=6!U%q&pj5Kn|oba=0FtTQ?p6ON^}X7t>?gEZhI6_>i@I z;*8uW{uX>c6*W{)5LNag4jGb+B%t99d$ysR(8U0tm=62!N6=ay)<|3B2c-f8eMXRC zAo2ZWbvfI=3cb-OMcaq;J12Qn)J4;IF*nc zO1!c)z(G+S4A%#;5;%=t@R*a{*!%~GHeH)+u&1qd{pw<^$cAVNuxl;S9}xY0o#X*C z>0E}`d1t1foM_6M!4LB7%PaSqDGM8XfU-c){ST>`z4#D4_rt}XUK+2~19&urx%shd z8vfAZoTl=-UL|AIm4U&#uWHi{Nc?g1ktz_sj!B0ldK1gh{_GwvbwRex&CP8vl4`iF z zg1vDh&T_wW$s40vV&-VZGd7k%!08t~+;jvoL!7eqmM@M8J^62i-hH!@|67F~iHQQO zVI{$nsW}Ded3M)dSUSVS_dOfnih*9a>Mub6YSSmAebNtIu=Ma73N%X5;BiEXD?h55 z@H?Kf^qV_^9@fR~@Cxu%h=}@K#Ta%H1c-E2PA4J!Ru!h3zl%Nv2|=`gRQ=fxZzL7===%yGpyHxM)27$hH`_e+0LSk6 z^8rV%DKnYQ>jPviKvPssCwCzMYU_1EhezzjL?eTPzosuHWu{HP)l(gUJJL8xhBU68 z`h93%p?18x&#?AW(6D=X2`NEsqGHr3ce*QjX-@rf%lHAPB{q{E*KO$5c)T-CaawyD zTx^K(fSI?4Eaah!{xE;ojp+|8Hs86A0Jw%UvdHAZ8rERnGlE^)KRuESS)kRFeyjja9Rz9ep8nO^)UJ+YGPrB)I+~mfngDoQrF_W7$r*#5tB)Vi zurp3IEM`63uvJv{#&`;SFKgFtsNnGA$-Utt<2gSw<|b@JbBTBuwH1XW+XTbABc+gs z26gogtTj%L*(;v~=e(0HIf1Rj#b!)$m{eEy-fP^|o<&ureB%Jnp{f6HMXDdw@azxw z>f+@R8$pyiCJ+&+|GNmq#cQ8Vr_t^+l416kA0i{9vh3t_kJ9#G`%{kc;nJQO3OP)uOJ_8Xm71nMXM$fqt%P}Bp3NLX^#fjPXZ77s z=z-dy^E83ox4z&@N!;1)Q6(D=1_A{o>k4L|G@ zuG6QwRkD$7k$)>nM(TAjC&IK(_iJVspNsJwjVCHjD7jB2%I#4lC8kx6nQ%bITqiIp#7#ZQy06_aaM-cE1&eU1rNbDy=opyp{{;XXBLNmCV3=E z(dRywDYZvqYl#Ex6;+;3*qQGZV!@sZ67n?nk%;A)N=lqXlgYm3F?+Fze8Z~&V%>)f z5}x(<=ydd-?eXW_@}`rvv0*ztPLR!p610j@6I1O*eNXSZVbrU(iE|I+B$P7xx%IMc zl~O0msGgmt2xT4Jp=;{Xx7uXY&f_VzvRM|!{zt8ce}$avnz8f1P2oww#+vc*h}@C9 zOi^jvt)L*-;MP4=8NaMAg4r9aiA~luP+`E*XRPB@tW8_Ce-shI(Sy%DK?(!_PJ#{o z7``<~@_xq~w*?bLeK$%Fapy%ggx0FEML9d%XsvVILmVa1gS)aXV0=6qFqdfu+VUGCv^nY_K|Vn6Io5R z2dxz*R+eG&C})xn zS+5bF7(*E4>%Plse1IBE*|Y#T%<{>2zKUn;i02^KPX0a4N3dJMF=2+F%eAlf-owbv zVO(usVJX7z6-^;!d=86LJna=oUh*dIu%PzX!ydzlZZjO!v$_ROC=xqf+Ld?O3zuFC zPQHF!wpKz!GgOh$Xv_bWa!xq`CGPO}TE)olZmcGCLGD0eTw!(@f!Y~p72`x-eb%kA zXMKv)Cz>6Ookh>x%(Hv~rGdOQTti%2z!I80lb+unyII5HcOs8oXf-ta-g>n{g~jS$P=qs2GqFTA@UIHtsg_=O&VvWW$OQG%bOhJU z*SMShQfyCUq683GP!hRUUVFSdLutFS8?Is&FKvjYDE+YP9DTR~-@?_^IJXRCaTk^J z&mU2F*1M!hH6|JIdP5hU@BK_syc$?oEuQ2TJOJzPsi(%VZ_xdmw?yzku@3jY%j<#@ z()t^MpV)2BA!y!kNIh7aNKAAms<$Y@h!-(PxL*-nnOAe2+&#dE;DLv)C zvM<0cY+b}*A)V;21`~9e4po+3zU2Ww7p=0^7&T}jPV7H7fuam<;+%7%cAK>ZZ2GpR z=_uqAvH*e9eoL;t2jl)b-t5o$$$EKa#X&11-`iluBboDD61by6?~B~2wABj7+@#L5 zf`W?k%YL^z7k?898otgVYblT)Te5RNY+z@IVfe#`CXc!GOSzD1VtxhjHG}QV5-I3J z*Sb=g;}T^>ab^4XduWu4iylOik%PR|!pZe2Ip##yrp|Xk;SGAEALfpVIXtucy(_+V zlUixlpW?YbuWDUjzVo>5Q-uF{!FEh=dd^5jLverR0qi{w6{F_UN;2ghSbIgAjrE4q zS(*X*Gse{}nUg0C1%m)Bu2Y?!JWdCAyq}-UBi7A5UE$e`QBDHQ)H~Zt2^vJ2R{BFN zOOo2Tv;lIe24~q!9!fs;CbuxQz7NQU6$Qhe<_tfz?azbnb#gQ61MMZmguSU5i^nIA zlpyXlJ2)!)av&xG&E&fL{d)u*)~` zm;R}v#$(i*dSgA+&~Bc?njkcm@0JB}R~>bB>OiSXwaWgOdY1#~_u5V4FCAD~2Y3@+ ziE%|9sd+yWy!&)rq?Fxx z`2+mgtaqAb$e`i_sWALJASBZC5eQ(x+uyhabaRJ_o7^Pp=3l-nOifWzvpfy@SeA#X z6s-@?vv{U*MnM4zg}i4dkMlL+kLlTYl2%qCu*^F7>P>{L=_Rgss9RqFLKaFjuKx4kO=eQ13jWmnaja^8c`A<($ltk`V{G2>Sc&B>P>R ztVI@g5iqhAQ9?-7AuOhag#`3pU!A1I0V71 z@grM8Nx!55E*-shW*gakSFfPn{w8yi>pty^FT=T|)5)@X;k58b1Y>m?0h9Dg%`E4@ zCPD7oCF{WGXfSj@BUCNaK}UG(_siVi`LebL?w8(6A!1oF#a}yc8ypra;4!^NZE#t{ z_*e|6F`2hL_6|L{xkpV!Xhq#3>MQUDC*poEGXFM>%^q^(_uhHm>M1S#zC;7S12Q)4 z%A`U?M%O#I$#t@ekoc%QCc zuQg`b3Wf4-zrLwjM?HVntWVRwS!RCx2=>@JSsNGd7~c>7|c zSZ{ie!GAZ>KwNC?%>BF#z4pYdo+iF9VVsTR&5wojL*Diw(V5SxrhB4z{C2A)lrnop zW2s#t?va(N71|qQR(7l`BY22Urba?q(#H7DHCIbCL3yC4sP>a-qsaKAyEKL!WYW9q z&vJ*V9M?#)mx8O2#IcJ{!qp|Eullq1*m4DF($qbV~bjqh)Dlr#T5p9QVmHn`fP^ysbk{HFs*S zBO*T3wKa4sE|AUVdfWUOvZ<(YIl+i4oP0y;L?x}A;^+m)DG%{l?t=DZF*Cdb0#`X)ORW!$Cm1H8r5! z9XnIi%xykQ5t{?`RWT(oEJpOL@{x`@3iaR3vCx`^EX+E(Ib+ct2bY?VfDX_B5-UX1 zJJoxeHgT0vq;}*8MQ%X&*1EJ0R?gShO7LWV+64Io>`k z^N$KLP{+o4t;ox%oJ#YT><4daJvNo$Ur=dDCE%2*`Z}{Hn*@Yv5>kN|K7kt&FSBt| zx}Q46s4^CN&C^UTz0=SR(sqxS8$WP}uiSuCH`IP*X{pt^18Rl2ODsF>5KSwks){=4 zSGw=GF(VODWSxffyiT%UN{glpL>=X_JzmUZ8Z1eR%k~o=v%ZngIsc=BJe0~Ef}OX7 zwo5M%q4?s(*Q^N!B1RHy;mh+q7oh5PzyY{)>lA4|01Z4NLF7T5lauq~CATH*jOP(G zjv4_Ke-wHi9tIk+zk9NX1x1}(OTg6xemi<-P8j{}UF%s+UKztzuV5PaRDWs_ zvgru~;oKLd>o9RmxQY@9QGI>TF{S7^VDkb9^&g2{VdI#eY!b5;gjS&MLuZHGkhGDL zdS_AZ69jq-GnV>YIDzy)wBRm};5}mqzk)d5x;umT7p*=SyDk%{+hm*+rrC8G#|yQy z9{WL=b)47!Egek~5d#+oHNk;XIHe-! zm><5@Wl6D6m7Sg2GrkYedG$$Fjpr7NW?L?o$L^qUa-$I*qbBNT#a3*8$4$UNZI}}W z){Do#O(#dUkXx#%t|z*f*uc>Mbjvna=!{jM z?t&hien^A#^`qavjXaL$g4UfAUh#bqWFkVi%bkvEad+(70Tgg- zgU3sEroe7b`3rwPRFFsVlXi1Z2tXJFGtWfMljhww0yIQ$z%X+^h-dxmK!_=xVWn*Z z-WhSssb2vhqQk*%8ePIHoq_t2jA%oY#5g%BF^b?m_9;XA%5LF-gryuY)f(U6`c%!WUSn|8MeGwhXx zVWdB{)>IZUCzy~i9&8gDo{y#{h&f{?F?cavb6!tqP7pQb4p_9e;eG|UGTcHAi&M3@ z@O36+$mwZ%^Q~Z;1S1Jh_7YHAR$hZ-2VkLAJ3PEJ-D=245*%=v(|v1M%~$O_P`uI; z^MlxYIe-xB_NTk=I1~92jZ8SVP8jZ&Dl8hgab6=PM!oH;DChG-?&{b52o4OsN%z>= z+qquekH+JoEW5s4K^w7Q8Gc(nQJTquHqziIcHfdAet+k%#ez*NaQg?HzFeIC{^Z!R zJcjUhCnvMvs`0lv2IJM>+VY37>g{#CjT2v};UTQ83taZfKLcoQaXpZj#auJULG`1~MRJo-8P=?#4}R$rk;K(pONfIWNUQ0o-l zwYy&&C!QJU5#@9n4>rS;l3Ojo{ zEpMf`gglcAw8)(2%flkDTQv(Q3(Wu&%wj-i%3V;*E6u4(7SH!+%`6LI^z;4ddi42?8j>73>tz?NG;%*O0EJmTK%3| z=>5~7kTrDP`%(@FYJI-sd5cViX*iEJkJ*E`>41vu%i6cn&- z6lf}I_{Ndqd3e@aqAJN`KWFLq&f*soL#sv9EzFhEK9*p}%VUEzsrcg3z~-sr z0mK&frHvj24+Vr)et6=fKqTpRL7n=%@QZB3msJgrT{)W5i|$oq8ovk*!+PEQG5m>h zcJxSkSs6BD=1@RicJ*>XSy=<6x(+2e`ld?k+~ME|AEW{l$~o5zKDJVszw7_`QCh*8 zNLXYFj4#ZcQ{Hq^co>^Ua$XegEMLEd@89ebN6N!ZK`kqL-*#bDo>9(H*1V!S z1wATaecY2|#Kwr8UGX)aPWj9<-I|JM#qHau2$Q~q;NqPJ-g8QaRDeJ}uot=r-$*;B(7uiH6y{XOvv|QFQuHhpe&v;m>uP&7V70+1D<{|X;m`ivrooB#`aDqS z*FV+V^gbAY=z znM<<+w_{1mcyxu32wou$e9;VQAt$)Dxpz-Rn>(U!Z5`MmuZ~8|@~Y%RpPZ!~Rf|dq zosl1#Zns{4HJDDmccg8Bvr3^uCW5KyM@S^FX>^FU6k?iCu|LHJ80i<^#4l6pRDpyVSWo4|}Fx_RW5u#YqE*w4idKmKZRD%l|# z`RPqojNyk5A0D9~|A;WulZSseT>f&d3_-s8_mwR(`zaLpGa+*dy+>|X@K@x^`8x8N z{_|3bY>K1%=MCiVgSU{U`OoVkG%iAp;=iv0EZn^Rd3e3QXZ`oXBB(GG{_C|$bVChU zY5C6!P6Bt~zg})v{$BgPAN+qfrNWUL0(oJ@w4y)6bw1XPw9&<`9Rbz2O?OCOe<0&>g((JB3Uj_tXOs&lkP92HrmXcG1llDjJyLb(KbQhue_Y)s$ zU;FZmIUaYA$HY9>5V=blic~{J%_6Q~o@J=Y1@t%xR7>$Y{X#{g<$a{sI(Hjl<26~O z(a0wq6OdwLW(q8fGF{;mRHO&z8_ooAHycXvuNIaSi9`&OZ_cI3fJRn@{23Rh~*(%tm_mmF!j+>(;06WTL+>}lCmRAXa-QL-sS z6coWa9CL)B8_v~V3UUHfGOX;qVZ{ZBq|+>})xB6ISo0*A9AC>VPGou#5;?{xe0~5* z5ComD@Oi|Ms;X+x;-U#i%R%9OF%!hX!qU;%*^;kW38lBgB?L1o!vSQxdLf*3Hu>Modqqws4Zg4?Cd~WB!?! z6bVw!Y57Kb2tzb(qOKSvrXtLP35_kBH;y6Urt}{5KW|Y^GA>4Qn-7uVn*DuyW#21S z_BwoVySZ0Pon5&I!~;x%reA>rHO7;J2Np8zyj+?zao8<}Wy`Gf@ZnHPOkNk{| zjwJw^lZ}m_?(EPWf6G5O`#v;bx83&sDruFStF2VArU?13cVR4~N&PMKd zFb;&S+Qkql{1gfT#`UJ%TjWJyXJUUS91GKJY#^qgu|^>G#bM9NYcF1Wvv_!$xV5j3 zQZ^7sB{W~n1IqCEuBg?N{fJq zgdiZOAPv%83L;X%q88n-Sajz*mp;!s&U4Nf|4-)~O8M&~?BT&iPMVvgZ@{*aAnlaiu^AHe;YTIO zz{df!BY4Ertc{o$55Y|$NQ&n>8T}_&%xULJpEcQc6DXMzBP#}F{IjlDPg8h3QdNHi z6;-FaQFL(xT@Q&dt9>~BK@;Npe3S2&)vIU0ZtO}8pCM9wL2u`vQtBiD?;+hg3ZR zKgP$|LCJ-O7^!k0Ar3lrXcCv_rY1VX^?o$CMvA)$YKJOf;q>uo%#M$nKW5FlHD z?K=h3#sC2O;KUg7m}kwggOnN|bF)a0|2_KQ6x8bYAcF(dL3*W5*v37y2m+`uP+P8T zIhPYN2s8141@h$F95L(*#7!R`z=59kr1x`hNiq6(DwTOXDb)?wdi_LQqiqs{34`2| zz%vBSVg?bn!}W&F04z6qJIRkjO91o6nz0 zZ47%pHJkmUm?!tZnUSk0fHPXVyx-lVhg3{b5@z6yngy8(T#d^*H0wxFA_VaJ-jd_l zY#x#V+ek*FK>vO?)An`*0%2(A06(eX*$qAIlZ8I_obAj9Yysf+i2%vbS^TZ2XM3e@ z?r^)WCifSJ_-(B-u-f;IMa(c5?KcgZO#l0kh5UVLY8@yy6IO5L`Ljk91!L52L7;j%Z%@-$u3MpbJ7 zkuZ!4+fUrDjZqiQV?twL#A~K=A?U34m#*^Px@Vn1sZDWign~GT8jg2Hk-mfA!I`_C zo^AZrSa8;TmpELPaQP~(nXKWet89DZw-0+8^@jGrPYm2z|8BdNXGsc;8gUWW1#z-d zUhm>zNF8sVT#c@-teliR-KTJtNY+@jvaqnYs$WHGGmswyi%3^Dv7-#j zB@Aj^tv{8CWS1SAeTFR~dYYjSanbo$&z_hR+ijW4^YYMk`y!&@=j1c!5VO~g(o(x7 z?rl6gb9Uc!D5iKF(~@dir^(KEskyyNpon?Rpzb*h0eNchNmzKTaGh zy&|hkgP#v^lDZEc^xkp5D za`IYJ_7B2ZI}^$F-CvLII3hkjwt!PLy)0(wlnWlp3)>lS^!SISE59t7QzB%S zS2 z6W=!{;+}F9T7;49QMX9znB+y$^5?`oe@uKLghb2n`(@~lcfvxyIz7Gqeso@6U!RJa z`sK*TNPj&Hc;IX0=DdX^=XEQ=hrbpU6lCX4ze6^%@pfXK{F|jIF1r4#?6=d}Z|zc> z`wAFxHU$%RFE3&~f^2K+_1OUw+=1Gxw=!>B4222V>A|uWXRb(aVatFb z-1x1@A$Bd9$fY&pu${a%W_I=kJTEcvyI=-U1Rvtb!?HYe>!?}xx`T^y?h(>bBY~U! zmL))!Bv^XzBJKR|7tg&k7R`?je{fvkqD46JW+Kkg$6T75p(irl9=vvI5{)K9Fv{c# zmq__er6W%^9w3p3VN$Q^m`)>aG6DVC7kMa}G$d02L3`3At%5H)5$#uJJumLCOK}Dw z;Zkhc9?exX!J404&MO=O>vm#2YH~v3(mUMry6@k`YYhbs##L<$S+<8xw)k#)){>ul z@8LRdx9j&=5D)%jvFO`>~o87)w0wl z+e=X&dn|Df5VrCCUE8P0nVB=m$|0j(QRf94R1&UktVQ3bh;}6r)c8_&G!-L%(_>~q z{DXMvYx5v#!~D@;H{aS{D;}3Mt|$Z&m@d*3LrqEa<#7-!j*mKN3b3%3n3*Q0N0}1x zPJMZxdf}uFD{^lXg7`2B!LuX4jlF4B$`TU!p)qu_NIzx4aj*MD!r|UiG4XSLaj=UI$PY65`BKkac@Bf(SV#0 z%9&uy=)W{)17=;3Ij5*9*CljPVCznHN{6U7{I*NX5_LuC1L$ZiqnZ~JXq#HuAo&YY zV`ayPUu)CZI{)PY@aAB3TZ@aqKi7;Nct3sDQBDS3mydNR6BM)+&u58c_Gf4iH&s9?Fy=w#UN&Z)z%saQ zRGm@*HDgbR1tYUk!bwM_Fu5WPx))hUG1q())&k74(kU#W!=Te>N}?b{X?3m8+gbO)q8qe4klye7!hO?bxl}1K#umC z&&`|82e#&D@)ug;^wt}ankO9yp|>SINV(`}$3?&A8PPsRiz0y01HI?#>Z+=V_jH@6 zV8Udy z_~(nfb}dxR+}zw893P#Wc>babL(L6|!y0OOQBg7{xa>YhXJt;%TG)dBgN((85mH3U zYo^x_2H@sdzf)F3x|jISaC!i}El(0ASLBm79Fnk&W+p>eT3cg~-qYrBYwn<7!$i(% zx~8daOPlv`q>+VQXumu8_`$K4gWI8kBZ87;#8cVDC9e?`o0FF(bMavb*^$|GLr%t0 z(kjVjmXk4ic-;+sBA@F5$~d4Md!+uD9Kk5^)L$dWU8yCCncPnnT=~~&;vLuKd;PN> z508BhI~(-H($eyqXQy{>n#1aykYN>P9tRcCYM)l0n%AP-iB~Uf_M)-U8k`4o2b0ES zu?wcRDd_X_e&_ZlosARap6e(YC($)}WI2xPx~&BH^wz`mc6x70Ub7%>K6O_Op$RK< z!(7tH)9AAE@oMnCb?b|Dy+b8bc+)%rIEb2B>1)pwa*-Pg3xXBZl})|9toP|V8dk@A zs2u8x9*d~}Rp+7J&hVBe4;R;=I(>a}2k7^im{}c!Z)4l$KDM$mY05EoK2XwA`wtag zB<2&Y3aD0e`KoM@yIevlsODgPen5HL$f+Ht{gL?^EB(p_Q_^wyQZ0rSqd$q(juzP}KL+8gaBw`dC6*X=w)~ ztRA~O)KBv9$;qA>bes6RE+sVr@{CX9?9H}Ae_Gdn4Cti|puyIY*D2Mzsf_F{);8E~ zhy;N}nm6YjDE?`?e@D3uoOEtzHrek{71UMGVmsCexFeR?oZ8aQ+0{jjHX@N?mp?al zOW>u?x-ik3pWc8oonv!b4i56{xPBWX%hJ6solSMDAjl6sU!eEImVJ5HiPW>UGn+5e zO!7_7r{EIFvdDZ^0TqhLBIQh)E5G>O9~PC18h^h1=|^ycY@<>4EpF=RN(=hoGZT$+ zC+?@C40$O|PEJ3T-_uMkE=GQ8?ZQItw-)nZ3j6 z6Jn(r!H83bESkK@)L?Eap8|C+8 z96D*iJ{Eu6VJsl~iukWz*>9YuxH6Q(LX6XNmMq$xUS`#M=!w{lT=4d_=yE@ET}`q6(?w;*Fw?sX#)1pc$!b`f@FxyG zKR+Q6k=&g-kd81V=yTwFwBlBO>)t)$ldViy7WH1}UO~%e3*oLO8f9O1yz<1hQsIt7~r5 zNR2!SV}k0iX8WKQlW&v%C*_(C?o{ib@`r&(XZJBPnsl?ABJFy*eS6O_B57CL`nQPm z>+nHG%YP;cp{n^P0&upiMOG_eHmBK@z-Rrd1(}(W27YGe;Jnj?j%lRAgG4R!dviwz zT~~tr=MjGHz4+BsE05%Uq-aT{K|q#pi1vtQZ>!?cnV`ocX7}&MM=dT`H1`bH*&eAr zeL5>wFu-{u2T_FQMTC(%*om~&f5u;hOPe@8j!!V{A;z*_;cBTG(bnTPwUjj!orMV$zmHfdM{_*tpPW8KLxvTqXQernKfv9EV?eBY_&8llph+nZWm8p+GKRvc=hIk$p_26p`kYW ziP0gOR#>=`8Jo)OKZ}-D$q`yH5ixH)J*&=hb9Y-0<&P=_wXq+2*R1^^xj+1&0@&In zy>qoN9UId3oqKYf6o0ztJXdbLU3cH-o43tVGzy->j(w=3bME=`=PT~AU>~@*@F}Xh z+vrqJK`Z|W6&nS!$Z}+q688S05XJBBo9;*vzH6CQXw~lfM5W`KF80MS*P+blJb#cc zlL7FGm@xMg@f~(jw_NP$``l25Ay{jZ?I+>8l_4v^=3i94Hg9N8kIN5$ksXSHFpmIi z_NZ9tTwCX9886*Bt?NPf|FRYr@h+#i7hVUs0v&tL=8c~`R9@Y<<~!oTnA!CN%wb6H z7CR9aO&*RmZDhqv2ekr?%BJqF{|K!V0sb#R;}Jo~VEFX4;`dV2Y+y8Yz%*xfxIP4P zyl-Ex%0Q2&Y9;BnqK;^I(|Am2ot<0jDvbxIlYiEQ@XX9>p<~6eM1bvU&PO|MPy>Pr z$`YJf$!#Vf5s^>%b&vK;GcP?+5azA2e$y;TI+xGB#!Lh+6g}icc$go2_X=hfWp@w>!9O-o{Y89Q3#sT9~gHu?gTMr;B?+>|+*ovm(3J2wjk zj0aNA57w@VJ^9?JTKY9eaP=sNmh+60i%TQAfWH~W@6e$}zfBASJOQwPkH|7*cenM& zSKz{I9o$P(&C-42yhNQychCD|)*L{JoVN9sxKG_DD}Ri-8mI^)g*()r5}j520w7!K z_(>_YT_(j3pv<>0n>U>a9puLt77R>U{_$XWDBz>@sl|ZP?AYix?hBV27+Yea3~wG<>X3!{#P4I+5^tReRuFpeU zlas+pEo7}=JAs2Zq)4H>a?pVaQ<&=S9+uaVVU8qT+YN}1CtzYu3bL`Vt+7j8$TyID zR^-!N?#BJiDg2F=>dUCps{W-95skZ!#;7E#p$e1in&8WKPF<%}T%*p%N=ygr8a4u* zI66k_3~cDnF$}O}xVWi#U<5ZQW>Rzp-X?>6BI4#!hwWsM_>sNz_$PTu>iZoX_q=!Z zZ%@w0w$G;l@>@N9o@*%=T+gO4<>O`5w*9#}P1EB5SGpfqO^+~wy#zzbc$pouf#0!< zG5Jfd-@pUGmdGI_2TKEM&B5gf!X2GgzBcByn)dA*^=j=d8+OhGhE=7imtKqdjsi~J zqr|R*aTu*~*#j=RZDb5+d*0u$AeME2Sr5$6o%gn3V9k|Iz|5PQ)%r&4?(h?OR=Qw| zloW^~Fiq8|H6ZwM?<~xVinL!+HJe;Bsca`>&a62|Ej7oE6tYoZ+ottLcK8t*1#b9WJvU5EGRa3?{0!?3{x@6aY+Xemdwv zxOrK%j~5m~T*)x%^X7-=2QFwu?>{Q%m65rirk*}izV;}sX}U3kp4w}Nji#xAhrX+D zxPPFPgyvxko`@*#vxD4gpCkSXMY09nE zaz&T;kZrvRYj|ZA)jU2vI6q?4O)>!86^DV3)n#wY11>R45rDZC*^bGH@Yo-gkb3|eJs&#$kbx{PnuGa5gCoUF{t(%RKZQczRzeNK-iU`p3;wVPjgDy!F} zE;~7NLD@=(X>uJjdX#i#HP65m1=5jGv&Sv1tuRVs)rL6f!Y;67;LK%{^_7B1@LP?gU=A-=vjd02tYduh=yOYPJpWl0Yd#qM}x$|f(Z6G zYf4|&qapDi80H2Q6u6Zycfo|Vq=dTEg^ZAdU~M5~7V3TC&W$i9k63Pc&eg9Bf7U$MlZwZ$tr5RCDzB&KKp-F!EyJ?+ycS zU0dl~IM|M`FIL(Sz=Ti;iB;a7ZAesbuHUa2A?f5N&mFw`@ZrM*3FKfR8WJXx>(e{C&=W`sZTz3VXMJnyJAik zW0Zi)o`eC*!_!xH@q*BMEJy`%}TAMrn5WoQ0NO@QC*|4p=x67x}b4la)h ziy2QI_x$&l=<)f>Jn8W=$RkZslhz2on1ROL{3f$-n&i~Z;SQ`8i( zQ8W&a8A8CM7U7?nN$}wV_pYZHcI9Bd`pyC&X5qyelzIsNy`EcZd%1W2zy9}Pnwp+o z6!f<+1||+lKZ}52%1gjXr6gRg!hTrUrmPN9&TP)Qp%opk6}a(>X~7wP@6h8s7uOln z+hl+oPt>wa=>xrVW$6T%3QGW9Lr-&Y)5S6gOYnH7e)AKO>!Jip&m-eegR}Vqa9rr% zA;@dXVaojT_b;XJdlB8;H@_={GqRNR0L2K?Y$RA=KOkcb1* z?q?@Iv~9298@65%g=l+Q=;s4+o~Hg40A3vUhNRh!4q_-vB})XfM73bSj`p}OOg`4p zA&R)&JNH~vRP^W5>(%XvNlDRQ@U*e<9GgAoFkbiO&hlxbm8IoPQ&ZEVR;utHa6Doh zYXBsM1aa2fYuD-0_<0Z`I!^;_E4|3#x@g zd5c_|qrMiDlV5;h`D za?i0Jim%L6I&T*wgeqfHvedEa8V3H%^qLXIIxG(<7&?)$wujV_}74 z&L)Y^2h_1K{cPC_ri2UR~dOqA0O4g+We@V)%_>ErKRI=ElE_XskQN! z`9GpbZL=v5mskx3nwvW&UBVC_tE#GcQQp9%(FTXBzXcnXV;x4r!x7)UzR@mV3f0=e zKtU=}gp7=gbw#USz^x(@OtYG&w{~~k-1Yw*rTG0@s+agLGc;j?IeF-nnW&?A5^wR4qjG0&TUeiCDsk%lqts;6bh=Rxved_-Z6Zcb*}7>>X63se8z zTvgYKIshG`s>!jSBP_F?qgFuAC?FH!-95Ux(GupBe111tO+5edassK5h{mt-Lndl2 zwpn#`R0H|U&6uyDs#o;9yKr!?Azb)I;L2`KhN4GlRDp--8d4zKm$EX~WFbn! zwRL)VWWD3P@A1LpNTm~0Q?Bu;vVLrbG-|5V7m}333m#4gwS|O~=y3Qpz)lx>CI6UL zGmF~8%%=ra=H>{DYoJeAxxE(DAdHXIke}j~Q-691svH!iI#>15BUSB4NV9PFtT=!D z`WYB&xenyz*!b(_$wBZ8<*S|^%LDXdMwtXsTovQ_|f%{FXAnU3ST?WvY4J{^8SCv`!yei=T0E*b?vB_#PJ4SYsTzDXLZJ= zTX>J2=;A>gAqrt3Sv$IM)4Y;XIJ9FM^Pw}u?u4z1V1jk+r1#_D#?|1Zhov{Tw2#ix z5%b<6O-!=OQTxOtzyABxz6j^IMUE)=fqSgqeNvfdj?8H4K%PAx+=`n)bY0YLjKm`u zqcx~R9@RY3%7c1PGQYeW2*Zi#-<39+bgf*`jNx;#JK^fF;}5#2TDUnt)D_d_=n$~|%+xy{F&Br5TlCDe+&OyQW z!`^XW5+>qA=>-L*F_kN*atef?MJG!|mF{>$l3<#P@5@h0QB&~3=zT@X*(#IlB(N=3 zQUEOTAoImXzw>HF1K#e1+b=} zPYg$3D^MjU!y}Msz_x`5y_=(UF|5|565ge*NtBvi9v&Rr+>`)jVJUL5?znKz=ayaD zrOeCE$6`G})0Gt^n%^}v+MZoqe2H|$JJ_XW<4LXxBw;9UbMG`4-jK*WUE$&Me%Izr zMavOzhdWp|uTgY-1+4dK3s2rVwL$ViD5d88FP-wjVvsu!Z`@TkZO68{P_!wp%}0Nq zgE}lIdmYM3?@HGs4J^rov!)-&9%(gAW%KC$@XKO8J6B)~4+ZC&JFkr^8$Y(L__P8& zm(^Fg3aE5L!*9RD@X7wUu>pYaM@r4`AJmiEeP|c4y-lkS&vw@!vud37lXlKVoF=m9 z7>j&uR=}4&mDLd;C~54Z`j-4L7&7?OnUKx6bkBh7(*u>zmQVaDB!Vt{kkoVbw6$CC%^vYKE3LZot0(m%bt5 zb8p+zR5JM4dh^=CJ9nUIfh^*?D6?ymQTKWA3!~XT0zeId-@4r#A!KUvCO|f|1(XEW zo;4AEk2*JOjy?;#5iE&UIWq|MC4(m4Z6InH5r3;cxQGJ_>5y7wVknjb(M#q}lgXc^ z38La8Kj)vzyL4o)yy8f+(90yBk zHiA93BMlqBa2LQ7m-BDf9Guq{JB2lk@1L3SfiLjeBE81w7qi$)yf=1`$wJ1TDK(qb zjjL?4in!EZDkNGkxU&9Vy2-LE0Gzey&4C#E@~7*hq^u)M99J~19Co8};zJfQH$Pt)gS`&DF6~nO z_oBdPtjN|Ko71kkrKV$G?@LW>EoP;+wUt0n zLkcC?34g{B-OmRU|EEu%8bKbp`!-Um$nW%oKQ#m#-+`8mgW%Q5udxvw{EL4qG`5`H z?Z)f+7W5)52DRk8gA&*)r}CzFC0ynYHSSl@Qo@SorKQN$&tYY;HrL-5P%3C?AcwCK zz~i<%^=s`*k>l-D*+z(9=5p{9!E8a-#GK|wR9vG-8*A{^4^yR46yMZY0>XS{a~ zh$u=3RUI5WOzx(sa!H?uIM+Ok1T-L+d}cbTm!Xa*VidDzabPRSyo$}H_RiK4ofT39 zOb~o(IH=G0jS345))V#UUmXvZK8(5rQ(GY+%JokngsVYtGsFcdSeCXBKL@4Pbu}*< zn5CzM>~;2oPHNV`m=Zc>-d0q8Rm%A`2|=94a=*&oX5a>#{%O5dpylVRDYTC6ew%jJIF~fPz~OuS>0xb0 z9cgfXMSYbnxnX!e0nDSso^4RWk&DrEC>N$Oxat3UWpl0wE&x}(Fy|ofQ&-L`_spw9 z?Zu1|v9<#HD2%3Rr_{an2bvq`fY!6#IVYX1k%kRf#ab}wc?K%S2Ps0j6_q#$W7mtz zHA!WwV%%FCsj?Qt-Zgo8x+NuFfk&#BrfD+)_=1+<4AfwdRZN0NVuO?O`L3~tM?H4@!3D&3&i}NbT$;cZvGUo92#jhpdv=7E!61X-YadwUa6en;oQA9))MYWZ~J z_t$Zgr5BupOJ`?ZJA1P4FA^4GP0 z;;*Hva6#jBUH9iLBK(9hsf={sIqcCbBfPe2XfL9X-lYD>dNs5lZN?_piIZ`MFxXU)QzKEGeTRvOWL3 zo;i|7pB8t(*C$=jd+=rwNT?6aK^Xw^w6OyuG+9-qO?y;(qtyb$b{2is5vpIvdOKeH zbTsu_pLyr}`H5)RbznB^?Hr~CqNQDp9E_8)tt&@&(+j`WbgVlJR@_gRc)s&miDUE7 z4DZ{j?FYsDqj&22y)Oz6l=P+v}w(*?MDdWDxhY<@cA>}>XMUQA5&C(dKa;#`gLO1PrFN)LO!W(R9KHBf-n+}QV&H4o|Fmg-Wx|INS~-(}|} zJI8UV{pakHyCP*$M6iBpD%FC*DDSsl2?}GE1fmfuh!9k@&s4fK|C-%g)(kw{bM~2% zaairo_D)u|C?0_^AQZ}$_L}Ks{o13fEPcZQHEBJ*rBB_e-7nMebTQt#>?W!uwg3HU zHF}mI4uLQs*ec6*WeESFUE|0!vmrqnulX@Im)Nt`U_qzaWMAs&czj$wvv(@9RtYN1 zXPjU0{S`MeiSpzSMlD-JP|fts?b&GzIt-N=E|86vyUpk)-zny+tW<2827wtS|GqPi ziHXUZxVT_22WsdWZq7|_IvKYCssO^@-yc!kjv*owE(^GK|32Dr6s4i1MM5J*&Ls4; z7xk|`cz>-hgCHd;iUF&JfH{VaqA+Mm@~%kojsD`l`6#R?z4wQrHIiqTYCj`v3pn^` z(l5cN8AjN^?}h?f()R8jRw1T5B)5BcOq)00siqnl7yv*kYseu2W^H@@x#hRqO$N2s zhnw4AH7tvK_=$k4`If5Tc<7ut9WC*fHCrRinrmXBIJUVnO|!dQY?Wqej}G4j-olNwBe-;jIu-%8RyF3%=6Z zYXc!604ma$m^0Vk3&52$$rZ~53YqB^^5P-Gf0LDEwrN+C8;d-c`yn{Ef3ow&jVBpY z3O}DFyAplps<+dN1^b`i%Zm8=a0l$1D;D3zHUeO2_Br>sdv~id5{#sNU z0Ip=wrt^|gkp=|}O%J>peH9fS#&@LwjV1x|od3~ie$&wJ2m!5^Q?6xK_JZ(z_lVy! zrQ1#01LZ0K zWLG3!_gdr=^RWZt=f8v%J)Rt5(syZohNgl#I()6ctFH1^Jp)zn$YfWfUgxwex_PHu zxIkC&)|*BmnI7dU<=>&Vdmg9-Kqt($v4#d1SLXEj4QC0{HStUNrNs8j32E}-!H_dI zIWgI*IjMkUZ~}n_iEIhu{9$qpAQ3t@O;^q(=GQ*zK%B!;N1!Bf+m|vS*r9*B?o!4_j-d)IhHce5EI)wg5){S{#qQcZgDlWQiJA_WBKnv1;syYG3|On# zPsH~x*(ESw$SiUW2VV2XI2+NinBFA00x$-jaHHWYHi* z^r^B{2qAT_@aZ2Bp@}>E^5x5WV5v@ofJEW2?$d*#{ms2Dzmot419u$s07#$ZUaU!8 zg!~4;Me_^m!G@9~DknR8ZO}AoGKj%|vc)9I%&3Ei;qQEUK{=Fk?lg~5dIHK5?P_~T z>#JAfG3!u27673DY$!ClEdGt2leT}D5N#dw-&|RLAG5sEdki;a?yxk}Q2@lsQ-Wh# zAPAu}jQ3M~TBARhkQT`vjf>juxGinbu}B(|PK@nuUvc+hKIG>yyx$7WNO>(!RYn?Q zw5z^XITS(e1<}|GFzZ?YL&NV3hku5*;{1nW>rz{^NQ?GW*oUZa1-pcGv|4WWB@vSb1 z_=2TFVt9s&pYKa;VS0xJd<;$Ac8@b>$n@Nc)m@mD^6ON8%|c~S8kMWJKlrxWGXe}@ zfg_W%yD>W-kA;D?@-vLbUwkHS751Fr`0e9O47=gvWR;BkLs9g8qP6v7pfaI??utU^ zx99F5|3#Ce3x?EyrG``5XHygSlZ|5(jGv(c!{(Bh5ghb*DJ%AjLSNZieoz!InT^H) zy9HER#vv{I|7a~Zh+I_RElVA)nEk1Nf&$9^!x{E*LQ$Gw)74D^#3>krK&44Z%h8N3 z;480Ib~~_nL5!V?ZcnT3wlmL>`DGnD;6mEO6yW1Gjg8aRo`B>6Fqpt!)*yHls6V3i zInXX=l<++g8}mE$nX~}Yn93*Qh|pKBnohgMd_qKB#xc|N7lyv^IWVt=5tQkeGlw=4 zbN;^Rk2n`d`u5$?8gft-cKF{QfvF+-fjGe$?ALos@z2~E0tBN3H+ezrfe>gt^L8${K6n1U2j4@SgL5;@ zv58ke;AOo|IEBlt07Y$SpCz$pL#D+%=lgOgN|ZB06u1=KEjfIvy^FLxtStuiUH4bO z+QnlL_srQn(6WaehFs=IzUETHsEKL{jvECNkVBTCR(aPBnPOYG+7ce<~OY~? zKKa!f8c7sv5gzq29m z7%F_Ksmrm#{4s--h2GZU(J=fD-rQ7OAv`cz1l&LPTQ2p^&UKIlmq_*))4<#!-Q>d3 zk~%o<-}bMMLK;9J;?o|!83$50t3y;mjHS@r6KZ=6r82gIC0#T8D$AHwa2T6a|-S6=4qaX#2#{Ta@ecX z0&Fgy!9V=Rj~}5C5kbYp+jU!WkdFd%T2({CsDuPYG4WELOmKxI+t}QsVqkDn@*V%@ z+3_3vM!rRR7#+`mPkJOV+L1Z^uqO&dKDo3_3K=hxvGKX(zXTopu5V8&aKC0cx)HT8 z*4%I~hJ61Y?~VoE*6Zhh^YZB4EQWlb(j`aRTgqLvUQ+7cT$GEeN(*ZqlFmj)Pk#mh z+U$k{GD$~_=SB4^!iWi1j|;woRTP<>2KWdNaBd~K^8BlF6r6Wmd@q77EZy}%;v4_p zzmvXx&6abum-*(jnU^mrBtw29GwO7x^)DwFzNMA{j7FIE_dTW?34-yk@XN}MW2h%z zlLm4v0nK<|41XZastq6DX{YT6_5^dVMf_n ze7ewDV{a&)RiK*xv1tx|{N+PAM|~aAaVrmT9N#>M-5vc$PXc!UrvGt_`TyH5QCN+_ z<5;o-o8{~r8yqmh;(v2*bsQ}L#{Xc->#u-I;Wkh${ec|W*C$7L<`S*)n|919yzBpb zeT{)&dgz6<0O6r-AFdFLaR#Y*nwHv)9 zE%;}w*+_6=Bi5N4Vi=Z|bYUZd82}fsDgp?$Z`CiXiImfwnY3WFaa#ylIUW7=>k7C< zz%gU2R_dysOngq%ehT(MO|2AU?%`5Ul-ox&e^Aa;g1qp1_uD5YIADzn;^KMR4VM)? z3INwlMB%>lC{eO>0fy=f&OI}6M8o)arWz_vOI1XMJ)PEU^E^C7_l{7T``-nKj zc)6V_#3@84rUcx$tEFmKfRB)p*`?Zw3b%D)mZL1(&cwTOx81(ZrmFEOmJd#5>9*zojHa5W5&8h$TMS4&^ibD6Cv%PcIU z9e8~G+4^&0=Zx@st4Cee?{HHmm37tX_!rcV^2_+`p4R@!ng%D@nHhR`twNv2D}cb_ zpObR~uJXG)<3ytZV5_qJokk_JOW;XG_oN7LmcXRtwy`lMVj{_*PVUK*Cpp_^Jl)UC=XX(U&elADa_>NKVeR_Y=9SLz=UGL#y2lp7VUGU5Mkh=k56NfyJ%8laSg`rsjHCy{O^g z1ACx_vx8g$tAdlP9kodHF{ffdZ|AMDz=C$D-KO7J9T%wUulMs)*V6h_SV*Z9$sz}r z7bMXQO|HWgww_~nOiV{d2jyRapfT&gfJt}&YFg(?>FP%vvv)6Ppt`^*JcFnZO)IwKt1@Y@5m?E?*Q7NXaUE#(qsT#dGm$SEvTC(Qw`heF2_p};eTc!*c8V)qVEm^& zY1nwtCShEC)!@>}5*mp$8G%9xto5jRL>aHY&+VADDV2ceE*RKSL4oR@ReMue8RzB8 zBTD98ttVQUy`1$Q&+PAC0DBWc;tfS@%a6$8fq@)|-+Y;rM8d+#icn+p0Dej&IXNK( zMPPlZS(jR}GDYmK3RZ-WWeL4TiO+#M*2De5<8On(jYC63D*TB<*)A`WlSx&se(0Gy zECci4Yu9f$G=eV>l+4Bs4#)F~dG?e!Y7q0O8=**fW~5HvYMo7&`V3B%)zBqAzL4;6 zTq>V{yp^0A>0GzgThLuN8EqE_{K`L#Zo5dpSmk=zm-P+DW*BsqY0Eg@VF@O;@;R&% zhrvyap5o8WwvfyaNv;F@(j~82yH;a_KXewn0(>v!dsrS2;O|X!h{lz9y(VLJYc!%X za&RCBQ^1Y5jaoTCsAgYPGS_5dVipGawj8@d``B-IW{%IFFFVREGOP7C{l zgYeo(ZUCPl+cA+{bDaE4b6kXXjWi@z4}H0opt7*v_KFPl>(U?Jg?Is|3wbqj5r_vV zQs)RL*p?lcWA6o(k9p=a+krXn$;oa#9s*WO2q+kur$imM7J3HD?2;Z%_gR;UIkmVO zqebiYBN(ASfLX#C7)(uI4er3++uy7`u5q*$?em}7QKG3KapfGGLgV zZZ}UaEf4wOZf{$wYYwF6DYtH0f}ONq5<8{DoSD zyZBkKO>{ph3MtxX?=V3u%q)ZsCe&TJdKH^wr253O8d>pyPTAeJ%{1?#gZo`JQ*adYtJGhVk$&!_(~>j!sUEKiadV zYy_^pCy}f!!`si_vpVE_a=(q5w<2lJ-OHqHYRJ*GOllS6U5#xwT0ZV#Pd?=Mg~bt# zWm#t?#9ERTLv!;e(!>K^8K{#{OGhA~6e_Q>+z+Y7yZ)dy9LMOOq;orvL-riCf58j)o866K< zq=I@%K1~&?ySKAUw}8Q4UR+nQBtxqA29eQH{c-oL z`ejYMzt_35+i2>IoF3`R(&x3jrJnCKkPom+UpMgn@HWw`XnADcM$C8VB_sXtP}{@= z2La-9MtO=if?+aq&rgYq7=BI6OUtV<{-n6sl+@= zMf!>=LmSgL+=lJ`9E`2#xj8QUDYMyF?V=?wV2Zk0ryjT-@W#J;H(CAA>wBRtSM)0k z=e>Kssz&Y8Z{%k(^16I^d=3;6wqwc#1JHeno);}pJvtJXlnkR=4Ek~JN!45YgacVo zt5w_xVGU%dnB(461cz8R1zZNh7=3+h^;a5(Cmg;RU?8KbuN~cv;fzN$qYlT=#)rK- z%~M`pCXHLJvK9)`#+YH?RSTUc*KBr5f=rIp-s%71?X82d+`ISD2c-i$vYeoAwl&Fy)Zf{;larzIQKbD1G4XZ8YY7RnlGwio z@ON79Q`sPrlVh9J;p5xH9_G+U?=u~bop43&EM~NLDaKge??0I9-=?0eA6C7Ql{L@{ z?fiN&bP#yGsaTl2*l_Om<|4lA8SLE(L)r==|Z$m$0qN(&7%&xZIU-$dD8Q50aksxHTcEsNFs#h zzzDP%2Vi6_j=J%-0y)|FqO!7WN`A;?k&!8qR#(5KL1D4CO2ByVmHlwOGDmyR#xS1` z|5pN?d?IaJRRBBz?xT9k`nTIR`xl=w*Au`?do9xWLR}qcqKZp=%M;~1IXlFn$$?9Y z8S`WN+E#PpGJZ$4oOV37ok(_=2b>)jXt4OMs2IDopL+? zl)T%cZr$<{1%+4sl$QgoeIZ2#iW(=vOEtaNmyen9MvOHX!bfLP7Z)3b%buH3#Nmcg zwRH<5_bPiIlzz|5y-R_Ceqw7pet`K7O3C4vwrX3iS6QFhae@Uzmd$h*7n0M`u%RcR zOiM6p5K)0C8MqETzJt;HO8e|RRkL152)5%%yVB9PUski-MseCtD*~U}U@*XC{VQ|Y zLU(@VPqf&F^W z-7HT}r$w5`gQacaAZItj~@~TP%VnsY+vXm9ag6z+Zq}h1CcU;)Jww^n_L6PhCv?S_B=i`@MomZYc`vEZJ^(Maqg2$cx7Jp`s{BO)c1(@z1 zN)i9Gw8?#zu76r3xpx+xDJ*|nGNOP_O&E}ie@#{|G-XtDz z)|Oe_Y&%lAcH^39N%k^ZNoGTu+x~-v9RB!H)|rRWQYNFfoL@-vHTe-V^VY}xr~E<$gnce2S9 zUg78WWYS!rn0Q;VGTd-Lw!Lk{_2zq?!VIMDyq4%L{X$saFOe(&!2^PKT2Yc(|J9hrO|)sf|^ewKou7*i7(SLpLX= za}h`nz3S~&676Ta7E|2xiunj1AD<&6vEFQ~82gaL<-inNQfHAyde?8SL|WdmvaZI|4!@(%gUWG;n)6*w~oc@NjWGDh8JL z%7((i&>&G(a7<>WnVgEsw7!&ej{Nw%rkuG2kRdS~92Dr4xZA9wC&+;~Uk>~a5Vn?7 zqDV@HXp`XizclV`OlAPN>g-7#qsuM$Dt?FXO)}3omk%cFV6cQ2fFTaTxSXrntM%L9-4tWRwm``#(j1Yx`=HZ=Af~xFKqN$Dx(FV*wCDGN>O9UAA)2B~M({+5h ztVFyZ>p^(>3AZs+{MVYjht}-LugCVCS?S|4s;1)ky}_R07uyxshg%uIRiItI29YOA-; zl+~t)u&uei4{&}bA>k@fQ#D0}TQ!@}#&YQ-XC0RMK-K^*YQxHo*VQDerU7&zC3!CZ z#*dEo1+13M2yb2iJYa2o{PLPTo6l}*fRW4NwhyJ7LgBZyH8CS?x{~Q_zK7`0MzkzCtW+PQw6-pA5q*8#XjBEU)U$W8GG$$HAk z5f+esWNm@}&B8H%aDV4UfYq)%p62zCn0Epg7|&B=h)@MWlgLd!_5S&jA%;L?y!C~_ zidMg_15av>`F)n&nexc6YXUuU0E94a7Wh@UW%IgywYb*Z)!ycLk#jZZO3~1*DImQ% zEku;r?bDT)7i;VfH&1q=!QclZeWur4VgN_H&q~>xX2+FmF|GMILIFCe;8r1ttdB$7 z{5uD%wU<9W;Ht9h?{@wADR^0aa5d%ok2kSAQ8&MI*>w0n?quS1baDnRQrm2jxERnw zeeNyc{T{}6kI|x7Oz?m>?)#N+#yWxGg{W>@{5W`%>Nvr4VF!9mEMkX-OK7{CTad3Il;;E&GI_WVzpEswwyUPO$LSsp7~37$&oOQv#|qOb(K|TEeS^u>-Mez2 zz7pgQ4EQ+_^Gc0;di}c2@5Os%7I)==`)ON=QNpmczLwa(EUhlBXLo#LMPzS0qYPMeKB1gMz3z5VQo1c4$p#q)8x ze979c_jyB>39uCu3{bzGl&IJ&Eh%BX+g1&c{7|N7KZ~^Uy>jx(XU5;>38xdgvpgWm z;86BqnP4hMA(VOoFo(a5JUGXsSvLXrn%N&6PswKk32>)yB<+vF`5M#a+rV1 zrK{*qFC!mNQzO#IN2=`C8zIt?zLdvlhZ=DYvJcJ8JL{^zSR5D#Fk~{jYpr5JCZnOj zbTCiOVIJF3`|p6-@ZLB?zjs2kkA>6v9j_Hv6w;&T38s_tzrR&g2XwMq!Qz$*3IM(@ z>I^$RZqBrKHq0t$Tz)uvr;YH0j-lpMO1o`?YIdiqwS(NebwwU<#=brq6vVrGzZ+Zb zxE9WG5#HpX_+I(7@l8Y}N^~0j9pYel)s6LeuVxm(mPNkxXF+Zll8L3IWlOZ`@E_ysO%iou`R{%#J%Vk>42qZe1e1WanE_x7UIjP`W^Pobk_)DUS} z%;Mlkm}XQZK$h8;(RIU>oDoNr$0JDaOY{18cf43BW+;=o=qvrwZHrzG`q!>%iIy)6 zEj`PpFWoO76n3>`*M6-T?QFei@W;Czto+}I@(xx`)PIH^HHx3^k-*>q`{VD><0f^;tkW{rl*)wv5TqcBPbMwo< z^?0WRBBUWNaP{x<)gkg-4$Z~mI3d4?+~*!<&+2};?eZ}G=&0|KfA?vpoJ>vP&vxSA_8*@G+j z`jesBcTF6yDA*ClY4U_IrE6X=!-k#nejuJRq(V#ZRA&=K$FgGPR84z+w_oblEfse0 zuU`tTl-s1}YGH^!=(hobG0<0bf^o(8%#Ijr%b*Qn!WrQR085)u@o3WUu6T~kZ39^6i`+Dn-!@S=gDu9=!I)UgFRf&B97v&ApDyV za6KhazF^Vc!#z579xv|BjU6_5q%_M(!!UNBT4OJ`HagkoadDb+;Fy(YT#+` z2vdIg9ihSF#d1xs+5N}6ZvEtYZWNoFJ&Y-Qan?f#B7c=AdW;6%=;dR7;o2>sRRSZZco+TBP84ew zU#w-Oc!e)+>3hy(aQq;V_}WOXh%AoeAEqz*<2BvYG4?DjNYKBO1TGI;R4Qpr-qz_^ z^{72~YJ0iTpN<3WHdF`ojP&#`B_*-%Tu0OIiuLz6o1?I_G&CQ+gNy6Av}B0z%*)Gr z=9MJ=pb7ZnEKs4=-+9a7Ggweia0HrxK&2x6Ngs#Te{bwVDhL#LI!JRuG(Sn#Sh+vm z@^g$JAz|8_Lc`|Wy>H+0@367ieF(zz3J*`lQn3?Hm7%Kpc1I=yl&@YnFFLpqE7S0O zd|L1IPIr?)SdCe(ibDDeXr=7z=)Ap;nzG)<4+7frqmzOEb$(roci;k?V!C>ZP4L@! zdCd-W*%ajEy$Z)8wwC9NcG(xk@$W|6ryj|hJUp?<$jyBvy>0em-X0U@M)7gdOF22L z9rL>o8!(=fP{w9#clCPUXUJ$>A|W5hv~X0o_RiD8y&8YT7yab{Rs$6dA0kK6&(~K7 z^Bp3!@XB!RG{1i#DFT=8Ve87|4W86oNgp_uk>VQV%cUu1-^icSj|Go2e^&7XovI&o zG5_$}`{j^F3zK9@K0b@7T!{^9H$g?!~@v6uY+Iy9Im1e9GK6%KmJL;wHY z!yQJJ$Tpw*oH~oa!~ABpi{BLSf9WgdGWmXN=ak5k8SVvL-6`*TO9gPEt&O?d1R$62 z=B=ZH?RqgWf{AslTl=CCw{=CSZ(+XEEMYB5*(-G7qVN_5=z_ zP0lplSi@d-Fwj1jVPMdaqm)*mqztMVL=y<@mN%`h*d2Z&t%UVuY@C!g%-e3Y>}s0& z-1(+up5=e!i%JgwSI#LTBINj?w*W>CbS*F=dhZv}j_&`%a8(M!ieX-;Y8CZppGD%! zXo-kOhRevU6o>zn=90t}$RMyp(B9UT)gOL@H`%?=6&-Yco8_6?!gbM(pD;DW%y@ep z&S!w_??E=y)f3@K_zC|JIC_~#ky$lY5HI(z1YS;mtm3)(zdlxIf#3fgc=LZ|T_Ce$ zl3Y9mgGjwDcdGsudxEkunHv3lgJl`U-#D6oWo`5Sh}HT3h}-f1fKJk|Vg5G~hci++ zuWnqhJnwi^Y=$8U-uqvp8ocvaWF8S^B?h=rw`fEJJGc-Y=1e%2Rq>$yYnM*1{C~p& z`JYslcaxLkh+g2MXQCs1&(7ZD;u`(mm>RumZS`N5u<7I?=avd=fQsMRd@Y$i^8B=D zAK@uYg|0P5g)^0k_uqR4TNrYn%_x`KpQAIXilZQIE>ZYEer!qA#R9|W8-?^Mh+kjW zu>ozJEd+2eF)8W#GKV$zTD~tIsm&>cVK&bNxpQD);dSsAgQx_4#~H% zu)gsnVCLoLBOotsZHU(*N`Z2&RpKeTAujIk_gnekH2>V*h>tS|OcnrvRlE2Fx~UgA z#-4$JPhgj@nM;Nlv;NLTMC%+;vF78Vxv_VKw-M|Yi-XcO`8Z6^TBDhc0QcVR|efCcfbIJ6o za?+2IlA>3pyZ|z(Mf~o9=lrJK%TJ=Ljz*l8_B*{66O%n_YRKZ1pWcT1VcooLVU;5B zZRAyiFAkY!bbp(Ji+GLyJd|0(IHh||;s<@JwD&ptX zR!T2n9PV_+dCrOTlj3#MAt*jAF1Ph!4%6j1?!glV&W>+b7%QUww-<|}>4Bh`c{13W zhpTvYDD-`FpL+lp9!zA9)^}*)oSF7IqN1byY2zJVxVq0Yq6N-FSK>$EUZ0-eemz1| zuK<`SH}Yx@h|{DJ%(qXHLwSvNx68~Y1A2OT?1xzhZz37raPjsV--IjF5sCGYk(uqX z{5wc11HIU!xnLAr9-$G`^AVBN5#FU{=5XqBBW(^Qa9amE{JLQNXL{d3W;BVqr;qDKQ0L9&rUe+wgkL%`2Z;WJ@sb~SQ=5)k z5$=+^j4Yh5kJ#Ob%1sIAo}5rcAmCQVMddjl<3JA~RE@ok2;kUPRNE)!iloLi* zR~I2)tqDOJaB0TWrjSYCKn^}V+$3oG`0=A>U!M%fBPfCB&Q!g11#AJ(rMfpJ_5K~T zx{m>^Cwt8qHKDI*rmkid>{sWLwsu2+^86S!X6hT^n&Lmh~!W(7c&VD@FWXno4>4{N8eN@XcG-n!6t(0C{6K zTb!T47GSb9CaJv4Rw=_?US2kXIKFvsO?o9Gb4%T2osAnfNibytN46`~%T zf6Va|X=!K#^2#p8mEq}v;oubnv}@Q*OnLTgU&6vDeFu@G8>j#umEuD7GtQLzsTtst zz{@5^VwY8x`2Wgw&QMsPB)Y6z5S;;4!@CAe&y-iUBICl5psxQ*I$uKuqrcn_LCR0{ zZRzK~5ePcE%y0e6Gi%v~Pq=fk9=%@p@xQnLvI40x;E9$bMIlvioe097)PU+{Q6k~= z>h2zA%^+;8drwyJKmW9bV;0iR?mwl@7dwvz5AuKXjNPJ(Dzf!lNOiV68=ygWM){G% zJ2{uO>?8`#GTXjZtvZE@4!o`et*nRu7tT2lnkt=R)-)#A{wWqne`UBHp;3L@+GihH zpf$$`5X##6`n^xDfh0AqMv-=n<4Ms@&&YQH&pMnFN0kE(dpo<0w>SoXQ7ere1Bul$ zIhiyu5e4(+md{(dlFHh9n6D)|gp7=y$Hv`-A`|@laiBg0Futj`S0LLF-E$S%uer$E z-XRnmx1Mv{Kp>9M0HYb+p1iWLk?Ux`oeXo#)`3U<_#KTeuHxPK{T^mZh9#UZuQfG3 zPGY^`$HOzBX)#@M>D{i&y1jcH;c>C*VK%lgYOwGjluxNZ;OqjA4blXg1MWUXAPMuW zpqtLgJ!~bh(7ZT1{t#qp9!>q}HM8x~8?Yds1swXvuU{{9bf`#4NfG^vr8%{I8m`-D zh;N{~MtbgWk}ht|G;Qa3oZpa?lq6*Px#PlTxAN9?r=1o?p06b(&21?Dl{7rT$VE|I zGbt@Ee+|qDiuYpWdA@YVag<)Yc5SC6=iRNWQ?Pu$!>G`hCAUPg$`-^X6lzkz*1r4rZPUr}6@)hd}3z5A#0^z7@O({fs z6F+KxioU+J(?uCA|CaxqWTBhQO*p|7$+yk*zMNcMsoyh-tD3KYZ(8M8<9{V5k)_w< zyVn+EGGMyDUHEJMXvnx#JOp-9k0U42NQd*=|*?w z?Wkytk5*67bR@oJ_G%gHXl6lXpEs&X9T&tN6W+NKaV5=0Urcjy;5Heo{4onVe%ms` z5@jTP06gQ49L1wqQ#wFC{Q6WKdBhh)2MUsC}rKX*~l5Gj{N1U}aI80Ef+nW}zVAdWpG>4^I43LzAcs zgM%DWn2Q@5V^nkZR*WE1>Xm-lcG-HI!<3ig@BwwA8H^1Y3Uu0#aiG20c6KVj+k^yh z1X5vjt+C0)z6(`DW14)D;%}QE?E>D81L$y%eWF4KJ_+uzPbx41wPS(?icK-!k+VjV zH8x#wS^JVKgpGf-<#GEmjJrp)k6Xk#Z;zCy4k-=i8U);58t}W_at^zoNjU8H%#6M) zN%)3grt850`$0A%J>4r&WA>jNk7rirg5A#>88Th8u8%1Iu>zdKF+V@kCxl=M6fQ0} zg`MG6M&)(m#v|0csR(@{7`?q8b|x^K1i1`PgykTdI-bNVSR#$x{&$XvD#4+TU-h{C!w zUO73@0LX_6C(YK}fgr-ON7EyQF463h>!4!6BT%H6%q@zi8sk!U)H&gZelANKCV?8? z`{O-6YF=hilnY)SR^FssWF{XB8&D7)M@eSfq4za?lvi>?%4)}$ma_mAdNO(zSO+|C4yPcwJR_sYwVJQ z7a9r$U-O&8GanPM+IFAmDPcL5Q)q5|O0g+pt(^6S^SF}bXv@~YN0{TWcxRRz)d{;o zr)a8-xVxcv*@0OW-~*7<*eDV^lGh%b%^k~abFJX~M0;kgzj?SjRs#R=m(w<(--#m0 zEqV}pY_lj{`bMKonSy$y)X)d-e`i5#ywy}QxNfA!$`xpA1{D}LuSI4{RIC5A%jnfw0E0rFWb$W&d(Tl+-C;&I^`Q`QN@ zW{7&DS0{ODh^8h)sdfDNq zl0<`zx%z!;bNqVfu28I`33-d_6g%VHsR-JnRB_8H;;jW;xm|_?$;gw9l6XM`1P)jL zu?pJiJ`}>Hz+}SVK=3>1u>qCuF>+w_{=IM*0`bXBkl9ePIC!&-5Y^A8IJeTmS;9=7 z#x$_p-3=H9?3Hu!a-0}A z>+W-s^#j-SPl~PBElBY3F@3_Rpk4dIv8Z)6Q`W6>{8;L~R?2o`NW(qNa|bSP7oJ|X zx@9NdA%Cw_{exG=*9w{(`M}J~5ZXOcQ`6?Dn%3#OF7H3(P4Jp zTf#2;>x4zQ@+dq1!Akv?-tF@Eul|KQq65QOALF%RAHJFS^8soeaHf$W;xT5$Hu#&#%nrlH(S2rz1K7t=GIbZ4UUng zB`J<--^z-@SIQojn@RL?Tv#5}b_skGPEpUtt9})>(Of#CR&0iHdDk6&j{SFV40{;B z53TOxgclnZGp?L{W?fV;Tb6$ZE?EGlA>>I7u9%VW@tJL>(2+CpYHn2k--FK@hr%t{ zYw&Na(bnpBIFUWqzSpHu8S+=9Wii)(mju|$!Pjs?UaFdU59%O`dRx=FMKdup9~y|i z!bb3m+dk8mOz=c{o>^OS!lZU!j})}hIF8H5ZD!L~FtY&2B3JZfGer&(z;_ynyfx*%5KTDWMb)m|~H2ZrOwNprL@FyLf z3d09o57b&d&50n>TH!l?_ZX2rkMvH*?|kW~h^uK_(Nnln3fbuE)|w1uWquIYR0S|O zeQEW)>c7^`To;)lB9LS586Hlx=d6QR*UAjpc-`;q4&@Z^E(!-LNLXwU_8ZTyBOcPf zrR1x7X^NIKOp>p^E+4ob^Wceit^HLrG#bpCdo629;RwX`cEK~(r`HAtOCa+MQ@`^I z0_f4}T%jKpnHbq+CZZS?2Z0ivS5ryI5_4cFz0GnlrUpMd}!Fs%Bv0d z_U+!b9jIP4G&GJ*PAFMfZy{^v%k$|=t^PELkwU#2Al(?zkJ$Z0C5%7-q0iE4`OIND z7OXr@H|s)S-hBmeap-YL0CL6UA>MzTJSTMWpRi^zT7KL1-BqoQrSyH5T=jfetE!EJ z^nKA{7$6yf$-OUfK~tC5vt zs@$}#Aw`zSN=ldCo)1~_Yt__I1Ppoo-hK1ivG*A~U)kpb_Qf~#N$wB~3c`@p;U|f} z&FE-1DvyoTFOY!AOW?<$p5>Q!_-R~l{%fRHkRh_7{9*oDi_g*ObR9%DHSK>E7boP)>`(BwS-5DZ%-wPMfPlrzHZ^BO?8_+w}c*2bE?`2-^)(Sx`I?J zim@e4A_jpVU{W`Ik`L2H)*ivZL8KRv{TlO>(e_@yl+MJnag~FA_~rf`xaQdb0SfIG zQD~kjpPC}Kv{_ZBQE|B?gw;A~(~f}7EZUIS#~&6wFi+@*N{UP_h{uU%m7iGhKShTJ zJDQ@q#^rb&Vfp+yS@;9$`q9zl5OLk%G7DxH+DR*<3+W`M$@Pp5kfS21C2nK;q8=xM zo@0L>>rKW0*@0Y0z|FhedZ0YhKJihZ=cpT;?C1yOA@91Rq0{dznByVSigvpZF=!V@ewx~0G*Wbls+k#kIT(1u z%y_X+Nn$!YeyK!afQZ6v56_l!H9T6q!WxZ)-4uavJ(}_J%s_iB3}hK_?S%Z^u{Avg z%qu`}MMHcDUY{bnktLtgSZTM4K=h|ewSmV?RSDm}HY69wth!f$vH-mLfE-GpQrY>5 z+uRdsGxIaP0hz?q8WMi;<4>a#>R$0>w>B78=1bM^9mJW?tF?awQ_V&&-TtOZvSY)QKYYP?tIpeq-qYdM<2o>uq`L)nP2kYjxA{Q+4}1{;^C!FUfQu?Wqjp2VEU ztNECPT*i~qva;_%6$tU_-{hoL@@h}6!<)QeW5>X{P!(ZnjBpUeK38hUCxpc z`8tJ=9cr*c+E0Xa?q+hYaGpqHf6@5Shu`9D;T#}&<@Amo#& zb|MI1jI7gX5JT`@{2@VlK&^GqV_pe~E5C>C0q*2+&QdVdo2^3Zq#4wcNNe z?m+g+Ub4338JLqZ$p^Nz*s2nZQ3E#1XKrC0%4~At_>|rFqsE(w6%ARHFEGVFTb(x9 zKF4v%H}1rlEmiU0^jw>8MP`tYa#^7OS$MWEJe>*Luc_htszzsHj+MN1O3(5@#joAfR2RFYR58spA?tcr_3?n5FPHl>%tCkV~N3yUNMN zcGseM+xLk3{{4m(-p@Z50zZ?J=P>> z&9ku`4=Y$1^Bs$4c*i=p4y zHie!R%%{J9^X&PzDf*i@*gLn!`+>bTunu zixy`0Lmp*Uq1JpzN+vBQ`}l%^ds2;s(W+cx@kOeP&hC&Ke%^WU+?$xyx{FnV-wOwq zWe(H5(u2pBZ5mIOuO(7{^tOGm`R#+J%**_5ceGFhTHX~lR=O*c9iMMl#Ec(hwlTO5 zmPDZyWHSa-GSX2}=GvzCgrUBsxqrLwf!^3=XmmxpZrDfud+$$OW5aFCH_j(q_zt^t zpH*)dOeHXmRn^`9!bC^M64|E8DX#RqqHuI=tlTJz-Fz(b+c(YUaguXmzZUK>ROqGQ z)pp3rw3%Es&`ZSi;hymv{~}Jnc&HxjXA{(ZpJq9VPo>uqYtzkmK+Yn2`Sda}#^P;V zZ3crn6SK(>z1e8tE1-}FWqbs$0+FzmzBgQ#9v6!qcSy$17GKTPSE`hmcb0xiF&i%> z>HtAwt^0nKhDezG#zY1fh);|}CM4vc-N1k95_EBXYSQ_YZN2;j0{-`mD<~gb8F%8( zYEMJMM*TAewu1JPZ$+FJBOOA7$yzoc^T%d5H*UyTYgcc(T-H9~Ws4)J(jP-jkp8jb z?I4J8IJo12Pe;i;ke;Vj`ErF8XDVYx5aWlbeO}Sf0I`-`-U#_g-sWWHhYwptm8C-? zG2;1z{mC)%^i@h$@rYq_hwrsh55n8Cy}i|yr^IXK%+-?jC#wh^W;n$5dRQouk&>p6 zw^gAs%AvyZsC3Ki=2eg)OOJk% zMtE*n5gUJv_Pt$ywow44#RZ#$n8biv2f-uu(kk%>MVXn|bi8`fZZ#76HXqRsq=pY8 znap=xlD7Xun1Oxu#%9lbw_v-e-ou@NA@{9|akhMq%58VcXKO-=lgTQ@8xP#93{l`B+S}Wm!N;}4bd(i~X!c?#1l*t+R|MXpFN3r55O}$(a0|_8 zR+ZE{|N6qxx)Vh7BuLMq{Zpx-d7tv%mq;Pb7?tIgam5o`R>Pvd;g_l1Ciyx+aa=GQ zsdw??WPqF07lS&I^N1<$t$>z4C1vC1uvkrRKZ37S6zO*$9Bi++nf;Ke?sp1(S9iaBsUT9Rp9!$jFB12a6i@9B#HOaN7SIMw&+`HMz*geGCIRUMLlS46+?fVjDW^vXbcZ-*6 zWE9_0#&qTmd=9Mg-M=Ky8{7<7<*XvPEU*@`moqio-E0d1m>>&m@YXnbRjSZXQmgb7=Rd{BYhNwe`9u#cSUwtZ@XJP;&^MP3qJ4x}-@Obe~0RiNZB7a76 zHSXPahr5TN+>haJ{FCCyP?LOIPweGiUuhG!Sx=07`oCLoNf1jQG}v(NfTjF}KR+WW zJU!?ZezGuD?YDE}--bYx6XGgf<;VE(<8>srT~>nY37gqusX}C-b^8;P8Q<>Op~qQM z;nY6fs3t#t*XEGp{QaiieA1Q%4hTdo)=3Jj$x#o=Sk+}d)YSXk`x>uT&&T&eHZxT4 zCr7l+^#A^T&r4n8PbC1Pjx?Cp8+RyF*%hX|Eh({Mjp+Xo41-s%9a zWSnb7Hd~3=*J8YXf7=VHd~{Fj&%Ly-h}>l0fie&~9RKLrxbpYE@&|s`!utCwczBp_ zXdr%5vT%PHTSR^hz>x5cd22g++W&qB;u|&Qf4{tU4}S9hetqSN7u)|YUq*GX$3P(J zqj}^%C%dYs892wW#Xv2Qx&Y5%}*D zB2c{kyff|f9K2ZxT>z!dlY8UWv$s)!FB^{m$A{wYp=ndBQqmI&eAf! z($;@%ZKL?-ymdx(wM^JRi3ve;C--gCf1d*9ipjUzJeND;bg*N^#U0!{j%T|zLPEcO zRdy?!nYQMP`^s(?o0P;!OMfxkreQoB|2`~5hPvTtdFe zqN;y?tfln*e00wG3|T$p!-iJ&lA7Ivn6se@(vaPrESn;$e)jbVH~yU<()cHeePa$sMjrZnor>7K;zU*nSDQCBQ^ zs$YLxE={?#FYfIJ*8_rn;QUO6P=q;Bbt2Y<)kV;Lc2Ze`>rDyL|a+{wdg# zT$G;hka0=>E;HMzzu6i0M6ID=@k04ooMcVWfzCSLm-#Y=2WeFs)n*|YnpKN2g347J zvULZt57g7z!x_02c}wJkr<37@lX1F=YgkNYB_-Vw>4WOguHW<$<7tBXK?%#)rMPq3 z;Xq*(L2(5ICcS)?oX1Ij!d@5G=8+j7%pb{}m+oVcu>Vpl1B>Z%5SE$u7a08dk_e29 z5d$?i6T#FV1LkI@B^toljAT@E9!h9JiW2rAhEF7n+;vNUL7h}_dI7oiJ<~9=y|5$0 zK7Y$qt12$%!EH1ap?mZweBRcFF7qn|3n$Et?-JuvM)jw5w1qrd+`W4@VyPl(>QnzR zxf3ZdaS~n=`gh^OT)Kz!y`Qq@GV{uepcYC^qc&dCYwA9ZC5SPIcad#Lrw?1a&`qJu z`u6=Lh!!{CFkl;(VDOY7Y-#_M%lhI}wdgblcp`u?S-c%+W-ap7+Uvz?NVTwim0&3Mg?3Cg88LC6 zue`amG@hAxV0~CjOcvPfOC28{howNB1&>aAyv!nM&{$4iKTSScF%hzZz_(i@><&Ku z(83%CnVMOFqAet9&;c((sH#AO>Q9E@&K_s6*$MEFc2#6R@K5HJ2uN;1MqnA3xChgrH zR?qnHqXGwU>k#%KXcb|6`pLj<-p#{NtlyHP{6iQB>P;t3PA2=qTE7dt9< z$jE5mrmN#F5tWmZHyf)K1Z}dh!Z#Oo&J*%zahbQt_Jc!T<;OJ~IwLAiGOZZQJ?3&e z=pNH5Kt-hAz4ql6XZe=%!CD58V_yjg8PETW_{yyHj=EjHXnB8FV;E`MsFWjLbG#7a z{M!qIS_BlR!;l`R%)I~}3K`I#J$~>&$YIhhs=E|eM2(nPNDGwJuO_+D;DHaj3UWzO z;pvJ8;xiZ@+I^0QSXQVzk*flsB&^UbAx$jQXH}EA6V;US3-~|;04;{3QHFnxI}bNV z+rd-=7;^Gpl=nIEtKnxy%L>&(Ws|qc$_vMnDYLUOI@9KpNx_xPG&J}StGd6`D{=pR zr-v%6Ix;@#Z`o_A`%6<2L~dbcEB8G1_sQiHs+#J7+s#mWEL8Vtp`k%Qxp>Rhrd$e} ztdleG>|OXu%%`Ni;uL(O^H64Psd~fxhJbDFl3)mn&qzuVg>MM(U!GcH0Usw@<|it8 zdh;N_k`qe!V?!5LXmKg6bDa4|$E{Q%-Zr{0bY3LPdRNl;x97sDtAdgWHa+&9)Y@M4=(&!og^0 zXv@Ml6GmqjtJ6Got!~OETtAqeWn{;yiuCOdtB#stnW~aPE4&v~clHX~Pq4;$pLNL3 z+uIQ670w$Ki@GJscE9UUbYj|djSJZKAV8|vxx|HaNaJjlU}daiW_KIrWXfgcan#FR z800-`qgWo{`bA1&k@US`G-6#Voa9b_@L=(zrlrx*(~E=U&GW1|izWZtQMXMhj9iyD0 z;&<|s-bwfI>9bSepU>m-dU|lr4?{P%$1g@(b5687Bipr)wS*eCc&qb2?E5CGr54l` zp=^el6bWWLE(dRPUzS6eH&(Igsi9fpB9%4;f5Nnz@k;Kvg^Z~6=97bY)s5ccg9EOO zv8EcA!@e(Ke(GDh$vP!ZA8CUy|JJQrpX1^f(u?*FONQq{`4~mMKDSBBt=9ST_>SpDzIWJq7oFBL>TVQF~y?&h^c6;tF9yz1PPu|G%bBqu=PR_n(AyMtR z?C#ac-ee(>R0@7giHquvV$a)lgPB?B=o$&#zG^6D$5l;+_m7&|E75Q8z7p~#nWyD^ z*SUZ_UZZ9A=1sm;Lad}8E6$?fpvmf(o2p;({5|8<(wecodHcD__wU~a;VomZp3}0J zWzi`J7}O1V(=%QdS689VAP)ZiLgcO`6oR2m`;J#hlc6>iJ`nxrl=;T2NW!0 zd!4V1Wb1zZ{7ECSz0J}&q;YtVUMS)pomb|b>@Yp8Ig$P+#rH5t!O-vnWb*`LkQi8( z9xl0#m%1fDIZq>UV^PgPpgZ`+YEPn|&~Zp>IAiG_@Ph1F=!~HeIh^*Wo3`I91vBK* zENpuddi+4?>$(SPqr_bH*&zDh`~#dH1j6z@b0gd;7*C4y3O=9aga!bo^8WdlZBRUq z%ftJO^85CF3%{y=5(4>hBB52x`SFtOTR4^lIX39GkxT43?I>%|GzFq!4 zFMCYcNG6|msT){(;#@|(JoM_xdDn37;nItvKSg=e-5fV6?E-7REw&y*7+c=ind|yU zfgogYe5MAYxaHDubFR5sDei4=Y9nB^uJk7RO&`D>6AcpzBD@dFSkMpKXuQ-cGbcwb zf!ED;R4JUH1c*QAy(P}c~IJu%&pTnlo zQdyTNy9v~s&MTwCG$L)G1jJq$WkSW&sVPU13WU6-p1wFRyS6EN0KNV78NUAY(HbkdaE4RuBuj|E@G2gK5VIt4$hE0n0l~aH@DX8Sg7mcr35m><=$azcE@iBJ1oL)bVgu zN=&>Dh2A45ZEn=gaXBB(NtF_IJTVYwZ!~sNk4bgt+BMQxy_Qh*Qiw48jx!mrvuaG; zScbsWlcUel8F^)5VR+9t=HrTc=B+c~dy$9TTQ&H`rYw4803!pYL&J|cGG|Y6D1eK zJU;9#w+$XO9Rh?mORdWOxY$1^Xn@J%SWL~$Zv4Ua88`>*wbLfN4s+j)nSKa3sKhDq zW$@oiq{i%Nc=!H2fG4wu{cI-_@2*1G0Tlw-bLR;~gGVNW?QtVqk^%x725%tQzDMD5 zYI1=%YrVi`SU|_=WZ;@Xwy;^zXz+YK{sS5=k}7)x;*Mcep2Sghg^;?JJKh->*zs-^ z*m>3O!oz4&V7RZDW!9}vI=6ir&7G>aM$51+=*}zWMlGiYReO;^{s947G4ut`r*!5? zyA=)!xWVu*klO}Qwv-2Lzf>a2Pqk?I4H*4UPq6*Uz>}pOfwCp zyS;o_o|H5(Y{j13J8l!o=uO6TC>DR(U);sT=w1@M+UPQ3$_`By`*>{u7=oyVJq0sb zEF#us3l|WPAC%P%LQC+}sE?XwtNv;w(2(yE5k1Np%mnsJ`p?czp1ETZM0`EnWm zc^r1mrEMU)>qUuBFkwpjZKlyO=a-V0^= zY3^XyAH~IHG)venOE58$pOx0iS04VSh<9w4Tufq_A-tVFsv~19mK%#?Y}!z#Wax*n z9mcrZ6XFZWt~Q=ZT)38A$DjIJBmlM8q%TFJP?3JD${{b_bw|VvMvNz0O*s4Te-n`D zGiGoosM=ZuV}nb4lg1r3h7aOS+0Iw4ePuQucz)QSBHyUXO0plQ(qEotVP&=aJYhPkGEFN?z#@1d zjX2cULX~yiGci#-^Gsvi)0lxy&Xk46_VWaoRS7X!56xKKIW(to7b6zY?k)m>ZQ=GZuQOy@sr`< z<>vQjDw(VmD#zet*PI)2Yo(8x%8SaxO8&WP1ZapT;I=>TuIiNxEzK(o6DOBq;QV}+ z2Y2pcPA5^ID*Fi;p$<)pf@hKF>(_5yym;q(lPJlkKW&+o@7N#i)?{Q+O3df(TEv!i z1BoRpU*NyCNzS6l0O+B0=lpPb^R~R0jI#1qbS&bOU%$km@%ujLZa!5#`VdV<)b{I2@s`<+jDN z6Tf77@~GRBla!^4Q*&AO3lX!4VmaZWNe!CIqke|#Sl4=?LW`H3o}F{agi^OcRT7FG zAMfG|mcCT66ltCE>OQ*k{!#7v3G!a)PjvDbQ24Cb3m2t|5D#~UnCp8x_2Y%)EKfaj zw6HkhaT`Mot}*qVcD8tqC8kt?=PF6aF_tb7#rR1X;P&$kkA(0ht>M~p;ivFwTH5xX zj=g;=UzS1f32!E|bf{R7PJg85f^+TBb6HhwX=URiO3Eg^oQ%QcfCaI8j>46Gz9+Rf z_|vY*Cu1l^U{ZIz; zes|z8iiUxg#rJEs&%|z#{U7CB`9IY8AGdA0rPh`bIZ|yTgi^?0RIbJ~xsTjOE7!Qy zXohNI#qcG!%qaIwOqg+X7>wJD30tr&|g~ zAffX@-$=)(C_OOsb90L<$n}CROV4^pzW(P#ci$VCR zv7o7Vp~XFtR?yeZ!D%t&hGkK<=qGK*2=-|}XbfmiaxB;UvnKn>Rz$?d9qrVY+Ahg! z*!*}Pe+S>y=bvF&y;3~)*@bO@dq23*2Dy0A65wMn5QoO>#`1snJ+dCZe)=gRHT#ao zMRxD4m2rRN%#lM-dv=-fTEJE%4WvDQV4bDyt5caeS2sDHvHp^v|5G~Z(W=T67F;UTy4vkL|1xN_#iX*C|9p%!sOjp%G0Rw>EtmWOhDyE2T#9T35&LxMW^-9l<(v1eY*2C-y6ccV(po{^ZG45oNhy zQ9t~66(k{?_uy^zM+Y{YRR`X#uEFKhs;b*@B-dcuKO^eA9h-ZIxt)w=bvv`qZGmnL9q3$WYWC0+ca?Tn0Q!7vch z{ENa!p)juQ>rB(t*T0}Ls+H+#m1+g&gkt2hq-A|oiBPT^5zXyfgJgxi`y``UNUW5Y zI{L8EvHk{kd(#Mn6F@RxY{u^6L0anyyCP7yxw&bc79BzL5(=2+-$SlHGt=(BH9qt@ zsCjqZKF>qlbwnY>AL0=eU$^E<2&2T)b13k^L0{DX);f=?lmrU|d=H^Yv4q_E&>JHnhUriu1 zFnmEX_nh5Q&>7T_82Jv%bk371=gei;K$4g(cfqzgxw)e)-S4&hQfGUM)mZqEV=vLu z<6SdxBjUxP);5xlK0{U_LfJ!A0)EH-r0SH4h2gv~imeMuzuy@3?9Bt?JG#JBW>q+bp$M75g zC!}E7dn<{UAxXt_BM`$inH`AWsfkcrK9c(;o;ko9ci6`tSMcCL{@7W4Rc1ca*sDxB{x99$n z$sr+*vUy1NtIXtl-c2(U#~6biP=ISv-sc6xUg`QCOM*;%T}mo5w#a%X$Gt~d=jt>8 z1_{~BaEfH=z=r5g)ruoqujW-y7}P6DO|w^G-2jG{NQkZ`0(fE{Hu4bX6Ulwwk9Qy0 zvtEXSwV5`dYy`s&zPB4~w8!7g2~0TAEqsAwfqn<(Sb@#n%Zc6SAi>FqX_c^^0LREh zU0g8ryV92>P-t?#mOkyy8{+@+F{676YY1?0GU7u)jV&a|Wm0*#`SosMjo09Br__UVfs1YA^xoGP>z!Iq;)I4`(@4ut zeile>cQXqNtdU}X5*9ox<>SlkMewMPP;Dx4zO;-5sv%)lvOs623ILHgpATpcRDg23 zqz&Fw8^}EAD&XLP!3ko-)H^fks$ajh2a;uSJ}^yrPkf55%F@XB6+}7fbHWwG<&2^h z?CL>`0ce3n7dFX;Odb?tGX3;zPYDZ`YNYpE2BO3>#-NFfvr6Bx#AHB~3Sbb~6YB6M zKeI0PeGw1@W36~fOV(b;U?50OGH4I&t~>i2J$B! z$9Rio8=t4ucqD*w5(w6N@lt&~{W&s!`sUN&)j#NQ!VXHcqv5Ye#hnLfXy!#n=-r0E z-9p}6I2WU4{OaFY&S0BKQ^-NY@AI#!qw3P?A11lE6T3Z>%Sy?mbPM6(k#P^z1NU6G zAt=O_8xF_>KeON?yNx4q=Sj^#5@#_eIY$1T;7~3FG!I~#xJz|)0wG!} zWEcipAq02%x=Yot@xSu(`>U3x<;U7=r%3?k0V%Qfu8rM;0#(Fli42!OI+)$xp(UnT z*IJ|Laa;ee0aQaZdk1QuAC0R~P?Q9?co_67smS&xbQPbh*UJcpW(R^?_TF=x-tEk|~YgAWLnCb2N_ifAk)()GNR`wMx z(ehC5=x-n>!p#o2kQ@|h&A*~GUL8Yx(b+`@5w=}wa&m5lWCeU*TDDC-$nRZWPM;SU zL)u*|N_DGMv_&RuXCWMfZl^j5Vrl#AJ=6+4Nsq-Win>M$`moLqerNWx7 z!(f4=5Ah~B7Zr!E^tVa_h}{wNAa)B`W~R+fsk?@pavpFfdzqB~0|b+93&E0j`2 zopm>G`0V8L@L;)ye#DPHyisX;ss}sDcRfZ<5gJNTl`$0(>jvqA*ZQ@->--{z_xXTV z4cDlKO0qRPwzMvM3b5`f$@vxqp4&Sb+@W;tq%_RXWQq22){^&)d~NLt4YeA4e4AmJ zkl7%xPZ+0BPLRo6R@SEewTt?R66NB*8@GJ^+Y0e0?%Orr)X`xz&&J__@;OFu$H|f9 zt!xlBUbm`|>kE}1KHL`&ov@MZ;?JR3efy20E<6SP6p!v=6~7hJ@&?X`pT51~p!-i} z0x&>)e?&!H2mg~#aUbXXIecNb{qN!5?pV|sc_ygQA&18Q_xt!R@PGg3JYT~may~woo~dr-O_#_20ako*(*OVf literal 0 HcmV?d00001 diff --git a/fig/homepage/concurrent.png b/fig/homepage/concurrent.png new file mode 100644 index 0000000000000000000000000000000000000000..9192abcefbc38f0fa831436c8a917ccbbdba2a03 GIT binary patch literal 207344 zcmd43Wmr^g)HXa~BO)lBA|M?~w~EAoG(&evcc&ttqzv5>Lx;39ihz!%bJOpz4B?NM<=GF~xNBY-& zSqS7I1p4y1qI1H^sIj%ieB%6(_DutJ+s-#s1XT30>9jxL<|3`=rL?NWxT0oly8KDj zPRnEY>gr`ImKLL8_ioH{u(JPN zj70xGy1{6Y;*9^_)fwTuq5bb?NZ^hC*A4E_|NE$oLY=O}{Oai`^lRp<9Bpjv>XLJS zWj|#jm790v*Sd>=p|#=Z;=)rrY$4Y&G}N_V?-7XXk%uWOBie%&x_Fee>lJ8ZINIj7 zaxGmD0Rg}D-Pv^BP+iXJWcz~5Ozp1YCBp-E&#jR@i)uwxt<=n<5yn93r?;ne4!2p(4#jp7 zA2bXdYziVWsFrDj4nvSV3Yj}~|2=>PbKRdc+-?V`y`4jKzIIkcHPZljczDNVcxL?0 zH}dh|FO~+*+o^k)H+;fF+*}WL+Kg$w$;QTFQa^qy4p$rE-5Dv=4*udlPX0^iRC#rhFIiQK$cGXT6?$XN+@rre@e}*pUH?^_y&{fNNnC zOKt>J?LJZ|ksFo1+gXxh^%_+?guWHWvg<`1Z6*#cwP}P^9d*W2L67c{^bD&zwN7uE z&bJ)duM1RkZWy0ChkvdZLEe{5gZ-8@N;2+yo+tbMp>evjP_UWj;PH2B{ zNC*@T4-W^63awJjf1s@u+LD`rH%+FjscP((LbVv?&qYV^^+)sS!AwK?{L0DkXR4o* z;%tqw@Y1y*R1Pge6cwqUtF50Paf3T{r9IU@vqK;%1h-yxTrK06mFvQvnU6V2`887% zhE0hTnG|PxYA8p_Zx4J((YrB+<-S)$B`S<+O8WM`rk1Ph(L;8;nnWb_lJEjvngouu zxLjo@++DDKiC9fP=jwlEfW$~$OP$NsVa+d#6?Abi`^jiwW`#D~ zplnD{Ne$;oovC4P<+|-X|3bnT0&Um>n~o(|*_@|UAzKFTf1l_*XmD;D<~lmEn#Qg> zW4oia{ytE%!Yl(G%`vo6tWL-wps6q2$Ksn=M4P4Wv5t?Cs8q0?1?G}d+9-YioI zZ34}E;-{Pp#NTNbW7qKUI7=?|qR2orW z{l`=xkXQJUa~+iPekskB8p7x|e6Z*HwTfQg1;T(9WD1>4rXALwb&)S@u8$Zyx|&)D zr73F-TXvG)lNwy@X0!hA;r86=LRPnjsl9nSzj2XDsuma77b4=V=HsOQ>w5<3%H7+S0;8U)_lqs(-$>Q!{zF$HRZ;{4aELKj%i=)+=FqV|L5w%h4#Mg~GEb zo0HMA!^j$~WTLpC_M+ZioE<*R-v^JPZ-%Fbb${3hZ`8F9v`v>`OvAr*&7eetxdQ*z zJ1VG&NjIOkW{9MT%Z*q4;t@g*2ty3&&i=C`l}`&;9H9n=hFPWQvc!Gb+4aX;E%wVD z%hXV^P-UyS6Wv#PpNCh~yjF4N92y(b;OX1jid!1&(_2(@Y;2nFh_Em+hhvR5-s4{$ z`)u68%lPV8~1N_CPA{{i2UhEc?-4vfn+$0YE+IQzcob6 z@N(AAUw%>0cWSvc7#40&xZK;cb# zFfFYRDwB>SwLx>y)726We;Q3T&N(@=>TakhOtZ{IkeXT*(Ion7ao~gB`%d|rm}KGK z+BShxl{p$cKLDL_7f8}9xf}TyIk9Py@Y2ej%<$H!kAFbu>}qERNi0DE#o|jh(fl>e8d|tB12}EK4A+9iMvh>;`(d z$Y~YI!0nsBDpxt7aFi)34ps(F&xpcY9kY7|p8+8Vga#Pv>yhcG!u)u}>0mAm{m!G4 zEh{>#TeLXni>ppT^5fn`?PGq+1qnV~c<0Mwd?k>Ej1MmA@W_Pn^6?MjUg}1}R5RSg zrqtB1tZu+d5eN@8FjCMw=cvl4G-!^;HbVt9r1q_{93_Iv`%!8-v@tkLDTa62?=2qS zkB3nWu9idldzGS>zjipiavjU4wv-oE$o2CVE7PCX^4Q;;H^P#b)KT#C>`XUPD$GF@ z$JwH|%CoZ(!e{H_f|S0|W8vw?$0Va)HkVWnj?dTGp!a#beS6N!QOWg_QF+egD!rRR zG39I`;P5>1lL_om*DY5b5)*mu*X(5G02oxQ4RN?0y50L}SrI=^cCsrlRD6IrYh>K2 z_oVLKbUziB+1iO( zZLt#YHYu{dFoar;ebD+tJan;&&DgbflOrXLqdV)OTXFXXy4b4Ste;>@c zbulpnf+uvJY3n`86lN&$v$T+bzExRH?zFT3Dk0GeB1%tHvuRmQMwNRU>f#hLI_@`; zTz>PJB4g|Y25n%37D;CJ9Lt2so6Ea?GlyNf91%!<3EzyMjh}E5<~SU9N)-PpIwST! zU+34>3NV&p*?eOjI+3FYogkh7yWmqfn;Dcl4`UUFkPp6fiUc}_b~^C zD7n{XCADw{6d$5M*QA#PF@Tt2e+%-Fi;Fe<*2{~XTgMV5B6m5njoED(DdmRb86tUt zrVUlEeYzZ|!wGP^_m;|QVi1bFP0qJu7FqM=N0EeosbOqH6jo^K-;pMdK)AxbbWL}@ z7ho=?g)HbR&r9yXx*Y!Q4T2_jG5%FJ%|i<-jw)f1t(~WfX4H0?xR=j)cAj;h6P+(_ z0?k!4ebqvKT4#InrWt~x^50Q9B$qPK>s^Q_FN4;mh3ON1Knk~|Be$Iu%fGnP3UXA~ zXCV*@K%?mC=wuC>eATTFPCPl-%X1NB3e-&pYIS+?ApY7ZvHh9Zd;c=tKEuG=C1{&* zxf-SqBLqIF^xgRrJ+?RJhbou1-fsc>aGm1Yqhny8)LYpI3wJ%qC&iwx_uO(Fb!|IT z&Ar@7!?8~AWe(KtMD8HI&MtCKSTm7Dpa{;|YwqlIM8<|;2yl zm&5l(3bdbu*ZmuYFmN=pGX_6R%b;+_wL=X)p%4t^3#kFk*I4)i4jORWuPmM@Ey~Jl zZ(2n_xip<;IW_!PFx}~(t)2D@nL;kFIODO+hmHnyYtJ$TY^(hp_J2S%Jx~%`arwiJ zQ271(D~Gws{LKE8Qi93aXZv0C{l9*_7aXhA=CZ6&`|yEk;PQBy0xYqgq8F z2LjW!;(fmq0Qo{lt$u~v+g;lKAPS(WrnRTKTeRT*2AhXNo|a(p*;_jR;!*wU?4bf= zg&`yY2vYJDRMGU^->x3``7Ai-*_8-kKEaXPt2aa27VV{`SJQ!WnAsy{sZA{3!Fj-HEGCKPJV)zfERkl8uuTFk}Jf&6BTosY8s6_V3X^+Nc); zaHuyTV~h^O9%7!>ej_ntVVNsl-WN% zl9Hy4VjxJU8PoQ(EKhQ^EkPpXX%Y*qUU%2mZiE2Oc-rH2{3gk3l9Z#&{#6{njGr=- zhf?3Z9eGa#LKXOf1xJM=9S*uQ5d#zRFDLR7qwj2K#VYNi5fSN7VWnJ{Dxt}eQ`KYN zl3fo^Dt^nJ)~Y3F|IDl*14v~k)AV_*H)6o%;{y9Hx1{qtRT#$`5zK$?J38$?i@BBw zx)S8j3m=Zf?fa%ANJo2~wg-lPlLdLuAk$()iaxJhc|VEUY}qQW`*u12reM`ipCSz_ zi0#}CZ)|UYSgNm|1ns(YEl3+Kobu`Pl-$$IFXjG&2L?q1UcnO}l>W{gezbJw8k+0e zV03RUQr9m990SNfj`vhZHAi`^7o1VM;M--(<;Ap z+|cleD*H35Sd<+aY&s{ml?u8x85U!Dxuegl-4qv8g7@c8-gHgU?v{d|(QEl3f}HrG zVlCTS##-#r)ki~j`WM}@$5aD~c8&w?{7FN*R{p6&F<@6I6v$K8o z^{-haBfrLkvlS~Xn3GjKRd#Bgcqo6(&y^cHgLRbWc)f!wKky*2P`h5KdUr(`B#ftf zU7ndOWCs}_quv2WgoA!P639QduP=5>?As)B3l3>kbM&t*02*5^t(6fII5kES_H(!H zO?BXdbYnuMBrQyj(?(m;CoShZ2AQF*oy!*#gPHx21#!od_>K&w@v26CkO3dAY_T-z zJDx1u42?+ysb0vi)vwr++RJ=3JTnefkxD-mYo^51)2P1oG*m$z?yx1bybW?9czROO z6*eqXw~LadW;4yNN%5ZWP^+Zv_GPX-)DCf<^m3xr(cuor0T#ASv|Crn`dE@ZeroJ? z>J=@8`E$mOW>WoJ-idb*4UdX@6>bd^~!O{SStw)qsNOBmBmjb-vSuKe_Y?CekJeT4<)Z@8rr5)iQO zF|?*-R)T!cphyK5{W=HhLI^KS+$OUmx>IVvZhKBE{La|)Otw+&(P<0z`~!2&lY`JY z%IeX_Am>g#Yr6xymMu4OJ8j)dT`VN2x5aI@ZF;kHmI!%vI=H@lo*iXR#+h?gqg|;N z6-#q}-q?m2VwWzat#t2lx!=UVa{sbDI?^O7VOrUtWPUlFAM5H}Pv50b(P6RTxp%g< zepwhvNx`-gqNKEFb#biT>_Jl&-Bi;6%>AEl>u-W9PaGaLF7l)9c`XGHPy70m4&Ybo3m?i>U=?!4jkdU zdaC(K0$&asclP*DW9K?50J7R*wR{PvRNs$So{C^&sUH)fLDbO4Ra!FpX4lt8y5Z1p2g8jn|TSh8duMA_}M4+2NVpIeb)LV#!gr z;ldMr_Lst%WBG$&ys1-+6*8 zsFmX3VCqRsasfkTW2XYA@#HTpfK6#v@qm{)4Eruu$9(^OtwfxPCLxhK7I}$ zihpQQ?_?T{Va4#w-pzKGBSE2@!ttu-{Z=ZuMNdM{ffC>?4hOrEqfw>`tlUp39Kmpn{Kyr2W4uq95F1A zM@qhiDiAn|Tomt*={LKPzv39rDP%pIzD(xvIy~uUbs7D})kBn(cA)cVREJZ?z|kef zk4e5nXDX$R<&x{1VgUH5KNP65)qR)JY-q&2Jx$J^ikw1I&2NS~PwUcFk7l@Wjh=7g zYl7D@)3U=67bm@d01VwJ5T5?-TJ=(t@}j?t5M!=`s~3mP9hN5iV37LWtKa*mRFI{l zm%90Z^d2QZi}}??Wl5fzQlj^Pj4!Kw*ueGbAPy2M{ z3E&>2s|rd>Wyg<7I*hpTH|J@n26lc3@4ZH~E}7J_{E?5hZ_9mjUqG0kL8h8kcV17s zb;WUbD(^}? zKoDMhXpiWTHgU`C6~g0*N~E&LjtuCTmLt3zr&Dm5YcZnVtU_4i!-WWVl)~u*WGN9e z-_xF6j%v{OZ*J35TB12LO6ajNgtizDv(x-r!FZ?zk}4HsD*L&kZ$P_`0kz_Sin*qR zj%zWG%JTQaE?pR8>cgK5Q0Ms)ab$V9ct^p)^gWhnC8Mr?*Y8XT@cn$zd0hsauwt#k zP#^tHs7S~x74%-8MAPDC*58a1%j&ZDai{gv5^QuPwD{CUDr5Uua75C-=^13}%B912 zyQy1Un#p!JP=&Ublv7#oH<4r=zo^K(W85q_pL70>UwTD+I%REli&f!AOWL^Qob%h~ z*B179%Bk@u<(znQ*pluPk1$f+-t>=_WGI=hqW?1t(%>%gr#8fD#lef%Q%m=hnNg80 zF0B-$mkRxdsq#JuxI7N6p>ZY)voJKdIM+TSiM@XD#oPaGlQsFWg4E7OUM}gzn--eS z@Nq3WZaLs`HprdpmpiP%hHo4B*q5>#?vvson^P+bfU;a%pFduTyZz3IK9WyLK9JA0 zT`#PhFy8x?oAiZg98M#zlKmE^j`}#e@@uu#$AK(OXW5~ee z+C8~)J0jK!qmU2&qh%C8e{`UCE~M#Az2?KY7+$cT-;XO1Pd{Jv=$~oDE~mw}Viy-X zN(}{^o(1E7b{P0qh6%PIP%927)7p`mhz*>P!;I`lXunQr~gahGd{l5u#*36<$8YwojdmWW5})ud3D;0 zAr9okL@yOs!nc;Td_-6fR)FK94ZBu(sPkT{xyt$MiKV&)7GLG@47x}`3*-L znO$V{f6FE+E0^SjQSVo>#ZegmIie4`I*^G82kp|?BT4V2(CO-P~V3B{%5b_}J zoR6MjuziCssj&py)vKdl+l7PUOn04)(uj0SY+cgFNdUqvR!8`o-WL8V?I}Q9g7wAp z6Wu_BUlpy$TpV)h8+dvjA>psM_OZ1GaojtYg`+Z}rhz05hPM&(L?u5ORc5-#-Y@qB zd3^3BA>J;TVO8tOd#7`)r1`kHL>hyj#WA=0xhelcSlE;5uA6x9d&39; z0EIbmB%ENK|8iOy1i{lP9qh<1~qOTR8j4*`wyn$a~6%{`LY8 zBycOQ-SSWWc7?9m!D#s^-dpA6V4>vhk^jruzi?mZN(|eZPxh{|yn|Nn7|~J&3M4;E z$|L)Ma;}OqSVZ+`DSSBh8!z+#R>XgPlG~Omdkp>RA?U zOVsuX;_w(%NW|wNV3q&+Rqw#*WE*gtthqJMvvr^G3bHz6#=eG|BXdYh49iOy&8FPt zs!GdLiFQTm*IG3ePl@z}nwU|u1o2b{Xrin+7UHHhBKO>0Ae6ZuTm4=v7PS5wTfHTPFI;=H`2Y<^Rr-Sy!v%388hbn2so7IgG!^@8PFqwfzAQl&ZN~uT zyQ>rel;M2hcMtTcL%DR^>zaj(@t~1}&GV^`Mh7a?jMfFbUyo=lM!IHu(AqM7r5BK8 z%5M$+Cs0E)c!9X0%4KWrK)ii*e=%_QMmL%f1f5j1r(55mH`}y-*($?P?r^=?<}9X{ zld7yL=wq)pP5IF~Dvs(P3k{R1?&mrDBp)S;J39Fw@XeD$SD%^Rq9M2$;(LmchW~*y z-fiHx0LUXlhh)rciwFB4KU|sC&$ERiU_8hGM#_njfmtfzVVFyE(J=8E8LpO5p%eSA zn*#>naOULn+ujd`=PIc7Ew2VG>Jw!xDDHH(EEOZ9BH@%a1-X|X9in4&0S3IYykCid--_807%Q~paZ{EN4T=PApoLabw+og&q6pW5dKdkgfEmd&o zqoc!1@?lxpQ5es^!I8BD1hfX13+jAMWqje~t(-y9BZmXJW?SdBlJwNl!8~hS$s&NQ zf~DZc&-D+!mQzsDqMqO!)gB7SdxhF(wQxI95nRdM#FTfTD;i&}9Iwj&ISv$AW!AS) zXEjFjB=di3-0(d^X}tblf;Q{cMWAJTEoBgxD2uuO{b8Z3f7zdbC<@c!E>MaSGy z{^5aUj=K{1!lL-uQ6}yJBZ7;$g8Jd=?S zzH11|eFXGuJt=^Y17v+!sl!sGc70A}{hsVf)1v~v8l>b-_f_ZHO2FYzPW;8F4SMH6 z9dtBt^F89gB%;XLHL<6rmu8D<1fUCD!MNlbBA2&kK+%+>qB3}PN9`N=Ntg>Fi~q{> zvYTxKce!d%ryMRJFxO2R>tFd6e1f$LwPVBO!x+K>y9>2yb?qu*t4-16OW(5}mpLWD zJcM(jP)Zt0X8Iwel?Y{GstPXKyoazuqe#W_@6lC%t&frU!kyq1u{Fv!ezI@nsYMJ* z^aN9nnX&~peAmN6_w7XvWrHRjLt{fwby8GSB~<8KUw2v>*c1gVEw4e*6@idCxoyj< z3HXTCwzeoxxMC$x)K`d)i|31sk4Fr}*%}(Wd@R>W+U%&Xzpr9ek4b6)1iMGkn&E!|;SQew@k|x=7 zt_xrJV45(z%^@6)P$WEwnty8S_L+)L5Z zO9y!nF@vM&HqSLH!K4_dP88D4E`n>yLseKk&Ye{at9wvQ1x*wN_3-Sc!V=|@H}55~ zF4tI&#zFT!pSP?iq)gKgqeg~66AEoEib(xRi~^|0RP&~WTF^$U4}yUdH*-u6aRyUX;XqR}yNujp6|IT;eP(cX9`x$IIq?Cr+S140}WooBWVP~E8f ztNtO&bMbvpI3_tyw$s?rYotcK&_i(k=Y;gyLB-B?y_+n8+E*@l&mVsrwJ;Dv1?sYN zIK{MB8w@z;OTiUFy*ZH_65=t(xd(lFPA22{Vja_f92(5y3=Ro4u{T$=xVg9BCSwss z#UmhktDu94qnb)N0@2NwW_ewDJ(5b54IP(0J3NGVy_b)^R?(`xO)BbzRP%h3X<2q& zDeaV2u2SE^J~VChW1B!;-Yc=7WS90xE~4}_kM5SWtdE?co~nP*nF{+KtQw%Xk?0{I+riLma7a<%&wC;hZB@GKOP4OJ1=KO#EgN!X`&lU&GcP{=h?A3ptThM zEA>^lrA+rEa_;z^(;Dbv0%f}MXMqN|Fsiar$E6dC>YYTfI+M;W33$OzF3z6|1gM3l zbz*9x^5Zc${QKoN=n_itdHo}~ztYiZswj7Zp%j_jYK#VD%jEY6zg7ldOzTS*&se9& zZY4_NzsYhj7@(=OS2GPQerc2{RaK~_miXY`d@6?Tk9mK@i|%E8{qB>isD6FHV+FM& zL-%&6qz?4?`jF0&rKP1XgKKU{?(QXkCF1cs@hhwHY*gEF2Ff3I+^(iL^`+3^S4w>MmeOmBts21JhVGK|c1~VNi!!UDP@eLwzx^V~ zI<7g1HoAX-oqf!L7R8L~`#9=82}uyV=D9{`HA0z)$|(XdkMvk~HQI z6{l4Ur6)czfRwZUtr(vE)Qm*Fsa@$lksW=~CVc^?H+ zTj-9o{_PSKkO-KTqz_@x1!5^GUcKrw=Ls|Xd3`>KSP|5V-}AQU717c#D2&#g5N+*e zzyg6weu$;`5R1JFFvA@utpRuXxj5&#yOLd=EDZ>cS1I7mHo2bqI-Qq;K<{7$dDXCU zfrwfZwJHkNC}ePf(A=xe>HrUa)ZA} zsE~Ph;RY<6&aesVi`r6ibC65{p!XZY9KTkso!1i@pExY1f&RyAWh3LvwkJ=YhIxYy ziQ0*gXAfEFN*xO71T(9uk5D~aSV_c4yLwLtBuV{v*4{?9f~l2I8XS%$;BaQ)y4j|f zXbmm@Bm;`%;`G+i4B5YU-q2Eu6@2ZR;w;ykoQAVg62P!A#i`Zw_g_AIY{e;YPKJ&C zA;bP*2C%$I$Kyk!KBsYh#{wLy5+}#KwyLpqtoRu4Z@(>JYqqdhUhgU(z!| zp2v0<(!R;sa=;`W&eUwS@eIHY$*~oWjFqQ9D>NAL^FT^w1~c2wOO`_SV3N6ZQ~wZ* zkj)k$0f-9nz3`P76DxLUQa|2)av4dJ+oD+%czA(|_*$5Sj}=;9)vU-z=YcR!%4_=q zZeLX=lB4 zAUL}}#Gp*eV7JtO+`YF;54QJ(RmQ9^PeDyBhbp`D`z=G?7ta$5tYD)7^tc%kzS+pT zB1kL2TxC0MNO>_WEZm5YezRe%VU;hc!7ECY{g6@#_Fr>t?0U3JC?s0}L%c z*kot=H~(f-yYP)hKi*hocaR<(2l}R%n`_JMAT%%*nebYdx&N$psM)K}BO`Z~jV~EVng8}%cD`l~6Uor72Kwz4wx z9#eQhC(Q(FxV(abqMQ;Vl!%C^Tj*@+o^5q7^*VrBfmnBr7S75QH)_=QujV0as<060&9pb^D1B3TC=+k*mGpL~$3T}Z0i<8we zccpcUVFAjn0MNx`D#cCP*NaRUb;A^ggTi0Z(N3-rWF@Lls>aGF)!regv{LC1r2J>r z`P(}AnW<>Qe*_brJoUL!euz`7w0tQ}0X!ktU#d%MsKqE@())%p$s)cLp%$54C`0 zs=F5D@0M$m>eF{%$r5cz3(A{+L=9M}`V08e9`S3NrLs)1s@xl^R|6Cb4=4A?kPL+u z%KCQabyh+@P*JJcU5c!lUyJWJK*0=MSGwIm@m*kP*)A@HDi@XU?Hh^3pu2zZ)6C@7 zA~nRbhlc~BPGjd~9ZVA5jLFGvsBU^d@gsqn*)`QW!cckOs5}p+*OvyzYjS`mK5z$^ zc)4FYyL!FGL~T#9RKkI$Ib(wypY7o+yh>EBUYXW_v8#V3j$1j(@>+Vck9?m9+?s4t zk|i;uj>H%)RaG&xK>yEM6^~V3r2L{j?jeORVtVLPRv8WlhIgK<4iP(O!!$=s(LS@1 zu0@t}LQhRjjduE#p@cme(im+2vs^O)VC^T6&VXistej$d>7{ck1I>FTCUo=+Oz=_5 z`0)9VE$153*X9bYt~vCr#)Y_Yxox+^IjWg!r-#10Rm|ocd7ksM@zy;ic2Z$c0@EM( zJW3Jt7C8~EQO}$h|H>dE@PZ(Wo;0Z67TKGT<|v{Ur^evW2Ccxfp!q7J(y>~~fg#7J zHp7mZh6ePm8rmJU+ksV??&J)qdP$4;6$LL7Jw=n3n*7*nZy!o9Y-}7fonVkn-#9=| zx>e6G9DRaZ-Z|$6(xqVklN-=Ihe7k|RFW%@7)>T-%3BUKXUO%U6zj(fXn4s}_WIS? z-?`TapL~m#45{`iGP~KO#HA!sp`)DGvbN8I{;;e8313SPrhKrEsJIO~)=@3sWW)=Lpz_6< zIkxZk#0|v{=t{OQ@wXhq?Oz8Zvs6S&ak;nK85_T4E zig5Z1LbB&Iw5di_pD-ArNlGXc+glWG&P&7nY9^_7Zv;!r3)c;-bIe5p?lYchFuxh; z3WB?A4!4KnnC?iHw(CJ@?^Fm0u>L(I5ao<2{-#Y3ygL_69{$8_@uO-)SC{zOop%gM z^8}KrFz%pfvn@npZF8}fups3xS-4RRL~YI6uolQosye)z=wCV>=W5miWM#HjHXzG|_o=s9)P%w&X67-F)B0Fl15a zva%~8gMY}iQA@kf}{m9A1hQ9JW~r3IH;kNJ5>7#`X(pP;H$`-bIkngmx@;5 zPs3}8hs_yI^>}+Tn9|fH4@lGL*{I?<4jo1*k>yI1SPpghcyw3+o}&Sbi)-|BY&NbA zfBoX;-V54|yK7U>M7yFF&u}rvhT23_ENfGm9Wo~r4^vCBFrSWV0i2tJu_&0MQ&;~e zy^Xt5CM<_7XiY8>ldg%`;V!f*IYRVm_xckU!BW+WG`2tOp+KB5f)}T%3raZn#)$CF z?0EwkC~-i#!*w|7X*B9e%w1S)v!i;Y&B$EREpZ17sdX)ZwC3`4z)r;)Fjn($iV&F6 zpQ{Is%=%w|4`BuUr{2^S0T~#OsF+S&lQY#t>-Anca40*z$f_4~k>j7dMRG@U%`2ja zT8#Z@(B!+LYN|gz@7kVIrxN(`N-dGCcKNTaH5eS*k%6|fmpGr(d0NuDsu3Q>%*WFc zht^_Q2{Sy5m<=_u5k}@*eteueArex|*}A_L=($jEr|3sibjit=t(Tx=z-!1BSHXa zlNVr@3fyd}e;^~{&=1nXzE0G9(;BD!_<=oY?lZ?RUH#4!1P65L{7gbgkL zm<1|}HzS>oIwel@!JJ`+??`jl&Zk^GZ|-zkmOeNMGl5sl!d#u(MnO+Cw!VPCiR+v) zOWSD-hT2WOPTpSD2i;(xbE=gT37SYK8Ey%8AfmNBv5-1!%0S5UfQ>Zo{I;scdXy<_O?7)S#h)9Nf6||zt_)J18a6V=M|Mu*Mo20vD^4&w+xk*CMmlS+R~=JZa!!3 zaMiP(<+Zk`+EW>e&yOZ~2;ECD?yCs8hkY-`Sj3idpadbccJvd+W$?AuM~0LavXjzmGhQx9X2s*f4PM<{I>FAIB{?64#tf_Q@@8 zxz^U~9zLeeU24~nx3^DM4bCY@`S4A@DSG?K#(_Ht9lxPqmu*H|XuSFm9zE*kC}jCX zv@T0&j~OIjIcBdCho)MjZ0CUAGP^WC+dtOCibidyG*%horo_}_SuwridN5OB1&w{t7_IFV?X|Thbl0Q()blBu{Ql-4 z$M>eD3EQ2mVdb%miY^g5GrmO4(eEvHy*9MGwuIwu2t!q4v-yj3^HuLIHGF-LkexTV zhAYXubUZxFXTsh8O?lX{M}WXc%~{e@mbfluHROpaX>+q^d?+7pmH7L5Ou=K&3?=*t3Fi~7Vb9`1gw-IxSt zl{gHSo`Shn=8n3P7Mtq3khzYI$(W}p#ofn>J zKcQ6Had6z#=v8~k%w=po;q~YdSFUG$y#AVPPL0&|p~g#{_?3zM4?PrD0M~qt?`8csY3WI+Tua3X z%@-j(ioKVNam70yf-;Bkv!sd4g6k$!E=5(zN&81ff%ZSc%K2ik@}%dRt_wXQ^hWNi zX~Ej>#YMZXI$6dBpJ^g?ccr&%qS%;bV$#zWfzcg%lCLI?Qnb9gPl{_IxO?DE5gIGN z<}uPwODE8VukIK!wDx&$e>ggc1UKW*Ut35RXlzy0V@xt0sF~SKK1Wi$@vqazjfKe+ zPoJWBXJv^4_r;1qIb(CAaWXeOFp%}}+*8afaW64Xr`35(hYXH+p+%2(4E>l@w$OYu z%`h`VTAU zK+#`n&eu;m*S>K~x_#RQpJcd)ZG?rD*>Z)iZ`F~(Z&F|9rR;*<&fZOWX3%b@r{7X9 zSyx>hT~P3gX&IxD0QbFw_$i4%AjWjnMJ_H%( zVQJTofBNO&_hOHbEVM3AowOv=zyo~a{Ib>UKN$8s|AlhdzA;Rcd*Bw?_`tL!dEPA! zfhi!g)(^qKsU;=xb%OoZnlW#}d3mdTip$*Q>15@HynLy7c$1FW@W$O>ru`E^j7Fcv zC%c6a*X`_LIGOA}Gw}1Tylts+LOtCj<@Ou4kcExvScZm%UTspV}8h_ex7q(VidQJmdU`k=}ab7w24<6SG zo}QeWHbu%7wU6%SwJUICZ(7u839j;4H}%<7s){vdGJgG>3HaqyK#SaJgxHS zT(1f3D!VT|JjMHTvPk_X^pDk^>p-ylLy2=y(D_Qe*>&!i#u00k^fshpWjHl0SXufR z8~Jlj#1KfjZr>E2xUZ0m1H6EzQ!)Ir!$z-Sc*D7dL`b!D>E;@E6#wfLV70NK(af=4 zs(oBDZTZzNZlwD4abcgw+R{?0wJQDYERU;KE#uU9A>xuy_6pE@+eQ;#-?Y4AJu$KQ z>6xO}83a`p48fVnoelZ1(>lMkIoUGI4bymHI=;PloBwP@KMgIkqr;+Rw`H#*#~I@^ zA#U;2e5EltAPYJ0!*UVh>lxn_gaa$wa-wZT4mPfO*6hC+w@ItEEWMy{KRfgrlwW^CK62Z; zYkroCrC!8L{oykM0>Kbp> zy4|=c{75t?YfVrr&k48_x2hhRXCe1#@Upv*k zUjh{Kr-#upn}_k_$e|(Kq>?7R*&>nC+54oV9CHI9A>IRz*}k_ES3|2iGU!?hkxef!A*_!=yJS zx6>j*{Q~krLuY}y$=K)1WJ!F-yeZ9xqGZc8(-`^2GX#~+-BF*jwccK;|2rMBzGfC5x9a!85Q!qa=VzG3xo()*Y5+^&0zZn~u z$g@On*5!3f3K88w^7vM4jC8Dt2<+UJgze~ZyOEg^XUv&JHELXKd>KR_F^#> z2seAP*RyaGvr2Gcm2)YVS4@B0Qtr3eHartxSRkp|*q)!sj)=qLqzgSI1X z2L}fQ>3n6fE7*?{1!#`A;zh?x@Sc*95kkgQ%h{P=3*DCG0Gu#*k*dw?+y@zcA%)r7 zQ(0EL(RI!;dkrkz;3N z>&b60lpUPl)o9)GtPhpp^pgBKLP8QUAw*L@QXJ^xFB8|*_>E*Bk5l91WV`ZkaCb*5 z$;#c`530(s8eYRZ+=^UdCL- zOvwkly?V`<-VFR_A3)8JdzF)d{E~&gB57vkB}AgF^LJ-P;l`0i&E{Oo@k#w5^oRAw z-1_7n?{6h}+L96yzWLRwJ$4ur&`RVk$RZYsTjD;5W_Mh{hZ(1vwXwmf_5#0tO4akOgLu@BTM1NHObpSCwU+mu{+ z07!ye)>g1+k0a#vQNyspvN6{r0`PtXhT9Ar2VeZ9ohTo@B0`n@7lLj(fe^HK+Q*Zx zl1{60rw22Bpwg{$!1Zc4(*qP=fID2(D?M#OcdfR=!sXT_*K@aH~&zS`cRt<&J z<#@)9u&$6a#gc3bXw_13psGLt&r(jl*dG-3flLUH03a&e?&eyMCYHAi<{{(Zlzw3h z3Cyi#1i`WtfJa9Mp3qrzG;hunVCu57(LeueHF~YKNTcTbY`S4GpkpchXBjFMzHdBB z4+o`sRC%L}s^T zEfSy(OKiq<6ld$1KLyVcTu99n6+uKAKOdBQxr5Ay^+Bkn|seKf`0yjEcfr9i*13yNCokFQ=1?$ok#RPRqv7cHG8NFg~`ts1ypmJji$x zm-5ZbP7k`drz>kxteWpw&kMB5>iqKZ7FTX2ldF>xD9Zk!$)5KN^h_5ed`0}(RCH2Q zzf_77S^()jO8e$x#eJ#U9VBi7IxZ+${Fki1QF!3s+VC3A?j#N_F1mGKXHY?N)>4^_ zSXky1qJy}6h8ZV(F!dpxq^xPqDhZp<2NS$4qCIVmWqic52e%F1LQLV`+-(%VeKEuT zD7e=l6aCv$nYrdO(S9UtoOg^jK9V2W_QgPwb}g06D+V+K>wS~9--@Ph1G z)$F6j+ZLCyU`Jwuj(*P%0)qRwG07xkf)*5jE8JF9x2$?INMJ$z!oG?JKKkd|NcRUX zUc?opVpwAYL`_T_9&Wt$^0N_{R2_+?XjT!7<%vpDagvN(V?I&o0+*{0eVS~hyr+SkM_2pC*pRt_CXv4xtW-7xpv#9 zE-$0P7-pgiOG_Ucu1@F@6B9eGW(&RF6_a>+EFI(@_`Pz#3sAcpyV2)16f}kRzh_qm z#i?u<=xem>FCLv2sph}4nVRyp*cJ8-N$B11Q&)0!y^eIsdBbYVZZ;xeT!8)zUPlrE zPx5eEl89R8Wh{&%y}PWJN~f}oaN6V-#)EfZS7xu!AXx{ zkXH$k2EKY6;lJ5g2%>T{q7fzKDeN6XjV2Ol`r zl~p=Cc#e+l-nEqwm88QgeaHplWm6S|_kD zdcnkEmEM2M3rquO;rzm24G)tV2Dne2G+wo}f2S1d>hhzmBb|5ERM4>CCBU+jQ$crh zhbavc4`E8fD*lqoDTd|{-sfwd;0Mofam|3|iKb2#QZ&GU>yV08_YV2iIr2oq;q8ZZ zn>zR#<%e2IA^6^aNyt2c2yr0r`}wNpB&xKK#x%^-P%UGHkXtT!wtaj}#f1JD@G#v} zIf6m6^=rP9lon0wkaRzA*8e&73yOPZZZaHTm8P{j)@xq5-+t4yfY3P6W0_AC9}Q+* zlS@X&ciDZR0GliwA^kp^csEgWJ#VtkS&P}Ap z#zichhSy7Eeu+xKQlh%gm>KaC(9qIuV>o*W-rffr3PeQJ5u`G%9Tz2H9k zPk^B3x7mL+@sn?CZ#GbFq5UI26|>%tuxRIT{fvmnNC**XaIid{FuJ;V8>qVxe*Vse z8sk~1O$h}pN6X}D6l>Z7G*JlX{#j`GD|j&V7$6+0hm0+-dUc+WnM}z2Bi0rA7PuI*hLG!P%?PeKGB@ z-LVHfNKl$v+#n|;>xuK8-&?a4AtiLXV5yu{{l#yx+pu7wulF;Qb%M}$O6{~!Nupsld3Vpf%yP@Nm;XDa(ey(bl#K@6u!Ck>_vZ>%M2;0g>yqf zT!r%jVAaNN&Un2c43%yG;xHv`a6`A#-bJ1MwT>UmOMqco869W)Ju)H+B$3-IzPVO|(gd2~ zdkyEiaA4wvrNmYzQ@DFm=BEMy>60X(rCt2c{{hM@3B+>Zd{j&d6&u@1WvtS=?IXbV z+kB3AmD39$w_C*EIo%uQxmUx|+VV8&t>0U;T7Z{EcYA{m>=i)7Nx`V$E8

^estuhJ%xAKYmG1^{3t9zNKA}t1)as#0U8<~|Divx_j~tQYFUMyw5&5m%YG=I zAz60z#ggow7eyO%UuHZONB}tf<@=)*vj(|-UX{xcq0bG2v8Q$|C*C&*nwplKSEZ!| zd^?*6h##Ju)HM$X!JXZspzi&0huJj3hPIIUDA#C}G`HT)XPIft(z;#y!I_!kwrS`j z?h;&%dr>5$s4dQ-wbm3rKzg|~0=xny1mZJZ^>RvQ@R8C1n=Q;~zp(Q>i1S_XDRIEq}j% zgh)u5*|TnQ^4Kd(RyN#m;fq3|$p*(QX@|Cci?`53joo82GXPaIe*vUP%E0i&Zj0c1 zc`4yKN_@Fa3$g2Amy)fmZCppCa$avrO2?13fMc#?^`l2v-@C_tsQ{2lv$oH0z{Q@Y z9J?Y#I-SxkZb1nVWUsnEPl~i|a_ek+wr|CR7!{V>Je#{qLQDJHoXf2yk;etuYTnJ^ z@wMrt>k;zNyid8w9x)Y_w1h=Zv_*DKPCB5QJiSQSlQ!h6C9QS0c6OH1Q1~(j(69CY zRinQ#q*lFCp(fY|uq2^tQCz``aSVrTZeg%CLM$9(v$766<^TFv8)32dDG*G2 z_!@AIGro)(FF|}1tze9c6^NnOUNpC~yl;>cKeo(1PL&>R>%}&X>CjFHmNsR5Mu74T zlLj!X7h_w+(lhw&&?Ftl4o8%J^DpNh)c#Z}nh#tBl6`Fui-|j&XX88@Gjn(>dr)iZ zph8GuKO*>3r%6{{j>L|K3`&-?^R$c6@o??lq}*O*TS4j6Dtz^OFd+7|+1c6TjEvy` zu1uJwTJ&N`sjE}tvU0zdlA`PD%V)>42fqyPu5SNR6qGeTa)LI1mGbk+DJjE8MwUeg z)s~GnD~@IL&At!GO68~9$=VP<(T%2yQn_I+aWzt|a#Bro;wH$+-TGu?1R(mOjg2?o ze0)MUZf2^| z*qHCp8%hQPN=v!^Li6UV!(rc%8;|R>{=SU$bluhJ4={N;M)_RX#+A)z?r$9q&F?k*nv|GpBf0uO)-%R- zVZ!yARaVQxS`KHwIgE_m7P%&RX3l2ATTr zlW&Yn#0Q6U(z4Vx20`YRZjhy1o@Djfey8bPd&lxQs|!>g`dOSlb)SgpVCs^ch}*fU zx$;Vf!_KbEZjtLR*nICrD#;fc z4X*(O^5(kHEv$0xJh3~c<~xIq4ln7l62TB$di1!wS(lz(`Vl|@ zr)!%dBWgxm?cE*?R9#$7Rzjas?I~;bPl8{^_O8)A&(CwRgohJC2$;B9K?dySqMSgU z%X;z~4)Q-hqq^bEb33H-|4jmA*z@-H_y3un_N?W}wfRYB$w+ar3?>gbt3g#z^7s~# zlTxKak*%<8S+`PvuX?U=ZR+@EdR(Bqi~A-_K1%@MHFIsddy>D(JpfNQ))sXQ#aVXZO-55H0lbG(dr)&ar6tvjqB@~}gaKqH`9g&`To zg83ALylLa|u!N<=!H!I){VK)Wpx(gWI@MpRIbd|+_uSf;Z;sa=Jy<|a&VMt<5$$+e z)+cg+fS#M|)9J$>%_sh`<2n@Vav99J+Y`4^b`pG&W>u}aS0048#0`x!{ z3qiYU(J7H#<-MKR0|1%-qrhPbz#{(l@9}_;toJjs2#{Fz+uW@Vwa5(g3#-F@^Cbrz zDEFSfeEEbL3GrJ&_sBHYmASpEYp=k7UqHaqDZaz==Y?#RFBo|xF@FOgkA=(l0Z@yN z5T36}c`9IsqQbZ3TNXVvZg-NRCFhlXL1{6Cu2ll!0gw(f0z(NIy;+$(e3FOU-&9WZ z{sM){c8y{dO*-|0oA2}^-CUS913BpvJNCf#rnsKch%jY<$S7ZHj{zLH@*8$MSOeQ+ zLd#;lBl%irW5?zIPzHwH2yRMqxz?RK zw=CR0d8d{z$~ggcz7%KVizhj|oC;GaJ=$Aw4&X*c*{pN2zK)IAhex1x_4KsI zVdFV(?jp-iS+T$OQU0jwW8u2BzApP}lXCLnL=zdG!>{Azm5++N-b0XZq$BN^-;nB8 zIe!TgKy0Qfxlk4@CsfL<-%#zTa&Ml@pAA1Qv$HK;44$^+9Co}-qe%-ccl^ywpyFWB zd4it8+hUlq_ap~xVE1j2`Rr7!itBXU)1)lGspb_IaP=_N|45y_l*0_*Bbm8tJW# zO)a)>POinz)Ku(8o*PjL%EP!-39YdG6e|`!@fqVWEy({JR;*DYdSp21EPwqs0 zVQ95eX>YO!amxG>8sXDu@W$DF@eD?X-Z=qd62U8U% z7GAAaffc4*SD|K&W876==dW+@;eu1x@fn}nZ8iT^Fp-t@R>QEjMZ6u-a4ef}y;^W{ zkjNQTrr>`UeaDrC_x^8Osr9_H7d2dr$wK%TIp*n3uqUX%&&?f(nXG0}(U6-{VAwM@ z%YEL8a&H~rQK*G!^VZ47Vr8f8uuJ8tS16Biovv?ApjIc)(v_{*)^Y5V}6~yq7~*sKpQDU3z7+?RseA z;+M<4L}{Sg@5K~0&id#$o*k*BSV1+!stiYEr>1M!)s-Lq^5?Q(>IgwxY4gbvC4{B? z6)!?B*SF)<6J-7d{b;`N&i`XR1Flt**-f1~@8lJYW}y!#QTkY;43q)~5&nzMU$0bR zm;oUD8d#>ci4#x3snJBdwTo2qMAnhnl7X0v%ztF$XU{1ENC68_n!(Asv1K6Iwsb+D z7WqPP?fnYojWfu!xJU(&NCjy|lcPoYB#H!?#>1M_g)lfykEtr!aV4a*zOO7uk0u_J zRiY?0^;>6n;YCC;&myXoIY@GwzWA>kE1RnU6OaVgFi~PNV(-(}KHqS>oJFr-ugJ;g zwklIfI{E)a>;6fuYN7f4Q8(>-zcYS3Q9^q5?dd})H-?jGW zm3iemAt9x~Gc&C{7DB$m7MEUnG*5+oO~;!9kt7=+t2JJ<3A#fs0phLOo^ttC|2P=M zoxAR5Iw^d4;5OcY8z#%*CFNNnwIlc7A2AB-JoSh5vm9OsAs9yhk<4R8$;im``PZ5g zyBIuD%#Dg3v!8Qcvndrn?R`4w#DA6GkjrsuYUcDJRS zN~K_gfc&WV8XH@C`-GIEu*Hu=p3Tzyi@LhNZL{%v!MQi4c(=ceW#Z%v2QE&1IWF0O zKmdIeN~x_m{!qE%Ec$zK!3|`uHnnK}{63q3^Tr;^H4DvO*GB;VBSZRs$hz2Cn3dAB zQ8#c#5R2(WiLbRkL7=3}-Po}!cI$9&n04x`8*J7VHp)VTruW7vh4yUYk*^pUXXWKA z@!iCXYZsjmcsj^?=a>T~O)l+%$UIRa2B|XpC>w@!C{_e%GJ|kLc}{H4+Sl2*y-i(E zZSW-l{e+r-pkx}v7BDeE!O2y-f%{?;%Vpd0Rd!lh0uXsamk^1ckNNu69M|5Ll{FX~ zoB)zP@F(nex?0v$u?a0@MyZ|}5Qq*OkQhK1g^=@}Wr*fzOr1{Uq}c( zHys^siiLb`X+;HuQt{~lCUSIFn~51dvP6GxjU{PSY9W9xz6@#v26{iGQ@ZnpF_7mgI&Q0-+msuA#t0{aLJ9P+ha6Qp8&E24IXn}@H@P1$L132!o3-iQ& z`G#4m&n5oGaJ<)7YQUsb?LDppY%SB3+vbUL@uXm793Fy_fRInYtG}E%44TBA8lT;#9MJJbSO4+(tI)AAWTpvlOuO1`@;-1GizU7H951qkCXSmh{KY8y zUZK4O>MpP&ze6#pVR8NoLe^2L=fHSYvtqfUqzfXo2_oqyJFDMx$&(h-kB3P%w1_nD zL}2BrhHFUn2g<|V!p=@+;Q)(40L`l`9FQ{-AG*_K{MeHe&Tkl%*2%4!Lozst9qFx# z6FSh?JMeJv@kPvoZDz(3Sg4q!NAEcEX+%*ZY!w-DD`)L%{wBJ{c_HBJ02TVq#^1k3 zr>J~Nn&nM3ZMQE8x-mbf98;uV)Ld*I>Z!UQPKf_kA?CPT9RUirmE@7&0f z)0(fzDA+ml=!BRqGHZV|Egh-RWp_+r?PlCTh zUdOnQUuHCFTC{8$GU3hBUIgE=Z($ZF-@YH>sjySm=&XzW|4!*+%w0x2|JXx&ba1Xv zO=s`M5999jUKxLKds)06&5caJ>o>or+tJUTX8w`Oss!!P^cs>^!sQ`YVqyf#h^?dBJu|g_8b$g2cEbO&mjzC$m54qvD$vRow2p6@P2J-i#AS;86gR9aIKu;#Xv0IfqG&G2cj6csE0qf(z|$>0#+GE)NIdCKr}K z0b6PUim!TObKXlV=J%#pBwMSNu%+5ncES0E!I*-$wT?nNI;hDS@$ElPVh}H@4khyqx++1%~93mA)_8YIa4E_B5mq8X_7eJ0nM^5K` zJr)O7HXi(o#}5%(T6_pC(f}g<+RM&-`3WtF!?k-%2%X1 zdS=aW;Py01!ESp_uCp6G0Hn&#oeldKr0@U93hX{J&`0sm6z$2W(!WWx>gZ_wFg!NK zU?4#MIVXqn=s;SbC;*gB_4V`n0iGx){^D7%gF!}ctw)xuysf-mXL)7@*$aS>0HOf0vlOBiCv{TZ+l^6!Y_h!b*UHe#1ZojHHCG_!7|vEA1$0TMH^>rCePHZym^U5 z0t2lNvY-{P+<}p-w7F^4KTeT@5r$dINI?wRJwZg&37iuZ$ zNT{cer1c(8Pqo|m`-{uR6UAH4>)mMGj}Fz8^I4hD9)@!3at`o8*<#FsfG|r@*wEmY zk)ar`HhsQ!dqBpTFl%}Vzu)KZm<-ML{>1(nj^@oZ7ocr zAqm9Fin-_kZOuQt&Rt8FfYS^yn}#0jqwGD!QJWuZt2#ZD()iV0<{Pt8aYg(XMhD^Y`sOCK!cN!HL9E$5x$yfm4}q%MDJ+%&PMm-r z;4azK%o~Gnfx|)=Frq-fILCe0|84_xE=!aIfAj4O8eklK_B`$<24#8U`CHBV zGft3#Y?xCX&u2<@Zm`^0)2iWJjM}A}`0Zictdj+NWG=DAl2*sHuT~px^Yx1Wbnp0& zAY1wBu#Fsid%H1G895#TKX4Oc`Sm_7dn)F^jfSI)ySGjF24dBz+Mz8vN-B$k)TZmo;;dZ<5gbrO8=Y2Y za!ha@rCh$@b@RSGyshsCI`0F^%Mz~-LIU?KL6@9{Nh`P%y63NQK%9VU;g0B;(&00v znLGf|aG-7e_|)~<<9Nd!)aCJ-C;Wu%ysOUCcs{G;bY2c6J60ubk@tYH1(go*14u0` zT60sT?li~>oX_$gtgyV>#<~-1%|gt^`GHlshBUi<7NE*ls`M-UkwDCOF>VL7YJkv} zYU9Q;b9<8U-R!pR6RAl_JDnk&bv}iOPhn1~jkfg>M*nBR74>Wdm=zO8LgDS68I5!_ ze#5f4aR8MvzEKx?1TwS^cj8)A(|~rC0m~IKGt*#C$_5fwFxS}1W@gsYtf zY#tfl*nrVN@zqwNi~R@=%;1cSBoR~hHZ@-IxxXkYEZe5GXoLz39#CV5Nh=h?GLW#b zF==V)fR0&VHgCe4f&nsg4AM$VI}jiBCnWR$xA&5Huh7Cn@H`IGS3&gdgbNSPIO@T4 z=T0qjxU()V!j7`_!BLt{yG?0g(`ZX-}HUssnv|fd{SDo^h=%6APsmLaBL99k?n9H z56;bb0k^t7HaRiziS?>)6Qu58<@rW2>*5f4)FkmL@$DqFT+aY@W5%w-Y{mvUBy*JP zzWIXHt;lOJy!-pU(jWXKxFYr_*-1rE=*Q~3J&0XVGq8UX&@K>6*ZGOdSM z*)Ug?!cm$uv7^MBw{hakA*T{Butwe!C#t8{D$p1WvK(^?I+hMV+7PL65&nWB5NQo_ zHvz|6%@L=xk_d~4fKzDCXru;zEg;@`P?V1#$|blb7VveHsfnW+w*&Sg1<#P?#+shX z^Zbc>OG)WFlHVn_h(z<8`{&Di(4&6w0@9np=F+^B6sJ_#m)d6lJN6l~QVu*aP-e;g zxceR_m0&v9V2Oyn-j@NN<00LW6S7D&1-~!B>WHWIeVX7O1>11y*oHi7!1Gf;nFrUA z!k!QcRq%8nzDu9T0Pg6IRRurQSq@813zMOitA`7d31)3kGHk;A@q0(Z$TG$JCpuVX zxSt96K)|jqNCV(5r>k2;e9BtqtNA;?n@GCqpD%Yee%MEW84SK9N;y$fx4X(caKRO{ zs8rFGV5OFf_rEg&>IB#BY^2FJ-7|KPPMcQevoDiz@78QkT@f$@D&N%blx*|wMhE+_ zc9XvIw>YCq$Yq^W=D78A4EM+Szo{I^6*I#2fY>l(4@ku6%~@$9Ssg;YjIp}HfKcd2 z76fQKnO~pr&$|JSrUw$RU{CmtWwt0Cq2FsAm5J-#n8CaEt@3%>?Vt&oBw!LAUCe?2 zJdB4ask8H$N0Eu~fQ+K1fVeH%`T1mPN1JBU379u^I-L5QNGuk4$IU;W%Ae*7$LU+D zy13IwstUnBTN!*-Gu%<*)@eeQ(&_Z~NC*RBNc9dlcM|)bLm`Bc;EqlTF7XsBus15bha_`qjW>BpVlX`n z&JBr_OEi8o`IpcEURubX#SyxAqO4sup?n`_p%w!~&8YSQ@ei^7{?o;Gcflawsx1jhwoB&ai6@a#--mE*H(P{mBpXq z)OMAfvCDzI_79XAl_HTxgu*8kAfEQ}e~oc|yvmQYQ@$%$Gy{fnM^ zgoRngF)y}OiTd8cKOxUM7K9_~TQ-mbN06Gfwld$?Gk?8F4mb)Q?zy`xb-uxm9p;q~ zsM@|W56D1*F`?jG;8#is341rpo{!kUB0wJj2cjQ`n3PX-_F`SPn}alVf3beJ!O>Z< z_paNd`TW4*P>wh!ykRna19t0@QQXK3z@gQr%phHK$kY#THUC+PL~DPd3d*C4g)j(Jk!2;h2(di;tT3K>1C)* za7fc-1KqxQ7X}3+bB%+t&!?_iRO+TxTwGivTznWEs%~y>e|)gM&JL17*IQ~fSly7k?@dM<>9pCz{e}TilS4`6vig#g zlxkU^;i~(s`bR1@Hnn-TeFE(Eu7!b7C5<1sg*6|-(U5&kZhW*v!_{f|Q9N$do}JFQ zPnU0Q@uiO0r?hvF+vQdBm{-|=PoVTiXKkeyAKMxi=xG!U8H_8C{@%2ib15Z+b zNGB*H1rePly}Vt@}gfG-p~A77Dam{FFZu!9-Q(t?|Ovh zu79IFzn87psHl>y0Wq=e?~b`C|8LMSd6&iAH#9W0hBxFX&=e`qcr!2V ze^KE%%q3qen1y0GP^%tAc({tl$uWihtaX^pyQ=dd-YY3#WZ~wd?I~GIPDy_HfpBfr zpl$X6+%HiavDv~Y z@kzKNABb=6<ZFF#%DLp^8!B3;6`f15dooNH9`VN6a@EyRP z=N^;ez!j0%P1G9~Ul|v>jN;;`3{4_s2q6i2vBoM;+OLx53;un@!ep>5M)GbttE^HC zCwLgqNH^8iFh4RS+P@ohSXwK8pn;uIgmZ2vn(oo=E1m*Gch$mF|w^| z6ke_8z6cKuMn`$r$gPgrYrdHPjaz0oOr?Bc*0|i+ZsyR0G5kz;%GEHM6${;%o)RpB z9^0yLRjR+Uo5yT;-vtit?N`kc5f_Z7{5x6op!aroWK>jkJY@)pk+rod-{rY(d~9s^ zs?dDB)dKN#%%w` z8QPo)=O#Ptj;@b8?A550mcBc$xn{Wi)}mw7a5N%t^e*qTpP>HalpNHo9vrf`Suyfz zc`Y&|!KQ9C2L;v7NrJwSAM5PL5~R3m+`4Ux@UWaz@FG=^qj^(0ps!yX_c>EsvR)TD zLAQ4Ew#rER#IQisYS{HF{sXL6S#*=0^wjhle0)C*D)E?#anYbPzBec?kK_O40+oF)$UKvSnlm`DQ5M>ro$yB)y5pNi={qd4RtA)mTzp`W^PSDPT`Vyb zX62`;HGM}%!Ua}Xqroo1-n@~a=5{rqqD#N(7&x9!*4CUjh>Q~aCv%ju9(zgFoH28W zLmQFrQys@+6z4xSL(;EB7p#-Ee}=`yrQUArHKb|Xo*ID+Ey2A>h~?&{8K%i4{y5!6 z^*y-IpOUp&`Syjt$DD7ck|xb+JFf~s6mnXfzN8obH8|RF>_lNtoZ#TL6!%8H?M^^P zzM99cM2Z?;E{etXulM*g!NKO8++qqfe}Ts(O=%)dK&eto@)v-XOTjxM^kZaL7! z(BQtY(MgwvX5B~p@Mu{r#@_x%qf@JLl9JKJIXqN17jU1SO+_TM_z=#=HZc0V`clfeR2;DGkVfD9V1tb1`OG>vAx+!Rf3jiJ}w=YuT7+o z_+I05jcTAV?%FY7qh4KuR!rqjt z0&57i?5@JAkEE^@GYa1YThbo2qzRT%X8$XM_RA|6)?P$xAtmzERLd!m$3r>kJvG z&np$AOHGlI+UR82DpA5((d7CK5Xjfg=3>+;;h!#q_nXu6z$}NhVpgw6kI3Iqv17vO zkqZA(s~@`AJu_ib5@k~?uoVE6uUL%AT}I4LHmYW+JU)>?Y_O4S3Z6x7$V8}Qb3e`} z`#1Zpu!?RAh}%VFnZN>m*jB9(n}S5q(n<>GgS{D*=bMjFD`civ7(DHcE8XFgD{`w>cv(531(L>M(NW zkA-TPHNoIdynip*zo~U@NhAC3sda`GB>&7Fj5OetN|rZK-X|~=tokOw0F*n!d=N1IF4lW(3WKS%}@J0Rqu8b?+AW2$@vp;WWXt_vihIKga%cgJ$Of9&@Z(IOo|f zN(Q`G+^}(0j9SG+1~#HXTE4q=qW6R zivI>Q1cM2AE@o)}CK&0l+UFO)ozkR2&&d1%nm83D8Z?(0m%5BH-VP``l0i!K3%FUZ zqjK10jG1$nwfq0|dCI#hUb{6VbZwKAx>#M}S^z7(S7ix|`-s8isKNPvR((5#jHgU0 zSoJt*-2YkiaM36Lk#DJfYn*5-s2?J&pU3pr>Brfigo-Kw;3cO4r#NKjIW&jd72H~2 zMw7IKsLw73D{UkDV=rp%Zi}SkE-!R!2Q!+K1@JTUE2U*`21%h9-fiEWHk2b6C0~RLTDi zO#1{@h*Npkz!c1nts}cVO;0GWP&5KC3-!6YkjwVH)ZNR}G=XaL$wJUj+sbM>#JksAuQKmYh zz=A7U#aj=XTlf1&vB{ynrUYEe z-KE_JYiXyZX>ZDRxVU!R%`u$`@g2eEXDOwb?aWcVL-=49YhVEBzBm zV*lD_xHZ^hE_^)=)^hFx+s(MrR;UlQ-F{tI5^?7s*J|x3rU2(>K-6U8UT|4rE?d*0 zK6de+ugARmdJ~7*;|=Yra(vB=?h=>{{9I0zSH!u^Yd3LybgQ{6;piu-Ui~fg->>z+ z+HTp=EDGDfvtzb=sea+Kvs7&2wK3GMn%(dUX1j_i0)@!ZG`_3u@CJEjMqjg3#Gmxs zB>ug@=M&hJwZ0-Pk1r1tb>U1eo}4JZa5WOjmyZN^0<`nSt>O3FEV>*q2ex!mPC4w` zz$#a%yhS7k|V_JOdMKxAjoP0drc|inm}3CU__90r9W12sX?Og}i)dTc@wjY3 zU2`O0X7likTaod7L}o4%fhiu{_7o z8afwEQ%0h#tP`v@MnXy3{`VLr_hIfwJ@J%ZI1T`~Sz1^)A?6^$RX5buc4r<(bzgD+ zkgSz5vu{+TX}te}3Q}r>9D{=x0yY&Go4JyVtO@+z+r2`+dmyi55hDj5W_tYE)&BTr z{0b{~_CS^j?P)ggNG4cKX7(Ue+>bPg&Tch_mlI(i-_%fW$oSt~v{_*0TDaPie<<1h z4OXzmv^vR7R#ZtJ%s9L?4)h`)d9kTfI|_rsx)63h2vZhYw*uchd`>%slxmW2gP-3U z2?+@?WS^N8(tzDkGqrrP+0E@u1+PqbiJ)>(DJiwzs{a6EIqbVK{n%wSnl26nLu|j3 z{}-(JJhDIeVDrgIcHgdC?r7!DQjTCm794`nICEglD6LmX^Cj~V-)bxJfv(MZ(;&F> zoNo{dLMmstOE02pUFtDn5rQ952!PSb%qQ@wO5{+!H$6 zjKAB&*h~-=y9xL50iW*xy?_k|p^PW=T2su2D8_rv&*igtI2v0pgI2|<(j*20hAD)j zDVmNF;meln3d*sWfXDzr<__O}Dp9!kYqY;Js+|Op>b(a?Wv;C{HT(KRyuAS3P?*bv z)WRJG5MEVZ@Dp!7IUhBL=C*Ch(F24&7AXf9w9P{v#KMgI47Zv_pbN zXfB$(uS7s}<#C(#7&I25=_xr`O1YMdh!}Q0$`7%dWl@WYZoks`yC&|e^_J@Pe6dgo zK7N>{X-vk>M0xvtnBfXdz8kZ03_Fa193Qpy7Bg7%Qu1*;@zK*f4&Et}b9gs0S*^58!iwtd> zN_WI_F@ygJ$&?R5VlhNQ6CiqFXMd7Y-j~_@)pc}SA?N$Tj38!z@5AHmqe8K8(STRh)lX3LMrIFkC;%MDYeGd+D&0hNleO+!OF!arXxG&%WEe2+cEytYB;|G(da}M$tINrho*s8G?X#<(qGGBno>5vTMk`F%u}L*OQ+(b@ zTdCJ1xevwxriRk^v}9|54lU&D5!?5pD|fJ{+w!ughEDY^UD4m#bcjaVG2tr)=HVuUTQ~ zR@l}v-&PV7>Nx2vOb*pgGNp-2NXXC5{@e@+wH)1bWveeCHVRj%;QYVs86>DZw|CrUOo{%dQH)T@CNlM z6;|2t$S~d3{Ovy1v(0B-?=MCRQ$~LILDtK+x5eQ-qp5ooLpHyntrEK?I?;9z4%kVM5`c0nv#^TwTWZP$eZ$TBisy1V@yyYjO?PV(vYyG8E@ zL%a6bHbnL`bP~W`v!LGra^K`C$ z?qD;X__(LkPxxmocUEf7GpO{jyx*sW-W*?TQPR8#^iI`uKiT<&l~YzFJ%5{UnWnp| zISVZSXSL;E#7vkLPwMR%)C2#P<-t7#h3KsWsJW7+X0U<5{fmtO!r_tQz(+oU%&fer zS9LS%H-OLB0Qbu1&bOFtB%VUryhdq|R;#rT(aDO7`+RfUJd;(oxOjiobt#I|YH_)$ zZkS1G_=}2&>@E?0Rs&9|p-O7izXIJwim-T=M4~6Ftd>9iNqfQQswh{ztO%fG9>8gD zT=2JSmchJhZriQ`eYj(1D`{rDc*vdg2;nZ6_-&%RJ`3elqW>fkUSXOuZdR%JQp2WN zy#`8W9^#JwM=R(r#CATroDbKj`mt3xZ=Zj0A(d4=n|W{`f^V`r*p4(8J6`w6CT+J7 z+P^*?7l&PQa6wfDjjALO@$@AA{Z?MeqVejc`ZNYm&KJ9$P-@3Mm=d9+W!DwHdyXQt z6BOKinY%aRHg?_g9UtU0ZRb^6lmQahz|a%s4`lx_8kVWBF1v04KWjdhZ?(8@r1zhh zsEMsLOBYOZ(E@OZqu8pLIScb)`Ki1o6a2tHFZU{XLc+#lalK-;dL>o!O@7Dt#HWSq zRbN&x?efeGrCU1Eey`Z^55RaB(NJ<7ymeBl#Y-!4aTSne^*|;@vxH$))#pNYb_RMxv@TvPYi+t%QZ?rEjhK=E+RrneNHo zC)K983mRBog66gMFm@gmjE#vV#65oqr>tMI?GDhIaRQ-FC0bmtEeno0aJIk;i6)+l z`ZTl zrKjY$H|6vVT0E`?8wW?eko&>e*ONl-%NdjIzOJDPD!Q>KCH0@j%>2Vl6&aaJ^8+KU zS-lX;?iJ2PPcYG_;Fs#NNMVg+4h%Texf>`y5UiJr?oti2mg$FX2}l zMDjTr;c#x)<~JY3UddtM7IjFhc6re+VQwDX-cH8M<914`k@+ZGP>5V!MTWp3MZeE0 zn6ZShzqaA<5cT|0iURd%Pl;*V>SAO4TwS)Nm)C)(#>R#b4gpUbSqd0aYpeWBJ=2%Z z2KVpT*`BU^ zi`D(o*mX;QK+2+H&I9<~zE+DyF@<{aX^LHUDgfcm0E9yny>dyi%RfrmrK{=czKoAY z_~h*Te;9kus3w;+Y&bUDDqsWYpa{|gQ9wGV^xiw6h=71}LT~B@RGRb-(n}x_A@rsQ zNSBfzU8E+VNC`;)CY}xM{k`vxH*2xZULJTd&&)k{xvp#GmNm!^Lx@5Z6wEiA?NZdJ z>}|q^2gV1e{EUtDohAqZgL?R~I}wV}5tVMXopZr5TUOTAMVSrWeT30Qe|!Eqx0==0 z9QC^gpjVgDoe6diuh(MfnhMdi#nMtz*`V8v4)4>JuxpD8xHy(LinPQ1BN!~`@5uZU zka^1tHSZy&fTiH`*!ZH~uoDM$DH=dWt_>HoJ)U|h%wI?ywiU9ZFiC%TKZL8K0ikHh zD1ne?DW=+D{JOt2K~vUNVNVmB8N6dl6RcFh&_91R1vbAl#BuYc-^U3mQ;|!4zO$56 z+)93?!|Q9$tDYAiaiS?IbogsfQRl}jw68`!R6hjCMjr}~tW^$)D3^~0QSv>adZ&NE zP{~fO{3Zr9^i1Vf!?CS9<<$VdKFFpuK3$Aa5OhFx3OO++JgnkO;IRnorjAymRyOL2p0?o4&H% zrBD*zTWtA+&oMq50sE4TuOkL)_whrD(ICS5uYCa%${07-HY5ID<|5U@&%hw5sBldh zzdhW`6dF-ecH792{+ImWY~y~6pn7$N;W{ zPIY!S7nW%|$2w$daI&RO_kg(5B>DcFX@-sveZS-(p6@O)U`4z%4G_t{KV~JwKG6En zpzK(7Zzpd<*x!x?j!D>tSO&dkR1VGurUoAemB7-Fx~Gf6JdqaV9b2QFU87~R6!IJ} z>|R@UA$Uw4#)H<5I@`M;AbHC&XK!}0*(d?5e{cw>hgzy#0a*HQ5mV;2RG>Ke%98F z$NL2MMfz`jyO5klCTlv$Pv$q>yYh8TB{XvSpIsk_>bU$`nz@kYBt)}CP&khPDPI45 zL=90TV~U)4ne92T5JkT6*|fHmdpx~v?kpq_*2P4t8S*={aqA*)wmCAix1+T*zRYbEl2+1b;$#Wgrc$d}im zDZRS2!Xwp+vdi@1;w(9NvZnvx5d#UtmOL1NE19p)1<%(!VxWD~*f78a)u`y#0VY#q z1|sZ|UhO#Yeyv)4<`iohK(GqU{ePFgC*@T8ZCg5uirMunWxO8w@bU4vtqvten+y~J zVnbW~ZDkhHtEjGy9_q3h99u8(&(aoOBLWfD{Jh-tUwH_sEeOgyCDA%fso(WaF{{*+ zGZ7*v-x++oWpnR}e*ZRYwlpEw2t_QGBT2dKZ`1zsAc(%`?_7{*^OV9O%u5b2R*y>U z2wor*Xutgp!`M9pDL-(kVDwABtzJl3AdwMG2O_~3r|Y#_B}afd#L1i(QeNR18aI;f z9E?y}x>~FO(B;kRpXDN|=N^&;u_d@Qbj9U+H~)r3I!lzTz5i6n5GrC{)!$gSK8!iq zy;3QItktx!Ubd#So;Z;_+6Q0X{zeh}L>#s$HGy}eQ?A63Ro3xT7KMMvK_6l25DYmn zcPN+&-0-cPNqGhByqlYb>Tk^=mfKb8U9z`fJ!jvIEJ+Z8W#io>h+sr>wn%e{VYYEl z&sYxnA660oe!uB#1=S-cm@EC{R5CtCPDkRZL@_& z&UDyS&AL=Y>mJdz5dB>uh_GSaxS?sihf-3?Ni%78Lb&Nb0-w-^E8Ft(7ry_*=`>gH zB`M+O@GEAJdxFZfuz3?cZt|oD$J|e!N2^kagAyYzARyA48IINt@|j43V5;lW`sMal zha3u{9SZb`a8$e77!G4pW?D4PsP9zY&yFFpF>=!|gpRpho#FsvCW1${y8`s-hW+^X zDi`$kBK0&4gkdv)En2FJ6QS)E4 zUrypVSTNL;<_J-qDqRZbjTLbs|Z- z`i-`u=S&(?khO^C)g$~AQrMDD32_pUE4m<`{tx(s=gL9ql>nA3Fz)L!n9Xgm`rEyh z9qQRw-82mz%iae;haqkhrFrYpSplTZSpkcQPgitB44jVjff%# zK3NMQK|kAe`;6;<@N_RVefq-&$#|+W_5)~N_s{3;v)^RvF2tVu1oV0fykU*VI}Nv$+bjb*|aKaZptN+7_sx zBJ{^;BfD7K@<`u`fjPSlQILM9-NUY<_Eh`hD_ErhBMP#aytXvHI8|2xQ*wwpgLn zXnu3NT8XadBkS|2!y}I|Ph$)?kQ$|C-P4?yi>@mexsMhik70$564W}ljv?7>O}$+E z_|7TCj{4&%Ev5jAdC7vn8WMcU)#B&L8ywB7)n&OqNI95o08l?{w&z-@t2E@LTVn`h zSbxbrW2p2MvD$X~iqn*zDoej>xnO67x}S`;y$63H(z9oO5NAUNvm9x)EIxMq|6`Hh z-(J&T2@Y8O)OOhkF`N7RRIwt^;2YyQZ1z0HQTI+jI^T6K>y)8OnspOon+MLN4V!tF zw-@)3lqq^eEanUWuJnH&ryE4%5w1N{wc%MtVVrYUzRhzFA1B<8Ff2^%Z8-sQstaWB z;Wcc*td_@9p-es<1hf4;IO=0@{HRXs#u4KUf$;DX_c3L1X)k@SVOL8_KRGgXSy2}l z@w&j}bj`BylAF4ffHVKQheD_f)l!TBLm|k@^xGZ5$dIbHL<7)SfFAZYs|)HX3hLd0 zGZUvt=8=6Ex)yin5tgAj@RHkeaD z`jQvew``hwV}yYOLum~^MY{4r7s)`jidFc1z{0xDu+$%Epa5+WOxz9kp1b&;{m$P-<=mj>LGaCWS~W9*BEN=pplL|oV1 zJ+Gtl4f>n9E5=m{#3szxjm{6K(n4Ja5A2)4Bd62k-6}T^!JuA&r09mM=j-gXo-KOF z_dS)MQz8z#ZlBk%uh3~1_TfG z+<}rR+~5KU_x@VO!Mn_;(goDMqyM|&F4FylDMv0gHZCmcXlmY@{GQxfonn=)!Zw}I zbCB=K%x@N|W(V$e=HDrC=$5`>voilxcK{x4756`Vg}bi+o^Wv?3N$hYi1ud#D>C3MbevQzYBEY%q#m{u4>KN+ns8S{CN;qNaUC9jWc*i1jb-C2o9 z-(8rN0&z${4_dV3VSGnn=T}U#w|TEW7&q#$`=Z-WPl}eC_0c9MwmwAsc36i#+ZpK5Vu1Q63ac^8w!ebNsT8WX{ z7PRtn$;y|$`{Ip0jaDb0$52fs)A@J|MbN>vEsd_!^uE{|s~Q>@7Ld>dk3qb;qu2AUZwkqy{k=LuH zGCd2q4Gl$?Sj4M^+|Ial^Z992c_S6%|9ouRoc`|Q;}tJ#pv12QvkiPXzZOrv62LS7 zis3zr_SmQoS&o|%mEWJM?B%5RknJ#8r!YC;UQRi3mvEtHzsy7@VR=E@htWG zJ^7%S<`TpBPBYV%{Q8L^$B9~KiXd7c`qr(yjgb6J+~z1kJfBya_h})xYK)x8X26H3 zE4;~FFdKxJc-M^By)pwiY>ws_+suLk+8#Cfk6-lJ%5aRE85 z$qm(rFM0zpv!^+yRxrGYEkKv(!tZ4AG3sSjBnpf?)oowmRP_#rs|e9!j8=MJb}k~& zorA_&HCEQ3KEg=ydGWn{N0~+<5U|Q2F~zqD)Bh<*n+u8m%CT zzfLEV92L3{rz;@g($5`HyC5BBsZecGSOkpq8Bh=O$A=ScCdstO2e( zI9N5}YH^9MkI*x3iDED>H=17^TId@X@R;@*hajT*Rudl#KE8KvZmw09nQgu^eI%{$ z6Vw|s|7)pH96r_e?etm&R|9ZEM%3SH2=Q=Oi=`32$o~aYbUG zR*ej}NH7i72z1|8))uw=gpG|hVBYm<_b4So{z9%WWLXl;8{E7eFI4fZWHv4l8}id53Ve6pik*U}+_7z?%D z)B>7n*dR``Y*uYSxf=e!l#iHD#J9mV-}ZXz8W=G8iQT_nh%4e-5mZTd@$u)2Qulm1 zx{NT19&uKB7*9v1Vpe1uTV7Ai;YY{Aq!$ml=Dht6hMS#&g0}wG6Irp7a5%K1za6lKyh-U2hNU4 zAk8^IRkb2GL{30hID2^5Y|K>N$-bz<|{--YJe5!DOQ_D_r_qQk<% zus{3m6JcZ!?(vFi-9D9eM6|ir_n_sj`0idqBEE90N=0z8oV$>4Qb;fjh=B~oML<<5 zB$m{44tr$)-facUpEFmuhdx@m7zCdr8k*#Dr#EJ&u8lj7qV7V|05O-Unomz2QepBz z1Gy9cm{fh_qK{rc1%B>~kI()oVAatqaR8+js7ihXTndA|!_eo?vE=?<##8~^_scCu zD@dmo8!2&2)cr(;esX&b0W@j=xqU4$QB|rM4Yq%#qFR#o-YF5^avhIJSZJ!Mwc8|+ z@W->v&OB*f;t`+UJxU2nZV3wi&-_iF#^fm;{sg&Kot@Eub(OLkU!{gHa6K7zEt zA}PHRlC)$}Kow0%N%;lqrVzc!^h15a(oZpJ}Lm4*USwT9wKtBV-| zq&kkVKwoaZW4gA2-X9jkW|fP4QlFoyv8zyw22S69ZCwsMyWShX&=-|$zk@v79~#+R z8>6migy`d0P#$i0!A3vb8>N496z9ThF+SX1d~ym^>KGnAy6BV+z>usXzQkMN60Q3Q# zB&VigQXKt?@rdD?+i6OW+v87!-c47i$#D~l>T_uv&a-&UGF`eZTQB35bEyxq9I6gn zh#sTp^(nYVOjslh!dxlb9GeZ)s)qd0bSKZt%ZTL}-vtC0WaQ+X(;PY$ zQr!lpWTNDy)1dXX_Unc3ubUEJah&X5jQYjz-yd8E*o~ZwVL?@FZ`8J>OHLnHbTgbM z;a2DeMcL+pJj($8#@Cly3;B(n%3#AX`R!SjPoE8(31!a$X(+%($c-errC)&FK2!1o z82;bMC~rKQ@ylNo;GCE8z*n{m1`uSsUcA##i8I=otc8-djggKlF5U)p-Pw({d{d3c z0QD&+c3M(S&MyOi$%d<}gFD4P%^KDXFM)S!@LF*P2O%0{Wm_p<|1B`dq6i4-nnOB| zbO9jQF}gFJgR`wn_jAbDIUD@MU=y^V0u(-@?{}WKxR&5$#=V|#@3qJ~*e|7K0TzSr|YUjyw1L#0W!9ioAMu1j>qFHdi? z-nDhUe!JDaW;W0M0s>4fUhi-(_I9UG*Rl%~tTfxB(DFU$R})l=iW zoQuT@e`;R8l&Aw}%;Djrq9V6Dv|kS2AV-ny?SaX(a7*g;!(Ez8b$R=x z@$(@!iwfN{?*8R_9~2L`WvrgAZg3HN?81a7)m8Ekbl>pJ&hv$t4Yo4N|4CNnZ~rZS@xvbnPpN_N%XxVTN9Q;R79lJsF5rrpCl8?tK2LHsgb-?) zK`J#nKQCL0a@+AHqy%`0%n3ge&B-|UQ3{^G#dmW=-148=UWHZtf|L1cG7>~=giywQ zhGR%UU!ODaz#m9FU0lm%-Fp1Y_S>yP2%yFGWS{f6b@F`18}^w8FCecUoMdVGE1H>7 zK{583r9nKmLAXwCONfbi2lQ5z^V35AVeG~&Dm3a z`O+BHS_Q>(bEoi==FZ61qTxkW+|(d9)h>HnsEgrA(WF*v5Aa8r=#=EfDfOS1B5I|y zVLCzJ|GP=;c?1T0cVC75{0m40)g#&KO7%T1D3UWV@su10f1D(%dcg<7x86 z@aH`r`0(xz?LTCqbM#Dn z^v+%!QfhZlpWMb!b$y?)LpCBkB)?@JJb$=9li9X0B)PZqtCuuH$3!=4ccoqh*fTA7 zvT49h!OmF{^e{Tx%rV1KdA_;vAf_jEDL&fb?}lGNqumbbO#bDjWa0kMM%a2wz+W-c z^1-z(Yzq%yJg04`OPza9RSBJXeFH6L<3HuTKQ1Z8()IWKU2gi_LnT05}bo zm~1ps4sPH*snNtw9i~WzX^%BD5JLGh9DbCLVGd@o=`OPM+tC_Io;T)6Sl%D&p2K^J zAudWvB#O;rwi?!Dl=34}B>mDxM0;>gvQ6vjZ!m~X}s9o?o zk;l)=5eE2@L0XSoTXD(`Gm4{Q82}iEJTFj6Y@XtcEGsMdnD6BBd0^q~;I`YXHVu%a z#U;e$rJB|k8G@ISkM`c((RrHz?F;X@CWpUT9a%W{0$1&eIh$_*vgsy222&1uWlC-k$2TnF~jFFe4#-YT5mPU4r$kf>dcpDmeeW(s+Avcw30x&bNN-x-%Fzb+C^x{_@K=a#z8wLvauKa7BCjG+R5owp6z1rn~Qy#?9@vGJ|NfjjY`ix4JvlI4E#> z0Z+~PC$*g0h509-iMI2hVSru>G!kdV?J}~#OY_<%a*3?ku6f$V?2i50?|Ky zT*$cn=sNBS2&;rzNijB7xv_DtHZl$$3=)n;h|87xL&FiZ?%C(pTxT6^hWL6{)J;tr zflDLIXd!V2yACn3X9{kerwe1u6%Z2I(`x_t^UFnkxyThddm$%EEg|}(ejk##T8tBJ zrTas1A#jU@IXU{7>Ww5ML{-b|`u43cBmG22#>(; z+c~3s*|)4~Y18aQvdw7qOU_g^>G)y=CpRevvRkeoA`pVxLn z$6?t?FL5k2YE`FH|Jj&^#(b-@rFG=wLt9RsTT^F^3!EHrE3aSF6&oB9Jn{y1ymh8F z_89eRixk<^?sG)FvKwmi*$DD+R2jY+E<5X9i>mU>HSw~{&+1FW;pqs~z0VSQ6U>Kq z7t8J>wGBBAKgE0-s~1n4Nv1iM0!fk|OFoi6k%0ptRD z_O#>_AWU|d>45Xb)%Vj~cUb`bB^Mo=kifUR-S@b3>Rpp(CN~5t%y&JF++fJ|LjL>& zTu&z@``$$c;mrIDuW@=Ws`F$ErEDgHhwG7z#DKE0n&Mca8V!teBRfMZ3$Yv+OCPiK z?dqMP5__R)orWA~C9eF55)Ljdt+?6HZg2m+0Xg*0kfX$Ucco-sz-~Z5=2feVVC}?Stz&9k~vm2>+c(`@J>OnT*h9>pO0)%Gz#;*=q1o&!1&$!o?yLZ3gXf*xq z@`UBF2)#|e*1jX&)H?&icZx}sHki}N3NLG#cEGu%>L)m_3Y~^Z9UZI(b5#~hu95II zQbOWg<;1=;xGVrWK($=Il(np-t!>7b;jpA`-+j-uJkIQ4QPSy=rRDqZ?LH zJ{x5r8DEq5pn(7ElRWH1&bm~0H|4U_ug--Sme#rS`+)Y0xPZNd%IS2U)woKX`>%D5 zv;c%3T#ic7*C^(IWd8fZ^|4c1;|SA(Ih&U0yZbmnwOp~o2_H>SyA!TZt#W7^X=4O+ zH!fmJt!9!n|GpO-)mo&cs1@5ktkWPP8 z8Ol?n+vMO}d*SUMb$J6%Z`8f~)mm476U4!qsOg}VT4P>C1P>T6A1T;)Zlm3!ZnJZ! zxq~G^W7dSxT+l5Hss6NyRLV=tLFW$F5LiM1R&8OL1X0+`oG{2?Cimz3yw?i#&|e3$ zd09tBrtti&h2;wS&Rh1QAApG7JWU9zXVZR? zl8Q>j^Dqw8xe;(M&!p_xP{4awH$U!B$yjwruAq7SD_pDFo|D7C?3`8JO;OSHx?bUo zwGW#6Oabc*AORS3E7GyF3@v~6G}#b#BH_Kd?*l^LfJidwqulH<)B}9Ijv==2KS`SK z>8xk&zrHAl6T0l?@U<&r2vEuFaocnFbjHWBUs@cvaHp|Eni+AGO% zam(j$QF0ktERW@*F(n2(9v`^C-8X|H=uXt(_qzh=A|)OZ<0bY+f*e=wZgd9h#tfgB z3${<;?j3U(r}X}zc83f6d#zkL`labX7(jZ{16H|V1ky@W^-f0(&3;(m@LXbxwZMjA z-(CA6TWg`0S`|5;7ik5Px|B0;7sr)FmAYI|7A&XIxM28pSI z#)GM$xP+AKH~}TthG+IjuNyY1Ej=OH0@)MZ6rD4xL%FjjE@*^k+{cuN41X&=ICxm! zuzqUE6K}!ySMHPdHqXu`3cwTWWMDz6{J+qkYChOpVTz8<-eM`Ns**QCj0@%Vo=EIf z4abtlW!&KIDu>fTT%oQTVG(rx3=#M7&x-GabJ341mS7BfHA{ZR=9e-ZkN7>)b9TCx z>DL!VR$Quqqul_#J{CY@MdGn9G{b@x^JT$ib;d{|{16()=P{A!;%AkocDQ_}u{b1P zpA7~()5ROxJO#7?Xe=mQBX-BmHu_ekd#+hB2xFd-fE?pPown&APTZy1nCe&0iPV8h z55+U5?Pzcf=9i-Lrk;o(EGY$ydgUH{V>nN1slE{8Qp2szyjNJDV5M8@r5* zio){BH zYS(+?Q5jGIhrLmqTWiMnrMHa24$$2ZPMbo%fgg~r%fAcNeG%JVF!mp6_+(a~%K52l z6PK?U)U$CQ_NP!aej{JrbpmWfPHygocXs#g)ftv~Z=ZoqyNT#Pb)280PbA7uC)(&Z zu$i+%w6q+5!MdLInKvWVnZ>XNVsd%+y@9K()mAdTV4Zs`mVub%0W}FEg+H7jLC;K& z6~bf9U0OCLUl*Z&y~GajQx{Y)%3NZzd?tTZjdv8(FM(tuySWu7@P;axP&A}!Y~%B; zHu%bKKjAkLANxKV zZq*W(xRKm~cto54lyxDvOrT8j=*&Gh#!9?E%+exH7;{MbW;*pyI+hPw^@m6)XME*V=}|@?qVw zYH_$aoubv)UQfSc`)tY$1EcNw5Nwi5TJzYLF>pd!zAbz}d;;9loc{W|s{--vm zH8dqcJX-(gpCCMEi4v-K0kA1hO>C6gi$x@XX<^dJgkVz;|Hx_mY2K)MtGtFkpsUj{ zoHi(8ww@>+Jl4-bj8(x5Cl0l`+eWhp)AMcCc|NX^8hnZZ1D{LFds<5@ckU*m8CEjf z0`hK3>3PCYJw~}M|G$?uChCL?+?cNu=rAb5*4Y#5kK~Hw+Oqu_1oqbCYWr)f>aCM7 zK5!6V5?7BIGDu+c3*t*>Y58G>OcJ!C?DPa6R)*{n?{=PsMo@!4R?Ua0plYS*>d=Nu z2AmnqqSFF@LX#^Q;ESm(=1Arg$R6VHI>aHTR{aoh_-&3invqV-_pkyNpxYx0h5aHc z_b7$et#XeQ%n&mK@i0%>jGH`;z8rzR^6KPV*xUjQgWw)<()lI?vHls&@bC#+v{#Y# z6i<5GAK6;qb|7)w(XA7MsAOMJ&J(=Qp4OMdFCP2}=NL)C2*A>Fi`I!-%cT{6T>L=h zh}SLfDkZmub;5gMj8$ukPB^LZGHM#1qZD$Tk+*m{XXY#4W<@(ydOVLc;mX_~fsIym~--yMycfi8iO=((gJQzVuPo?ZE zV(_y2Kej;QPhbna<=1-B8bRElU>(o&)Y+8$*=|jriN-K##da|_*bHZA|8Y@_E)Z=p z{rABj=tnnCPMz=81WthfAW~-nZTlLAtC||RoGrsp+LW`kDI!+$NvlM}F^+RQ)asr0 zv1LPIh+?9$ccnu+H9}e)o$ukaW(Q!d*hr)!jdLm3)yEmG6!zQp z_S+O(IetKsHgQYQTnkrHu@I3id1B0H7OMBDwtqs@fV!w3T1+-!TSB<4760<@KWUv) zAyJ{oS1iF!)>>>Q08GPb{9HEcQiR%74wt#7FFib2CLUQIl%tO&u@JMJ-(PdTLR356 zh_kblI83l%p@T5SYFJREB5cBz8)yu#;u z7!0h7_M}E2HzyDCnD*SQV{#kV^9%J*IR!Mz_C)X2%-UxN(U$|<(*)>pc8q+A0-u}F0X*ZjHm zeOp!+W-MeG13uPD7MkgzS86ZtXTJaUswDiUOOSfRB}Bug>N)I;eDTi; zkMMCY-Y(*o4-^;b;llNYY^lL`tsHYq7)k>zmx9AkhTpr&N^bp`Y(C;09XU?QSX}8i z0AzqxAk2i(Ny+Srh%%1(YutwKkiM&`fW65}4V5O{ z-=m6z!>w~*6Frp@n!0&vlZO)tu`w|M)5pS4h}nVp6Gn(~_SNVAc)*`lp2#1H`WVC( zx?Q-XG;v*}Es1A8Kys&qCK%L8oF?k*m7}Z*BCdH#jlU|{;^g%u@lW~l0``ZqLH8`K zfqXz^EA%&+=}YclZvto}m9TZ}Hv_99!Ks@$lr*gmkMuyIvzMi?X1sMW(@8T;zg7Qw z0kR`ndG)I@6eYQZnrV+IBp%B3GP31!ETOLBq8ZiRE-e3 z->3bebHL>B0g^UFutdXh`>{elsL=Zn$MXk~r}{nYP?a~BvpILKW)ltg-AzmhG%6+@ zMXOeb@k~(JEi>_268EdaKTvF92kUW1(kUN^dAd=U3O>MfQc(c-!cKHzM;;`}g3oF- zM=PEwRv+FuTvC9Y^+pj!xU@A&^#goyv$59pBUQ3M6f#{!U=SX8XPq%w_|eJp$tpuT z)(wjoMGob0VR8r(Y&8Kk?T@_b8383xQ!8f`Oaj0)l3f;s-{Ac6n{YUKtp66EIX<^r zmFSx!=`tK|n+GC9l|*_&aGq)}h< z7Vp^DW>Glx(!73W#Aj{-vw3Ckwe+3sZ)-Z0HT2xmH2v=X<5xn_IA>TJqyd)3jJ){dtR<1h z`ppmV;+@({CihBuhXK;%x6yt=K~pwZZ}OnEo|A_snkCKF0o{qL%f92cs;ZzFa2V)+ z(8)#-N*0nN;jxup-hj|mbl2S3ZVnhU6Dm$H^^vQtHUI>nOvgId4@AAC@7%2tBs{Kp ziukevoydAF1@EnT@ghS2NI5>U?VDC>U{?SHcg^9%Xt}pe97`b*X&QOB$~wM>#m~5@ za??#tOc2>RmcruV&&C%?q~?IoQndqSf|LW#2PpE#i3Sez@j5}mWWZg1i6mqvIGzB% zc>(06iH#-~R#Xi}aYmvgy|3D)z32DEH9z3{Jq~_I5KUN#D`CNBayx7IF1CJ4u0|f( zXc@v0TM=+z@`AskzTFKpW5nEI#xYHAOpl4T=`{; zMU7N7eg6kvVpz>@j=Civ*K3GDtRe;<552JQ>uD!x1Y}U)04`3)u8jCd_R+*t{K6>jy4KX3oO2d|#*;*w=Po|W7+YRFT+q?geHeX`!lqp%$? z#+vs|!+I8$m6a4j+3MSn0lVcjZElItIq>_)feP-z6FfvV594~Lft6KCFJr$;C4RkH zczDkF_O+PKS;cC(+wr`}uoLeJ045d#5v4!jeBe2_ zscM@$sx@69^{*TRnnHQM0n)#PdxsL$28vgsdzhXtWgQ)gUDIZ~Id9h1SGGKj-$?(6 zFLk##;{&XDxxi_#h25b1tJv}qKEv(srRh!>^W~eC;*ydAp8=BAr^e{rH5M@1ZKQ3S zN?QYf({sWw*jde!8rf2&+uI=09({Q0i7}W$AR>uhCyHxZGhgaN;tj{3EgN3*B`3F+ zj%mbdRSaoskmTiCcMyvyDG^>dNKJhP^EeA!t?g6xtgx`b8r3urd&WEL$%a||OYTH5 zV&>OC)omlTaZ?SGWQ^a<%Th!hvWs^h-tImw89o$QTUq{uB^CvmN9u5w<+W}y30W~F zBycO@w~qz}c~Aq;^b-)}>rdoW{)u2+$(es=4t?6aZP(Ic3>B0DX-hkvz&6@a?}Ef)4S z20g}Qh3T6Ex_od+LxgsOl+^=xeZ7|C&gu#wCWKK9ZvwOilKU;wIC9c_$xuK_&W@0+ zRhD;7_FHYMsi`?>lj+H~_h>fy%oy;oY>r)UEaIwO!Esq19N#Pj3EU1L?%6ZMDwR2- z{Q3aGHgG$aZ6bdwIKmLaIer1vSNpg%quw5Wi*S(?_cC&mi2T7P#aQzPa;2Fwr5_6| zmfQ7S9{0!J2l3MH(C6Ypz`@LAz@u+$*47e#K%nZbkKSn6br|@NYdP;_yTjQ*g;&~| zL=Xi@z%lnWZseJGZrcmflY@xmI*A=%;N;>GUa3#_pCg08!gC}=MQaHFn5XtwSt~m4 zN8s{MipCV1ASlt4B7NWWrV9slMjHd<6@$M#A%Wjl?ffHIgve$~Q3QH%-)+Iu{Z5VB zz7w+{8obrgHjybGavU#ST%4kKBF4zqED<-GyAyf1{16O;C=-t-5)0ZIy5Qzs*6{Nr zl$0~r10TnMqBtEYA9p;0lrDC@bkui-~I8){viUJ67Swz{{| z(AE)iMEp&_>+A9HM@S-hSl6k*yYx28b^tUo3%x2EiVMHrIYODv$qiGJI5;{y1!|Zv z;#n7-M@I6v^G5%~Vp9-{RSZUdlp#g`JTg$}0`Tcof3>|VwCluPI@49P->QIWt>Lya zx~E(CxyA9pu_u8v10VO`O?$|kK*Tv|ommBwhU}9-oBhzvgFS(MO&SU$JSP`klge6&!f#i#DIVZO4i@2~8_){m(Dg{C;@BN>o@CtmM`tar>Ifd8&1c z5E@VwgMSdI@BP2$_`!bSvbsd2b~?bbn#MftGk%qe^6oT@^NGUv#EKQWU%v6My|qx^n-$&g;ZGBP@hlO&zGOzA{X| zBh2>gf|)JgVr^fgmloS^^5_NmME$!~t`Z|6BzIdoh0c7^G-`y|vyNv5(yV)XI(Cfc zPKp@)3&wErB3Ae|)Q-jzYbOCfHrF5?( zwuunJT*XkUTgaM_@ksPTkE4scz;a`WkBxctIXBn@Z%8)17D%(&^YsLwu_#-+E z_vg_~($DOxe*$*8gEvc&<%fvV&tFwOH8rQEgEM7){N~S!xc&M}s&mdvRVnq*%@s*; z4`5>3e1?~K?<`S|rF?MrBO#u`KKd97jfzO8iH~NtYUPK7I@P*O&O6e0Ob!J2kFV_@ zMl0?7pe2t>6VtsMLV(6EmW3!bA`;gI1@uk6noP=2IML=x02(<$DQsh4w9L_G=ot4O zOY$AY+$hQ{Z4o;d+_Z~L!YU09U?vLVG34t6dqGEZT*^=f$UlI_?Cf%*RLh0nm~+9G zbwIMHgg%vcM->1yiO!;*3db`m`u@FBWIk|Tf)!Px#SjMIxRbc3yZ=*0Ys($CB>_6k zP2ru{gtJp8tS334){GG}=`imb21vRBhD*jPc@>v#$0MF#>62`cUu_}fDcI2bqpI|u!Tqd z3+sXKq}x9SihX(oP3Fsb%u({fjevWKlmE=B07`1TXU5{oe%>GlUHz*5q!o9DS1{Lq zi^iGR%&(rssAIFsO{QtjnZ3VYZo!~)1U*5^?aW>NZ_p`05{wg6Pl;`3FQ&au1ueS2 zvejNpK?<6d>XL_8jP0fh4L@U?pPlWJFM-TuI4RrWP>2#`d#kf%=S`Rtm&j2qm6&+L zBO8D~x)>_XWEB;4bWIZ_nxJe}&}8R-7ZO0o`LX)V>A7lxL)}!-xs`!$Lico$b;VC) z?MQe{g|wOEq0o<2VrC=0^ul>%r4pSZ?vo+_djF4$8v9;>{|LA&ox}`yx>*J&Pl;(P zNFXU*Q^)&_B5EJe&`db5IDh}{3~oJ?Z~v~{{U*0eQ#4TbVMyhX7NENOMqxu+P5gYE z;)lhezY=}iNteq2##Z-Vj17|4Q$emlB+d`!sK8frp-RmQigGP$oOgeXXTGTS^WGU| zn8Vv4i&C7586n06!FQ+Eh?t~uO68|r3dO&Pt_8!<8D;)&pEXAgraKxOI4fM)Yw_dc z{LqsLpoS6O7Fxf3_?9xYi#L7=I;UU%Y0r4vWj)@Ykfv#D2y-VW`ycniociji+%MpM zz9dQg+rbUARz!KtQ&3T1LnaU3kdTm|y(;8_&Uk^Nz)2f5waZlxfrgP*4TXbW^mo^_ zJ;N)Rx?I`dE}S#F?XGjGi(9nWhf-_oL&dlrTgbxN@riwinfEuwS0ZdcuA;=nH!!i{ zCPF}+m4q*+zk73DffJG_QL?YuTJVenxKwN(!wT9*N{-u(a`{ZnCnJmhyf>qf+?>kU zPwJ37qp$B9tN?r=?oPf8vWWQ7p8uE+G(kF~VCQQ7Mr#UFVXf%(=2z`Y?zZL{&pAPf zKc@y|s~olTcnz5=`1?xCGH~n4P^HDSBwhZ?YFpif^l0MSzx4q_v(C0p1vG#wCOg{NB@&hh+r2ni*AYK+^>>%hTA z=Kviq(Sd7M6DOeL<#AVPD$x*lx_s2 zLAqPIySt^kyZi3r^Stl>`^LThdxkP_Nc_%@wdR_0?zQc~+2HB_nQ&`{Dh0=9%K0{| zg**;?ZV_Lc^sgQyFNI9OLja*KA$dAF{Y*#0n&LOP+i=;>{qEtPNdGhQ#(n}J2UyUY z6O)qImm5I~%u`OH;2|eX2nm;B9|$TE4t|%$%x?@n%^s*{Bl&mzPn6b=3ch)CvjoTy zr?L!S9S36v`B&IVIp+8lxoIYpPX4biCbj_noY$U`b-c;Pk|t&XtUFf)ddGovq6~}- z$w&MUD^g_ub{PK*9wkRt?DK>5{@>TVDJOZQbRrRW%G1trKv z{`UeO77geuB@ebgxuKS~{G`6YIQeu4@qf)ylg3l>Q;5!nPw04Q;`&U;4jkiT*k2Yo zr1$?2x6HxjyJ$+X5!q>9>rEQ6nc8>?FP|`9vioDun5}bFM#E}c#HSN-W13VJ6kbH< zPb!%Fuf0kPVwC-^sbW=8XHnM^uh+RX92NMAMI7v};dN~OW+=M()c?LONIwhsdWcSy zsIcia)g7Fae|jekyC@I6Swj6po?Xm;X8PsK*m$vDSV>x`PcODp#h3LnY1Y3#WI7lM zJoEXP34&mm?Rna60hGPkLiwwsRR88Zm#ca>?of6g<~ROS+-u=XXbbI^rtv=K_aD_2!_1!o^ zhSN?vF9gR}CPS;Rl&J5|WXOct8vTFLV&_fQrrLRvfTzox?Spsh!_5!__Y(22$f73y zZUqcvqtRdMaQO5V_Rkm#iMX2P%Gj6Plk>s^iRhH|vN~P}WFt|M7R`R^=s6Uhg=VW^ zg?Dw%MN>@J+&|^zJ(G_|8VH+pH-ZM2E5VHLM&E7NCs$_|%lZ_5!fl5pvpTAfOOVLF@L1 zWEnahp1v|8JK4N^9zhaB!Jsix8iQ#~c&lqyqb)C3YXEaBPbR>fE9}ecf2Xa#vdP`z zgn69Hv#MNEd!bg=tdG^`HzlYTr-sod8w9wsvSiQdZ!K<*t@rQ3>Fn?G>OX-g^QsxV z5FGdZc78LKqMdY9Kf5na|5KI>U53lCY^2!p8W1QAmW##k@8CCFsM&Ogx>f^U<={21 zTdV+3G-~)d&4msc1a|w!?O*Nw2OWq9QDuY)itfXR6cZus=s2!@Em}NtMEJ38N4*@8 zh~lsDJ+UWiC2Y;ipPV#1!$-uXhtyxhCgjhml$NbUIBU)3oLN^+#j=_FC1=#7gju*B zehSa`TY{cog7gd3;z)v?7X%yjn=|?g9HiNa=@+l%FOU}XGo<|$Y+f^2y&h~JTHv)= zT)zD5ei6&J;c6FB`}ykAPqXhswwzJyCRf4M2Qz~{(N>geQk+re;SYNRtNeX5G?Dd_ zZ}KQ5`ScHLAIvDPYCm~=d_8&_A98Kawvc?xPvx(_=iYf+|I()0K*nWZygB&sX%*YIZa^v*F?G=7=!(vD@WS(KPGask@jj*D)3_ z8B_yAknSRcyR z(azV#isO?~<}6|w?H-fVwD=y(FF?-3!rn03kH@M#-zWsfV5*ZnE|7t7^OcyQ%>6Yx zpVcEcF*11}eOl}L;#ve8)NhU$=Vv~4exM-Iv1tg5Sa`*4{)x^SYm4^K^xs&mu@GV` zyx+21Bn?*NmzID(-~9f#`n5@Ce?w=7N3wq}pZy%toY@=~vC_k6-&Wf16B4cIL;t1} zRr(C-b<&BS#tz_~SyBN8_Sb(IZ0&~k>xP*o+D$;L@%44SnXpV&CPSYZSq%*mMdCZ1 zLwQHm=2_oAGz2d*AGKmfG*9!O#4@RXZ9{?+_I{s;p!~n!BpU&RJN3pyGuWZZ-STgJn~vS%kd0VkoMzIhDi;yqf7fd1 z(`R+g`owcpzV*!KH8i8`Iow=m^ycgm< zc!LwKWEK`93tHw_^ZZ0lD9MK;57;jgU9&3us>LQL|M18^9=TeDe8O*VJmV8PtY|zu zU7S+W-f`h}T}{8p^4O5?w+UY!V$7;`CXo41YOwihE2p6|hrK-6K!tR3{d^X_nXR|S zYDN>lAX`6BpB)@TVl(8cXx^d9R%z6YAc=;ei2PvC$^gbhnNkUe?MHfPe`=jP2Ootx z5lxwYkbA7{ARKZ9SP!qerjZ@(f^ry)x9mvBJW0qNn=?lyOiaQge_U%ZFm?~GxP@cm zI4Pluyp@#w#vsQFf~~8HM`lb|%D8TN2G`+$b7X8ac~;@wgHD*r0)X zQLE4O)#EAQ^T$w>XOv~rd}dee7db@nO7G+3OPS4G2S#e1gQ<@@ zNUvPR1b)4cQ;D<}x`lx>w*w4$?sqe=`Ysq-bFxH?_ICPY)6)XLUNa zdw0AHHoW(wK}}<6;=5b1#0y?pnxk0V><)M$=lQ|Yh0PQqjjWrC5-rOEfx|}OaD~UC z%*?SnMz7#is}Cq#(fydAd_1@CFx5IL_c8Z>+#RS38;JbvE;q(gTwx(z!TRbXv~-|d zScwcct$zUy5nQGnyxc>M>jG*4I0TV_U!_gq|Lu_#LZWmJYzT3c?l^FaGM0Xq*#%$z zrPeesR*rrCd{5uy{=S@b(QM_IJt-&(4Hr+5?ERQRi=^`rd1>j6OvM}DOCqu;#vi_E zlMX&*Rg_%G6*-GHQ5>| zo>vBHu8-HYShRzu{Z4>2?Z>Hn2NQ&NG67EDiRTH76wq-LH{{&z&Dgf{Hm**7sojzA zpnjpFcdq}?nE$-ruVG z)xA@~xwRXM=XE!vdwj%(FgNhK8b*5Hv7M=Sn)4qvKJNacxM?ZOELR40ol|eH_keTT zZmavb_$W}O)4@NY6I^wBe87>>g?pXiYnT1_^)W3fubQ?+D&&xB*OuO3Giii=NgjU` zU;;YU)cE!H1T!-|Mrb~tP7b*-SQr;xsgpXQU%Ugm-$mx>DD#wSM3@ zRa6hBkJcLu=QgKzToe6xEh58zc-=KMvp;!$U2E^&IBy6*VBn!gttefL!M11PT$TXg z3$;OaQ~b&0exu(34T82+&SSRYl4}4Odx!$`~=05^%k;c7LoxrRL$GE8HDQ z-X9TZx=9BoQS3yVKLvujOO2{cAu;7pdr&SfmF|0*X2Z?ZZgl(hf~G)=ZqXmHry;Cfjzv^Q8@$9fqDI&nLG|Crd$ zuv*>wQ<7e?(!l;x$Bu6jW(RCZdet|ZM~KshXF8Fh&oQs>dBcfe51mm5U-11DOg!al zN)ZqJ>hvyq!7GY_$flPgmsyB|Wq%jk%G{YyvF=j9FqJ+}nwCoLW%W8}#T+ZBd}3P* z4;_6Z_D3a<*v;pEV-tHiKp;!US4WR`yTr`;F3;aus6LPCiC|mYzeuaeM8;uKN-L`s z!i(vF+na7YPA*CcI1rfNfKSUxfW*;j=9AVo5gcoObSnDZQz|^LBm4KFYc_u8Z@Rx{ zR_>^UglPF` z@g*({GP1Nf#-SY1f8fFzaRem-UG*bJ5J7wH+>lt@N{w?f!jS$e~dE2j9v6HrtRT{m1zBQ zeN^KOVJhR%O3J_YJ*U-CU|R)M6qR52CG82Ru724ontft?p4H^8nI-HepN`>6iADcX zkr`iKDXCvfiGIsKo@_D4M|vPn`k$M<*&Z_30VBRktPxi5T+FTUGH3h&1wt0d%&SGr ztF`ELyK~OXd)|(Xsd{oSVj8&Rp|tveD8UpCZ+=@9dE~y07%q+%0Tl(oN{)!F*p`}8rl@H%=NZ#NpL?v+=I|obKh_B(qStVUgaJjc!u`hv3h8Yr2ey)hH0_$ z#n8wePNzKBxDvs79$nDibE_QUc$*ORiM&fR3d2`^I@j}EwyJNoTNH-PMRB@ZUQy$7 z28J(aBBaq$wCx#z@UqlTi-b4MZ)BC+2{knxq>?p!dMJF+;F1+cpwFLHXQ>Dz3BhzZ z{js+ElI>+BtGb4*tv$c7ZSS#lKVe4ymWbyKTK;LRZXUX1Y3gKt-cOBIrtK7f1;Xxe zD~!{cMK+FEnPjMyY~t*9VIcIj@_^15A{pE-_q1FUGv7)&)Tb)`d^0XXklRHv zm`Om*nkQ>4pE{i5Gm{bKpA>x@1O^uWNB8(qJ>`7L4t{*eZhG<3>1}4IKGmQQ<;xQ5 zdkK2&w;U!Z^C<+)nRH5{r>j&$B4}$)7rBM8EHzijg-QSX;%vE>0Xs~rtWUt@6Wdx( z(L>R5|Ez!1CkZnC;O~a>y>l|pDBFs=3o+VQMo_Lw(5N-*fV&W`zyNKoiJ284Y##u z{uV7%jO3Y(o6)RDEPFZTSCZz<)78)_ejz`xBs9OB1M3gEIrWM3cUPH75j_URmWQo$ z#RtN#F*-M0QC5pRPR#KZUx7zWWFgU zFW8U94v&n4_U=6GUbm5Ar13F~K*Z!vA)cb$Sw41>rT9@sOC)2_sdOzPZ}A zi&83O(lI?+IlraPvy{~&CB17Oy zv>7Q`+Hu?d#Y#+*GkzMbS*SU)L0WTS@XhYPv#0L}IPEZAQsQF3)}+efHo2@5awb_o z1pSns)Ci-*^gSsa3J0eHzucXz8~GP7z7rMIM|dJ9%9h2>?Y;DoBdJW2Xk^K>ol-HF z;kmwTbQJSDxrqO)llh6yIJ)+5BiO}j1{DZ8wgJW!!NwH`BU{gYO(Z;A*hjIf$AYfJ zWWGsyN{Rg-fdulVv&{EpSTFV6Q(0~78Pd>?(~0D_jPX>ZiIk;w) zetR!1lB=k*ztf@Gz*5uNVP@$TU{Uun|0k26bt!zE!=WViWa~9+>3$@Fd{J>)8BvP6 z$20z>Cd(sSu=aS%oRP5^W4Q`0ok38AMDVr5qWR(VSM+j2yttU~O)F@1XTLhHb6Op} zhAfS5OF9|!)#^4-f>pA;on?pss7;^GT!OICg$Md?%(|_o&3-nFEfR$TJZlLu| z{mGQ%{}Htq7Lv&4i0pZ@f(GkN0(ka;AtJB@eV|Kl%+w+eg(3<@;Z*<`-uQC!dPeC4 zp0fYn&|gc;<-Y8po2|pUhv{ko2FxH_$zGz0{)ZO^2+d+x^pseLD_{0*RP4%Pqh)EP zw~|K>gWca--^?~14$}U>un|H@SwsaBSA3DueA!e*7@!j2vOf@xj^dwD1ODHl3I}XN z?{8G85p*SPGjl%rKIx?rW=4aWEyq!t-cG?Jnet&e%omOqbG~M=d}g$iF_;v4>T)w) zmjZKid^m7icHGDsjHn<-UU=9eoo)_zksv)>_N*R@lNQC@GjgJ4I=YW#f0X`{HD;1J zX0m*IH9Wunp_Bt19l1-jSQhjaM1{!aE6L*`9uJLx9JZ98fU|Ud4LV$843$S2flCX{ zT#_*TV7=bGFj^U1r=2sdTv=JCeVPVR06O?+ELG+V$;YW4kK@$pWX2r%#TaB?jV^H) zTrS0qjP303XY!xa-UHHIfx{bYo9}<7150L}7+bgJakEc<+d|4{${Rxhg9%15L;^q= zX8a%W(>;2Mb1O%>HBu+xZmM-PUnXt0wd`9KIY8H zbXyb-R!Ad1N5p=>K4*d!z>yLi53cCm#y+I`e4Xv{+UVkh)RE@ZWD;yz zw3ru_>idtc#>{NQNgKKBVIU+yq~M#U_oX=r7$ME|Glt@~^s(Au_yfuTde18CIhZT# zR!Sy?L`5$-o~DUK4i*t9DrvuOCidz09{gP#e%O><PI5X<^w^R}__|i+VtX6GFF0LcPkNGb{&EDVlOr$E8abrT$$OzxW z>hq2~?wNPL!5{7?CFcPpGvAD@;YQ3+( zK@Y`>`~cV07^q~10mtQW)w6AbblnieI01Y_yH$!sz8Itq+bV9ee!o}0<^F~bZu z&BC|J*6q8e*>eC|=7z=~`a+nEF6`Kb-IKVKDjFHH`@XCD)vZ-4)| zwt3(|h1q=b`-*Qv*PVz`lTWU5EoFlxA6y05h}>RQ2VGdL1db!HmA?_v`5pm{Oo#Ea8SNxdv^u5qgbRo33Gf}Z>CJ8vfLf7BWj)c>LbHjune!<9y0I^ zkmLUNFe@^p^S>n2v;!$ z!2Z5wwMRNu%dY^(g~e}8x7We4)R;%tu%rB@_B(y35{C5DG=8JRd#}lAwL`L@^I>W6 z>Vo+0DTsefP4}-R3p9M}&2ZAQy8{?8`P|(^m5bYz)2$1TuG-AQz5)n)0u?LkLgzyUG3sMWY9yRHX$zD z>gxo0G%e4}9?UcWnc6;C!389KPe|j^$NyNuaf&S+wu6!#aRn>M<%#r9r*4|Xj8-_p z*{PK6EARh!0njr?Ckz3=j#p(~bF<_v$QU#EC7Z#IrL7tpo`sDM`PFqmpxgz43>)hy z{Y)Eagcw@plR&6(#q1wB9`E(-78P|7AwTKkLnr8P$VLw<3|U}?1nQ6}q7XAni&2wX zM_w9{29YLSZNN?zeM%grEcWEgYZ4@R*3Pu`5`AR2Q{7j%U4UHcAvozuyU85~fRXny zi5-CoIIEsSx~lXws^zAUY)y|wI$~lwW3`l|kjOTPnXhR85a2PefW#Urw zWjdQL6Eab^JMjyhP77twL&c6}l10S5a~iaXqP~6{+H->&%aVqL)L*vXvCf-XTuaHj80E$JHE5~L*vkv;=C24Jzv3h`I5@> z0tD;nc0onFSD`NTM{_C5I~CmK3)S4Zv!D#5{=Jq~Q-Pd@`*n8zhuRoCtPmSpB`joP zfCh{;fl%~~&nnfR_;S|l3<5+tIWV*E{Qo<{Qx~?PFPTmi?kkKCqKcxU z&!IDgT^Cb?7`}*8ZTSbIvvtnwk(pmW(NX@6L8yO<+CGd%6@u=0CdYPkZba&Mj%cbl zd>;5?o(fKD54DP2fs& zXYjaDQK==ocIFz@&+cPgyE960!EE^z%VxpWS&NH{^4;VOe%$Pf=B|f$G;_wNRK8Lcr&Ntxe1g51B5rMh*%d;B?x;G~EH= zU%7~nPn!2^i3Q@t&w9)%$td$MU(KP>QN$rCx(e{MiL!=)kaZ-*dS#7m@9-R69e#FU zgCwG8Qj=_;F`EO6#c|rF=eOMkEK*4<=zvRs20@S&zmm*N@;lTi+)#X8*elC8XZTNM zr_L?I9BCL|C+^Hh_+9J&CeS#MbFImX8H4GhV63G0H8de3ADEV-dUDT(_6=w`JwXSo zPC?@5X4d?2QY0(TQF7(XcU6}P{mTWUAZ&V7*%eY4rmB=J;cD}2Nh`rr_E&d~u(vpT zxwtQ^z9l;q`^iTMRVfJr!;>?@5#u*C8O6f#_8-YV55>RTy6cxBtgYJzoX<13uQclI z>~AYSfF4~^j=%YRg~R&$k*epl*xpv4a5YsMAJxGJ2loR2C7O;tJ~`jqe=?fE)u6;F zmDSiQ+d!JzyW^LE?_F2BFaInbbYqx43=J^m#ka<aBxh$K7eEkSw(m-_tl}mEjA@ z`b@XrvMB>CdHAfc$Fe>OTSvZnDx%#zxq3{&dpM!4+iN^b^@W@)tu)X`QLOZVg77x` z2M>3hw^MFkTI(89As}L)#{G4L?-z0ZJ9}q-^!aiaV4?2X9yZqU z$EQ^9-P?`)rvkZ>P*A!$>v>=D>aj$i#_^iYsP<^cRsAS9iNeXL$htioAMw4eEr6&U ztcUfS!CUqM)Tp*nL4 z%IqA)ULGxdIU#=9eEcOFXRB)o zv&D5?q?)2BYJa98b&gwSINi>WK>V$(-G44r9F@%|5gx;^JfJEn+V%#-7-ePl5BhyS z2S3ItXc)9yrre@vIiEiPnqQw1H6sfZoGq9=5S<@VnbIRcWt4XO)jLOSZzWR#0-m0P zsftN%Kj@JqRWXavci|BSr*8gvl$@vfEhm9{6VfY%*SSixnRfmWZQBxmr0t5gqAZK> z<(2Di=*kt@76>mir9f$~>OX zra+!ypCoE`S)jpBjDV6P3aA<;&(pWb?mKc)^pyZPK&$yWf%$Ow)810UF+6&87MyHp z?XO+CZ8N7Tk&&q_Pt0Nj(csuLEIWx8=by9>JzC7ve>Rw`(&NuF#RJK!sgx{=7>j)7fxL zGNY?6X0T&9-3S6HZhSz0IO&oR$sSu#dq>#F=ZFPiF74*syO07abKWUmc3PcAA|~}? zjYByX4yY6Xu);QTHK%W4iniajVW*6Foz=;xS@fL<2dhk&6eJNSOf@XG|FD0O^#fZ3 z?M6DIMx*qXe4aKeghU~Rlx}9E_grOQY`>F&9XqBNw5~rJgO6vB**V zk}dX``6ZgH<<0vI!`X$)1{rZ}2djI6F7gN0_~jspMiGg|oqfD&uL_#2vWQsq$ko{UAUdD<)OEgml+dD&dODq?KpnI|LlKEnf2QiL5=tC3B zE>7`Vl49+5;oj8aaj|do@o4GHhC#t0aKCanY|3_o@O9Kxwfr*|uQxUC{bZ?Lhn^c~ zl67=_f7Q`ck0KNy-lal~sao+Fe&`VceIB+#%~y~l0K|efjt>M|H=0vVygs~!jgxvB zC*?OfiecxdEeIAWRSI!uJbT~E4F@|raV$hCDdx1kWdt;{KF7Kgfti_4fUpI?l=EhQ zLMn-y%VbfO8p&60YnSsA*9>H5>G($0wSi%~!B1G%N4ON9;_RkzXb2plwX?HZHhss3 z&7>J^_2qoLO2x~mo0fL!h&3Q!i{GH9;T+KuD8j-ZTUj>TQjsjISnU3+cs}de_W?z5 zvf9(cFonr#_V((|oiFgV>Ts9oqE8gp!_%f%JI&SPCuzXj$L9MI%eQ0Ak4Axmefd$H z8~!u2;n0Qg^x}p?vx?1gooQ#7JgdrG^QT0!=r;mv_s3sRu+C56ZB_(q&k?ED%(dT( zww2qgV^IH)P*j?1-aYO5#$Q?@r3TxTfbAB%4#LMG3pK`+CZ9f6*kcWeKMJ&8AXD>cve1lakqI73j~wt zxtemTjecjaM(Q6QsHV#d`T93_MbqUpnihP%A%v$F6kz5nGj#0Duv)F#V7Yk^)6$y$ z8Zj<09^rlV6{)cB?cU5OJ6f@9wdq6|`lrQu=W?r6F}k#oXDCFx-a5wLI>PZ9;g`qsi^#O2;o$dQ;xHf_RM$~Q5LUW zsoBjYp>?c1)Iyfd&*9_FZbD`k-lxn=i7^qNh%m}XfAMcSW-6;)Mv(K{b-?go^Y~-~ zlR|LS$$d8Wp6ZEVO=%P(p|Fj>uDQZfdM;>is6;G94^6)CDZTFPQ8kYLmztKl0os)v zx5$>2@9;L}s=zn!iR|n~SLVU^Ce~9Ml{Q&e<~}GoL=KjF9t=e*&x7TBfyA)^NL+`j z37^0KzulqABACh|??qxkP4|~}+zdg81G#kCYddfD{Xq}H`1nGry8GC-6;90wDlkg| z5B`V9E_X3nZf?W-Kj&stE6m@5%nPIjKxVxuy}Fnk=PNUzcx$QZ_jA&rT7w8v)#i=v zSL-8o-rtz`#A=A@>Qvm^F0$O2#?8`jAW3odbz(*Ik#RcTkO9mrBQTF36hV%+|Kg9b z^5pVy7tvRc(X}6(s2=vGEXmQHt=wJrae6-3$osYeO48?*Vc?I%wG-7PWzueWt!#x3 zflY%l<5maBA|mckg2VbI=t}JhOX@scT@nI?SUa#7CDE&E zwWIFPBjmXlpMecHV#d+BtZJVV$0|{3yfE%z78|;Ij+|3yDJ+?fT-@6-* z-1FI6fy0Sg$*zG^hBP}>Z6xEh=9Z1|Sys&EEb-=oJ=X@7w2@8KC5h2#+JS=K1-{Uo ze!!T`6e1l11cvhmq%^*V`Zk)@4UCZ;q%?+zg-ZJdO2ghaBROB+A@b1ZzRk^uh?)U{ zq#Rd>3Q$Gr*znjt4m1Lcr~tOk^bND-6^w!o_*c0_g2%04~&Uy*xV3MgY9$$m*7v zb+$ZBsm(QPfqFToNt02^f*;sM{y}+uK1WmSWUR@Lr(L*yAUqi12H?GuPN@kY?`ndF z0Qz9M)Lo(1Sxlea?RAi@C|P92N%#Cb%64gwarM4CfXNZyYAQY&v~>s9s7cXii+n^c zdGe#%{J_;dhUQvW!Jb9?K&!W^p;@we`B6W1UUkP9Wq$bx#Y=W(F1(~*GZ{MuZrKHG z+4&Bm7q5*kV~!kP=-LMv+XY$$rjxJvINjF%A67b~G*9}Tj{B38L=NT?ZDUd>osCya zX5*W4bpsM{^Ki3Q8IK5OLK2(KRK{phA#nV(Z_i_C#D!oDuyy)@GCtV z#E%`)If}W5cJ%ZCnc3+h&rD4%f0?_8w_ccGf%Ysl8l*~owMp(bixswi0SzFw!6!e? zs&kkRD;If2KJ5fr-@9LULmUm@qF(erFhF@^(DD{a4nc7!ea^PeG7C#6uG>$ z@f{57wy2dp@%Zf@EK-wnk(RP#GR2}rLV+3$ur*JyY|ataaHFA+P+wN>E93Rx_m@i2 z1=lfo(Z%lNU0JV=+JpA$ju@dM_xJU*3y>nt4g79LBaJ?#;Q~C&OW+c=?Ner4a1mqz zN0G2*8!~7C4HeiB!V@EfKL+?m0~HYCqAAH8y5M)B^~NVTQd3LeAMYO@2+=GX5Fnu6 zgldi31K~XvS}5BaC6XYlv;eDaPncw565TzR0rMP#J9{ti+dDa`1inhElxS0AtHNh5 zr0fr(6%>)EC6P|wr8BKX7ynFc{`K_}A}39xkt~Tyv2d6WIw~hkM7m1ruZD7__jG+o zPW#&!^;d)ELYsvp#>bBh%+AAH|I8)$==h@8D@*P*MnmScjq{?oWTEo6rn`|br}Keu zfJzGK6KvowvPih@B@ z4oI^eB(h%>1$GI=v>o3%0IqdeD<-eSK}^* zu)~p{@voQ${ZOk}X3XYdmP>$V4}`d#orH+EIPbuYpLLZouQ6KAJFx@wa5EKxe#tV+ zyiXsSVAOHcT(m7DxC#%JA8HskY#Zxp`ha`8+B*=N!dl;iqc3Enf-0 zoLm$rwm`M8%86efi94IB?pxD4`tL$iwy)7^$#Tdw)GYX_;IAO+Pv_&19iXg&c3y-A zn>ylHt1q8L0Plj;-|%H8gm&%oMMZAkzRG~OeahT~b>+ghAZ%i{7Uq)>33|E#ALp;M z^L_t$Q)$*=fj{2}7Aot0zAn@pZcO?&rc_mPr@1m&DImO{u#_Mgx75CetdMw93q`Cx z-pB~lW}Br&8X`MV_@+u=)_}kO(%c_kau5WBx2kED#t)EE$rpe|Ptm%8ls1Xg%7Crb zxJ$;OQ>*0-`wh`q{+GC~#y41J7YZ9BiM1`P?$F2R7iD%#&g~5GX>DmVwRTM3{@ok_ z)G$z;cw=sEU${eX{FiX*$P|%}*kmMB>SW)UG5PcY{PIxu7FjsxwA(1`8D)d<;qgIv zU*MYr+y%KKghZWm3+ie$NEA=u`&}qbkZ9Mo^+~gNMYI!Y=R9HA@p1adVSU@Mi zah4fvo)GuPkTU%KF*3YR__VjYzhNbDhQ$9c;=6K`0_S}ywg&RT2V6Q8Y~A?pSdZXg zjjwpUOB7pZlt*ZMq*a)edwK{(>&SiH#spxipcO8R;8dsl!jygOn;3m;DlL)ArzzdE zNlAE^^VjAAeSq4^6Y#^)+kN)u+ zrRkuKQ{>z-bF{M)iFnaH2;u}-kid_To}P}6Ey)OygO5Ze!5KTie(`_AR0|2Isk=pb zGGL7>;&qz@il?afl66=FbJx0BRc+aevSwI|OYZg#ZvxBo;>20Q=J^f~IP5h@b)1`8b8k1?%jO(82frzLB&y{wj@u#{JKc z%7w?RXOZv9R?J7QJnNib!(Sol@X5oZEG||p{}26q#<5-y&9eUdxp`$+V=l1NpsmI7 z+mku)mr`tl_Z)h)T9+b1`SQr~VIKAPqAn#pwUokoo|eMnsA0dmuTT0j8dDVx1LoF!rFRx8c@s z@a*ajBK*)B(#^5^b*ClM(P>j;=yZ=?lh_K z;JG!^FwJ-D{8ogr7kZ`1%o)*@V**^V1BSQg)_i*6Y*fKMx_p`lHt;sqv4g$DTKnwe z`VN*X4X#bvac2H3${T9oOl4BDuhz}ZTE(bJ6H_CAidgG;7po!9S&*RfpAFtg2bJRc zYibOWPqGBZ7x3Ai09uA8CB3=qgyqhzLk;59x88DFccIsIgEbnsx!6=Gi73Jls}d&Q zdL~4&z<)SU#V=+niNAvvf7D~7y;}MOHIp}N=D=&>`;`G z;N9me><-^+)do{^`i(i=2JBEqqpw)T$c|JYh4L}v@0tI>yW3}9lJFmAc|tk=tbCl4 zU2lHmG;ZsYEISrRVXC3>CU`>-iVzQC`Jhu@twU7+%6s~)@eL>v&T;hhI}W=1N%BMj z8BrBvsFQPvHQXQF15!v_>#s1oN9ahNfe;yItO%9*1@4X&wyzf-H&Zeat8Sy707N7K zI-jpOKl@5B#9x60B;v)W3Jee$p|5#M+yJt)H&mRWef6HXQWjAh5gfI6M{ZWHLTK5~ zTnIgm?z_O_4=De3?1ZJlmMj9M2?!w3iR+IzAC6Nz{Q<)2=$n`>GeK6QH5`5O`G-tL z*gk&drdr`sGi8R>h2k(62&g|Zz}-qzOk}v2Fc4NFsg@i6pOUwrE;4@{^wVwPm>Xv) zuBj-lhzHYI(d(k9sM%F6!}oQRhPTI~iR+;0=aB{qO#XQJXeAMlKJ0EP7eWUf)qIJp zmrlf+_{ZS$McVy66&%#Nju6PV0#s~v0uU0pSFsYgHOT&Gl9@}R7c!y`Bm?u<@xo}Y zA@v(-oymNiu&UhPihKo$FFPk-A)E8bP@v7{f@VV}@u!N0Dl^@ka|;GJuhTng0d z1TA9n58uHD6Nk=ls^mp+(1 zsr=FZ;g7?EaORoQ>dgYTj?=ciH=MgVnd&2PjS0o zhxK0iWm^l?Kxmn$UI=)YPyzDz&uraMZ+;!rEqI;Wc9oyP>l&{7x7GtAW%sB9#MI8j z1oe&cVl<`QJ?VIX=pC4g(%GcZRe+VEZ7HSq@DC5Hk#1vakqXZ^F}7VrEG+5!Gv#R% zloNI=AH6NDxI9kk1tdLCl3i@*AS41z_@1|?I4>*BD)4|{hzGore;rl7@@}}Xne^;a zZ@oG2nvVqyTVv&h_5PY?M>x-ya~TP0YCbNIJm@xI)eYLGb`XAD-O#3g4^u7{2Kx>6 zv%7XRjeu51F2`E*OH{6HcE_fB;*k&5|L@dGIR3)7HK(x7`{(3aFg0R58LcqdDwrZ$ z+tz~l6CXI7f%!mDagrPjQ%*5fpdhFk*jIqBrC<^TH1|$5kcauLHt?lk%5Qf-ewc>C z6_&f!MShUSrJ%U>IGD-YUkckyATA~*6k&>?1(>NuCRPOjF0?75nwwPnZ;XVRmW=zw zP2;8x32HeAYS*Vu{p1fEr z#aH#LoIIcAt1z}t`AdWB0o5Z`${PZZM(^1NH6ts~_EcA@E5yXA*08AQd7cot#M+TO1&Kv`fj)Dd;jvAqIA( zLwsKB+g@?noBwzLlFz$0*j1}_ve*co!5w$1r!;@w-AZs-A%GJK+g6~`+kC11UCVP( zrOW!_PHAo3P=7OQz-e_+z_0Iz>PL1!2MM?>A4hW}ot<$WfwqmQb&JSv8n6db0^e5_ z5FXv?&;nxd(BlXmtqJ{e%bksz^O9(9%@pk}I4n?ao->i>^X0|H4@r)aL~i%z;pU$> z_Sn(>35e*S2uPy915b{E5zsqn9JPT;;ppXlR*UpI{YB&l+*Dc4_4z)B_u6$l?i)!} zbYr`$OIFoO=O~0lE>3L{^qMRjCT=;s^7kg=v{!c?zpEB+d_`+7YssV&t8=-d-|@o) zEA@PN1>DA+-?NG!rsgn;>h3v}H?k(D%ikwnM=wQ6tD z?^zGJ(qICC3n*{UQCc2KnfYlIvJh04l;m)?!*NK45$LX~2z+vVwP9fvZth!{Z!I1$~9${2kTRrW!-7rd>(f@)OJJ^9`c}g)k$d1lg%!I7m*u zrSQzz_YAJ{^MTRTxw!&${)>U27XwzD8b?ldM=tK&-ME}fKce`S!)7V3R&ky^~96~3Oy8O+;V>3SS@Ub$SfToq`#wg3<=NpROx~?}h};GUakZoj4twg|J8dc(qnud^}z0|2=F9-$$*{j)+L33KNK2 zg_90<59i^)m2CB(_fD9k_x}zr7<^Mf^=Jk4jF#9P6JTO*q%XVd2u9D&Pwgly7N1yo zzQRN1CDNAfp!>HpA=86K)mT}$0I`={H!1<~?4+m(HZMwx^_S|bZq+S%TTbp)+Vy>e zfpJN--U?gV)AbePXyRcNNTRaTK2S5{F;9|FG{xeCYd$b>L$h9h82NPhCAEY3Vk}#> zqdQ=C&=zi&UYO6+Kb`rMm9gK|#1#%)0@kHD)b`XNV&Pd<2!rsuFFUiSPu@t{Z$2)9 z8J{;R9gad}S0>Ig_fS-G=QKKPAq+y%KjgV=D(>q7_X9hzQ(@U4AsQH+?6Qu0xH0wA zPN@EmFHR8ZiyJ)!g8CnxIvB|RAMW_r+gm+LJnIFVQIbP1=$&JEb0Ozb37Mu+4UJsR za(D?``iPSPzcv0)#`Klm^Wa---}?!Z=HmU-DwfO>vuCYKFMgLI9aHQ*mr0=-acO51|Lb_24qApR2^EYNg1gcj;;gdT4 z8w|NPBmwv9S8E%eb^m2v^1Hk_9**85FVtX}Rv@BezbI~~!B3Sn7cx^}@X<(~lRs4c zF%cJ5lEI63y5P^cfoBrxn`|Ho)>cwN=f`TS9Ig-Y|AQ5&N#c1 z^qq+_V-f5v&mhoaFOB?mcma!r4KC}gJX$>Snq7A>QJs?|P$y)lUaA%5#9IN=l`jW* zvu-)Y=c`f)sHtc6wQY9<>!YG_kBF zPS9bm?6ze?*Kc7O3UK3oh#wO`nxJE1!atwV|KZL@RE)`AYg3^Xe>z=x-sWNXWP6eJ za0qx?%1HCNuie}iRVH@Evk#+tRktLOeB~|vQ&{5-{Y9S?u`4Ta5+B_7-;DVz(k})) z+B{G**6h(}cVUL;nFI&WsSK{{w44G_Nj?}JVr*g}q6_f5^Nw8o!-fT0#q{v>c#!8! z$06yn6Q3GTjJMQQNQ$l9o3NSI-;GK`1DznINVQW9A`X*^P-=waG984|eS?KawaRXI zO0C}bEvOHGZIyuA2G2h%(NCjD5L_cnEAkuwX5cT2i_1OykTsGjW-jbIoyh!Fk{vy> zxAiamk`p65yH*^8F8?W%N6kua{GNc|<>OJ%)XE zVz61FwfgAflaeCpsBXE-1EFT#MB!SrIL!aaRyzINX0ScbeRZBMOt+XqBtMeKzs+5n zdLCT0AebqiRq%FV@!`qh(n&>phV?%P$V@oDz9Ju+oQcV`+y2}#YD$1$LK_I48imTp z*Dn9UQ_)rAzjpeJJhV&+x!BzP=x>fhcZ?p^$8cJf1rpO2qiZ(2;1o<`nB@q$E#Ndh ze7OR^-@(n19H=%h5THae@jepnJ*+|U%!Gskf(ee zXKhrax2yLhmG)58N0%B@2_Ny5^?{K{gecuk;*G~c`Qh@a4C*gHM*`X3s`_wn1xP^2 zZh9}k$%LwUkJ_p*_#(>CQs|5bEem6S^CmB4io1gwCqL8ld6a!B_;51xxh zJw+l(H|aCsUM@cm7_#uL7@NVuXa4%yupcSgB5Z*TXW6bcs`J3KY5)tL@3*X{U0bU44 z=3N{O!s%z}_nZV`?jo#Yk<^sf38^2g(>|fW@jlG^!$9We9e)3=)bCv*%bgIW1x|{i z%g1Y~Cl*8TIs|sE4nLm@xLG4Weq~l5&(BL^;w0#@>>zlQ(-^av@}93LL%e{3*mka* zqlgGjOyNdid)i%UVw#Phnabd*r^N@sa(y!gsN3<(NjRtUM()dbdhLLJo-%P!g|l^` zI>E92M-FOcIwH4tixIu}Pv2OAv1e}7Z)xeCvD9CdZ(kkOsUM-<7Cw8I_Iws+J@432 z{YB|FUFtMfGJ)blMm)$^KT|E78c6rx>nHc?vh^Iac{oM$OAsExUx=8s|rT zKU8ZCEJmFy4RaSHq#>ZhW?kC-?dZ_8VRz9yS8OvWh*hjXWa(BICsGvoFQhfD*M(*@ zOzT*zG?#bjJ^DbA_rkhD{GfTw;gvm5+E)fNOw!ep`_~&A|Wk;NK4nDIUo%N z-F4{hu0zM$hkNDkd-Irat~1Wn{o8x(wZ8SOZ|!x9CMVu>{myI3M@$2-#z3=jd|H|m zo>aN!Pcbpq%1475GX^Oa|37#IN}pzD=bz8b7CG~p1?ZG`$por($eEc-Z*+FFBzncK z-ML{k2?TnNKRtBXL<9K(WlcG_8oxtHt?y=Cb}z`Ok9&0xs{(27fTb7{!O6k*ZxoyC z6O@ZUK zjOVqx7f?PBC-1n(;rb><>jRL4fG|;LvMCQ5Lf_RRv5b}@xNca1G)aNN{HWOF&L}=( ze8^9+E?^_cxuc^p`~z`8#M1R-ivD4>WfZ#Wo3n* z0WvyE=z~gE66aYQ^4aAC&199_DVxfzk+`aR7{AZyHLJ|014=I#l3v~dGe*ULj&~ps zOh$>C{Y1a|O^Ft?5p5qm@|Racd`P-jt9X?wHvVy5YH?<(hXu_`Rrlf_&!gE)(7o~w z1l7sJDh#x*YLS(2oq-1P66s|FCQ#A*hHx)3}T5@Dq!!PfGknvo`O6J#OQc$I)^ zh3Z+7n_Q6nhhMqu=v`h1PMqhQk8IEm+gG!}Z|(OX4jdgRA~7O2;1PB5_j;xRgI+kV zeC%)rZ+Zimcj&&Uw>Iqb5P11d;*)i%K<4}GMSBxU&4`#B3jwlh@TB^-Y~3N>%~y#` zPRlb~YUPeC0WjHA`Puns#~FY{ZUd=1XjeeRR}5b*4Bcz>%Pi5`^_A&Vw(Uxgp;A3I zpj}4?(m~zRw$(eE>e)(DRbiIj>6)JYhhYDsw_7{BpAE=)KLbUalW@A3-ya2)f2e)= zX5auMd+(@;ZkfY8lmL7wI+h#2&potH?!XaaVoqsN_@`*953>ZHrM+e#5nSpX!RL#c z(!%y98M1T@4@Nnr>ys5hJ`Mugw*$IZor0gCn^wXAKj2;@?*G`G1V(`Ut#Y6H{VRD4 z^kwE!HP8+VI(0qTAmMcA=e?$^OWUdB{2#~3H~ryxAOlpx5PJU5)?%cPDCki~WetbG zo3`i<-#6%>^F!jpPs(LJ6Jz74--~s3KYv^={sSZxK8Yx%Ujw=VMSo4NRC;LFUHkeV zpZFyHF;`N7qy4u(h7xbr1xdK}X=}}&JqMb;yga*ZE_9&6})0MIin-2#1eF>P@O0VSEB%^S5TXn%HhtNV z;#XhKd2ifk+J*p$)e#_^@>Px3p7{HQ6_19-=u0Epd-j#VZpDp&#ISebk=L^DYsspk zXN(P9HnFEIZ&MJkn@HNU4%>*a~5KW7O+~aek0cuO1SBnodcD8;#avCdP z3lSroPE2D``otTg2hQ#!o)K3xs)IcD*bteT(GirxE>LT^OU3-7ywEWFW_t_b+Y5og zmHhIKaU~YUd_{B6Jcg=f3{Lpod3qx-a7X@a3$blohB2|ZIq|3LHbHguJHdW#{e8Ic{{~Sm1PWWxnQ#Iod z5|&-Iub&DU?6XHcqbu&97gWCPZ!=%5AKipvmygDk(gQY(Cuu02IlLEiX-wAC-c2Eh zXskWldG-|V(Ue;kUnBm&AJ8CdkGQN5NHRqnYH=C*X3f3SNs`F(q*!6}++z)Xt|9!- z^^2{IjK*-RN5A|&6q!k~v?@QjEKj5l&SNJ7m#6G*zMDL_AZ8SY6J7P5$^$lod2RlP z@GDutVL!2ZY-B3gwn5L~_WE!x>ZUJOfPNFvRL~-gjs~Le>_LI&hq|NTT{+ql8G4|r zn~hR+q9h6RO%DEl`bt2Hnn0qT)78R-XYX8s%CRwKSw3M3+K8OjN_+dS`AA8}LC1p^ zIG4GI*J|w*H#(6=r{7iNal)E{KsRG}TpZh!R#+2Y6z=*^J|L1_`(X@}!UJp}-B|-b z`$nKLdK1Li)*STj<797QmAqyi-+dTJdG2+*?Rs)uUR_=EpC7pM<)*(#akQj~Esz4booc=Opv?eiyJHf<#Ob?Hi7yC;-#`z- zzujy4u@jeqi%UQKA6sN==bJ^TxsVRSb%1zhF&-q<$CO0h!+4TW#9T_mTVqEDG+w=G z;8TG@iY9rzxq&J(S&6pL5vdQ;sbE@0>blJe0R@{H-2GZ$5d9F`O}DoQ#b#NC@rK(% z@uHB2Yr8@~jS2N>LIZ`?PaTj#n}&A1nyv{9`S%ni1hiPRzeJS+L5J*xfZGA-AG;iL$c2Q$4EGYhNr3D35kMYIXwMCj}(Rz4X&7-Jy$dG-kzSuYYvj zL`!XHz<2T-RZLJkAtD*+o9dpxw#GaNm31k5PT;5CA6I=yia{R!6NMp4vHIw4_v=LG6Hk7U!Z8EOA|U1l7L1-kY1vV_%E zM+l3`hD5;|(U|2or1K}H6l;X=uw;~3IZIO;l2;gX!Pa-RYuV{;wcwN3any9@pOV;N z9*Cwm*YNZ0|GLX9rf6i!pZ>Ka4fKWy*3l&vslNF1W$wKYws?XZXK26FJQ;H+rM6i}tovK;Zp=z#__( zbtm>)Kjl<%MV!WJfh0J;a-mQMY8hfxeU2fa;N%px;c1%g&aeCHwwk3SY2`&j=DZCz za)lTY85Nb!RnkVK8#hV?R+g>`eZ9LV9o_J@+b@KCWq@N)GGZ`)lxW9WW5CPo{u_PCL0Usb#>9v-@$!<~bPu3f|JSvikBg>f!Q? zYywTyIDVf2B|rK1gPg|EhsBPeMt&H5jZfCzpoOo|u8@vsZuZu4`_mG~k06%1o3OA+ zH@2H>HV@lLAwBKat6ERq%l3g^v&Dy9MTWDxM|<@?bOS?U_qE!Z3)|NBn`gII!`AvHRnL82TKT8>u@7?(mlHSkHq9&K#X=wg()$G$PcxQV%Q zRE=@-)OEwRoDH&|6w{IH89H-k4QFr!sp(TLC3QSX8+YuX9o$%&m7J^2l4*`eJc~#R z?X;R-FvMCW`R`>(8{SiaEi?tmU(MCmh*KW_zK^qWa;DLD@G89LYX;+KAIAKA$ej&Y z+JtXBbvDqgRfpB`Tb{2yTaHXVoU(daS(Vjp9pQnEBafo4IUhXE^3Ez{{k(B{iZQ=< z;QzLhZ;`d;Y~pFfI2j!7@ImY1n0WPlc|q;W57dPOPwm(@Evo$5l04xesUdH?b=?C_ zva1E4sJWS0-v*!Z@y?@yFf;eNGbQ}yJW3X+*w@~&?jGG}D9eN*Wt7=D;pGk430QpQ zHX?41m{{-bz+%(b6;YBa zDidQ0DJu;wZFbc?NBTx=e22PhLB}SbxA%Uvsy_VZ5ZO)TiQuSpwd=P5Eg+HO#(nf0 zI@uo|xCtqGx~;h=s|>Vm9+<6`X*axc2xKWFFW9j12jX0~$4_?fXW-*XSR4jp&O*RY zf*Nm^-FN01CB@9|%ST7J|L1uAy$|=WG)SgtXyOP6}_eFS%?GhjMV;p*(c;$*_$4#c0wKu zdz!N&`J%d}L+g)6dFKg^(C(py+Pa*MXfF2Gy86?uB;=Udn(NjmY7&#I>|uItEw{Vp z8b2ITR_UJHXU%KQu;6LPRB~Lh@mX1MB_;+RYd=aaWyFlUNKCTZcTcc$yjYq=Olpu< zpABo~-*F-5fQ}eyJ8nO&IUO-SI^(Y?KZa`C4bVJJw0k4-rFQQ(u_0XJJw#R8<&y_; zv`(^#mDvo}`^QeBZ))lc-<)KjWsr z2M@)hMFt4=qYY=63cbT5;>2~B6W83@PwTxyQ>_x=di+C@kdQs*P0z(j0%wsR+<)bX zM4lz-%dn)9VVHv2cQ9^*#o6uk^x-w~*-v%nd}+w#tP=drftg*m8(O;yGKlJmKB4^} zjHcC+a)yDQVZ6DqJ`Uiak*wTYv6z_UdtttZ6*tK8oYt|`ONe0_h6!$CCuoAStG37R zf{A3P12*vA$~}m~J&#mHvdJyI5NE9Y!0geFQV$H^qPH%_#Jev?$M+kE$1cmHqaa6`ds zv60hAvCw*AEusGUu7SQ;%n61U!4{T*oN?tGu2C^o0vl|@O~JU!D=UPDxHR3+H=6y2 z)kQaSbP?SzqT5J|JeK5y-gF%-%O+&{lc_&JhInz!+Ba5U`5(lSXZLOXqh8g_uBzg5 z^!?zPms0oOdLJ;e{Xrh3;Uxv58X-5L_>r-x;OU9&7FI%g4X`IWU{4CoG?KOZf?VU0 zU<+#K%lf6JmXG+)`oq&zlubZzOQ)z1nRSSb zxwVgwW}IIgmoe;jmdh=6X~xoQi|V|B^_F}>5%!o~eQ_a!emicr3oRnd#%q(g4rC9k zx;nYfPqKqCtNLMYOZwz~Js3JTa;HqrGa%*a#lIKO+AOXrDq4wgw+S*aW;UWKpVyS+ zs_s!ID_h_yJ$bUf_~iM}eIj~OMv3w9vrS(EXwskY7Cx`Qi?*#>!bc=#Q+`>^Y(kiL z@`_JoF()Rf3N-8%-|$Q$L7^V_c16`_A%egZpE(Be#6gqT}lO^}&?y<4NgBk+aMB5T&S%;gRb4jQHiG)~iPRV&A0iQJ*Q zsv$W{GsXX@pMEvH^e%=)tcq!0={9@R0UN(GU2%)}#vLI!@Kkf9pD~CsF1$K19ZeTq z%LZ`*qa##J_sIu(28gvdi2V8eUMAMpj$8HRacGk|6ckc0t{fL-M9KE{w>(f*SI#B# zR|L-uI#x+@aj8d8hc(gEO){pw+{=8Rp>eEP2Ckcd%OB^%^bF;EFrx)*J9z6&q-wK( zn=esoFCMk9Z0%LHn!>$9*dXG+_^PUQ!WXR;>h_ne@7fK;mD3_4RZ|oxr*FmHw;WCPn$>R zkRUCrYieFPyD6B-|KgufT;xfS-hfg<(ldDJ3nOU&dch4Ep<-CiwaY1P$PR3r@Z2=F zF?G^(pRLz_NjaWXach~DvCwR!$1~9NhFpD%96MZilz)U?0%Z@wC9R=umX3~|C-NN) z$5{w~FyUp6k`cXqc|!cUyx!2%o~+{RCqk~ZDJk1Iu$(9q+-kEbH=Fq|p*iEjfHOJ56q%0oM+xw=0{rP`ai<1zA7YR2^!On%W6B`OeQYUzB zT(3FmLwjUO@{Drtc!A#=Qb#E_6Fq^jI!dWOYiCrpMw^Dd9o=umS1Ddu$=FdM= zDYU%Iu5I6-&P)`N!)+rM#=osFX-DjEWF)2U)^H8m!92y7(?s3Vp%Y zCLzVAI7|Y#H#AmHIrS}z9IReY))Q!!sYL-Cg#zIno1U;61OBD@!@uuI%SMzsZC=Y}9E(hv6|9MO`cA_1e=(r zbKu>hliScT z!zINyR_cM{Q?yh>EqCTV8z=nYtGp8r5lPN+=#I2gKx>I~EFLBw^4v}1ll12uzlr!8 znEreARgytLN_K@Z+S+`_XXNAv%W^Hy28Rh8MBZqj^k%+?F2|&eFBrAyu>2$l2^yIc zfXU9v)}*YeYUL32(a_=kWBRiZ2e}b@kX}pC7a?-jfBxUx2oBd~tSb`*CIzp{L!>9^ zr0ot0Wjo02kA95QItGb3UGQsAXzfdUL5 z?|QqvRju>qLXd$c;1&FMUW^(t{p!ZWqsPTF0GC4Bb^CEl$5YGUGR5)n(C}~aX77^u zOHO;IWT#9F(=JcMM5QKQ$Uzz;mkmt)Lt_@umCuW23ltt2{PgRGjJC(6EtOVB>8`s8 zA?B9rauicIW%@=&MoTPh41`RW0EMc$xMZJi&y1Q|SVVO#V)-QpH?rc%5){N)S4*no zKH1o%>gPV_41kN23SHy;$Xn~RKgb2IYshXz_opRF7#W@F??m@sA(3C`!R~3-Dlx_z zQXb@3$cNSEn~AXFh0VMDBLgj;EK~AywMhrSCTm$d_k0x*_Kd!RI;o(JbUn|BBc~ds z^dNA@VYEYaxge~DqHYpAl2A+SlY9Uj?K$u=E|Ji!*cBaK|Jmn^FZ=dFm|otXac*g^ z&)B@JECDq(v9GHZpSPf???PjpJubg;Cnfwu$Fb!xK|V_^k3;wQbJEI#S!qg09AlmGva#J* z7eE+FfaoA3bi|W=5!qrt>?0BOVtb;EeLeYq$LhLpY5JN&*ZnV z&lzSrZOE)JppSKF|5{aiy(;68y7E`4tvf9ptH_lY6J5mljoY9>>-{RakG#o?ui0&k z+9{=_=Lg<+Q3HTxfOFfX(SJ8)&B*@t7WReltaPU_dcz`{00{H4IpGKkCIW{}^6+dUwH1qBR8RdnaLL&* z7%w-lTbk8+m%Em6ctgHkj;|Y;VqPQmda(AB1H3U)#6u1Pt*LK%E4z3iyR{JzAtI?l zCo2gfBU4JzPKT4njLjwH#@^aSzU)zpqpa}yoMjQmd6V6mhuH;rc|rqh4qz~3=8yN) z4|Gp51>` zhF~>8%xwyz?>}4VVK;)?KBt*N3DAG@Gvg32#En-y{vo%hX>#B1dPG3qx~h=B7<-BQ zEiM`X%>LeaWh0@@tg1ltXj&pt6lr!FOUA{&ZcD;sL6fbfK7Xh7os-cm6@cR)TKE?d z-aT|29GVps!!NZV%;{#0do46)Z<>mSVX-V#?rLOo0-jp!ud-5MDnZ6ww8UQ)FW-a* zv&)>fmTv+4zVGW{`EmO8RBF^5VY3kBHFi!Xe|O8sS3xkl`;KI2MQ(ahxn%C;@)tv+ z_q5_Z84OCB%rD19KYC6B-3oaT(Lc1Y3~{r)B{sfpX0voAieQT850>ZVn9MK(P!l95 z{nv9_@1n4ls#RmR&VTmx|oEg_BjS_=ujc?K4e8odog~y&@p2xWi^w)8sW7q~XRR*U3(%)Lp(fuR)#>Hku{y<(JdgeaIcctMu zkcMXNv+tQ|mbeXdS+Wt@tLScJSJ!70*A8Zy`8pa68)T{7{Li}3IDNqz@8Qf^(^&E^ z-4lt^DJ0QpQ<&b9ctj6Yy&g?w#$r#fPKYaHxfak-C*bzXoM1(5fRA7Mpo&k!4xuGY z0Qp2cK=uwq>j93H&g?+ZOh-1((h{RXA6YK^RQ!iStn?SATiG0^N;;;>eo>7yfBxNY z)T}#k!fC4Ab{~s?$JoA~N=M+@4F_B?btGaE{^ z*?QNKIeq)=YUsNl3aZ}kq7Yp;G7ETpJ!i0W@CtvsbeHmPy47%Lm zy6bfq(#9$!o&SBKB#l~o@lDH`rE6>k+Sqittf*esWT7=*yFZ*r)!W^ARwyqZbR^x70-xvkzt|$jH>n7b$QMTAhz_s9?`>?B#^dQXV1BD(m@XCd7vAYTohBij{fHh z2%ft$-=KK%{qz%Lf#B%_Up(O!aWV{=9(05Z=^mKzNMF{elK2vpQ`+bcNHdIA<*0ZL z4-z?COfu5-rVtLh9*&Fh*|xI<(G0<7+TR^F&Wi|JUM)?O;Bskv&3;*ESn>pvwO+Es zTFL8j#xeG_QI5~1l#na_mX)h9M3$DeB5NJ(Y^x@8lJLFuqKLF7EG5YBG;Dua_L5il zikR72e^!hZ{}5${A56WU*}d@2Dhzyf!{u2#HAHK4U}h5oum$|8F`}Blj;0RJ7t?

)A_bzBn@;H8@_VClQrX7VmP7|%yomZ#fHH5k! z_j&capZQP%fXKi-3Q#a_7Vp82y zBTZ$Bpuf7VjLYlr@~;}xYspaiAs5(-YxIp0o<$)b3&0E!%Tts3 zlL0aqNbAgvjME^E!7Z&VokX8@|4g#-nnRHUlKBC32-yVwuV#zh?@h!enkUA`S<_=sgP^F&U8ji$o*PdS|Wbyf1y@JkPQlX>ae90Tv=oM2+Aki z=k+0pc{G7*M2+LCSKL#>qi+d}Q1Er$#CWn>V)j7QZMVn!7;BF0fItUB9W|jyZ*Uksc^lB2 z9d2iEuaf+@UP#y$+k1BqFBS#7P%$Ww$#{C&MFL5go6&)fqwuTrB`27q7V zw@?MWJoPG(ICg7QC#N|GkBX+IKY;Vio11ycdnacgYaJxE&PNPdR`vJv;Pny4#cGtg zuHV)u*NX${=NV^*-sn1;nuT^+1V^1Cgb9qCS63(GUWB0*(@(G3D_Sq}i_V zO)z2D{>+5o^YGXS3mF7q`Em8?%2GJ=0Q~XfLs&aY(-U?0*Tygi8t(6PZRbv&Nk{A*jYdN@Yj0>&d1yA< z5n_sJyCG@VJSXG>NrNh!Zy*gm_91s)-}-U;j#SD9pdCUd z)N9lM^0KG?Uh6QrF~!xT#Yr>AkM8qoUQr4T>GB}W;(CE%ojxTU{W;0ErL%q4!AYXr zaUn@mFsra$z+e@>N5_0!mX?`^DjD32U&!lN>=@@*yG)9+6Vt+ zcm5Xwj34hf~Z?e-vkj=#;2#MWGHS`dxNQZ3#S?o3{~dtyObQ&hSA zLywN1|G)L0+{=O+EiGj4U~w|?__f4k0n*h99G^( zt5pe#aO(f{@m54y9p8R6ZLT?}Zc^YF*to!Ewu3#ddW zY1^q4)Ru{I=y?7P_ap^e6pM3JY3VcPsuIRILw=bv0~Hm~Ek{jFv3|tW3~JFwsIR`M zswSk=PPO}^_9?b+vfApgn&o+7I6TN=H`1yQ#EN>;zU8gesxY$dM@hkCe#eU#rd4B~uDkF#UVRiAPJw>Gv0rbuegyrCYg zT`e9lcW|;dv2u{gzxXx&#n~>x8f5v@%Q&HX2w9(n3@$}14l;~xTf|A0R; zQ&_$^e&oAqR6gprl}q6Q5cLEz!;X;VpeEDH^~YER%_&fq z(;1zMnX#I()s*7CZ-W`q$y{5Fr(ewg9fah6Mzn_0S3OI_E^40I3hQLFd=t>-khpMI zHJneQaTJ~?DzjK8_jTH)>56W#KP@>r^I&5q`$Qc=&LR4M=Ap2b-C&HJPgs$OwQ_$` z40Ag5W39-z&rbkR5N9GL=9!gc>uC*%r0$4)DdOT{Fyro9q>en`c%KSt&oe*5Yf@9M zNoOq0m*EF-Igd`;Bt!x&^uIq|b3dAS(VkPf8zw$!?5aG|BbEP^%%rxe%`=DX{Cbd~ zv}3XBA=B&g9i7j$NA_@TzUgt#UXdrumDVC3@w=L3nulf5;BY=?x1BUlecB)T*8Abk zk$k~R0!*T~xKuz1acL)JAxOgh6f7>JksO|~3WR9yQZfA+n8=`||y+Tb`E>I^L| zpo4(_k3Y0yWNL)CSS6laCQ~y5RJH3(duI1nGrRg%9&NITj$a&CxKz5id~7sIh_!Z0 z=$sHm729F3;0HRR!vg{lEIf3(-x0tuq#NIgOYlp{%oMOtQsdnCfH-!q4euT-3MDZM zA__opu3*T0Sw9D@^>A+xRenqR8WvZD)<#y?`1txi6P+zw=LJ{U2mzl=hpZnO1@<+G zMAXF>TkvHQCS2Zaps=vt2vpP5H&7z8DYndB8LY`QN9v21{7P(jcGd-da@_jc_YRjP z3+yO(vmZ7OvEEdW*xoLH zk6KnDGjdZm@bKqEV5Iqw{z)=V`IWmL#vR)RN~ta9@RXv{Q@p}E}St)*CF z=*hAD1PJ1bd_kEaLwkiNq%R&;Kbje9?YMrH}i2r%aBxmi8I z%b$hzoi=j9sngSA-digCS^Qi3pqI>OXqova^d0P31@gcjxR$n?AHjAfM2DB4rFu6J zcZaDAE|r9-hyZxB414Fg62}tp7zAm>%so{GP{&hw7FFI z+9mrwZ<|dg?);D{`V$4hoK&((N_b|7lDaYloqrtOjE&Po)(dsr?5bFcmCdbQJEZId z@{Y(agC}+*fX62&UIe_FUE!+;R!C#y*RDV7MpHmdX_sKsplH268~JSu#|9*@ypG>< zOGoXV8Cur56hDoVS-+<(SbUt7A%?gxF7P;45$ZqF)z*YfdV1tw*tS!7lr^9#e5K#T zBlorhD8APm2UxwQQ-a>9|aOBpjoFEH=(^- z+yBT=qu6x1Fn;^g?$I8fzi!>xtU*q8-d+AhFqH!7d9nMdW@jY!D_L$jT>C(vRDoaD zx_K4wjL#VvRpGZ+AD_SDj91Xz?T@crfaR-m(1~L8nLoFb> za82AbvH3l&x;|;lqkT3k`G6+TgN*?A@3IPzA&0LdYD~hc3t%be){p&aalm74+(oyS+ly{b26T3BoaMx_Vl+8-K2~ zT?{E6wft#Y`#Y#p&M8cYm>v48d^FYnVh09D;ai|T*v_NSA)28IY}XCyAoBCJk{#z` zT|~w~Ii&AoG*!p#=!um5zIsiFTeNnuc$6~R%7)`ohZFnK`T_fGAGwvgDj;w=s_8KP z`CxGmR=}ya=9^OOqL1T>P+$}t)z^sKt_G~@qDE#eQwYwS_w*c&_yNCmZrr&79vh}Z z6QKcohnOXT9gx@PX^b9LzQ!_1kAu?7$5%`VTHi96PcN4^G_AM!AY@~-U~$&pwPqXV zC84jffX@JR4H1!9hecAva}F8^?jq;;MEWDfUcl`~ebS4jK02#?OOR9gUGCx(uBm@a z=4wG;PVrRng2S$AJfi@*oG}IAC-P$^2$5*fm(usRN^!%FF}=4~(a@-4pn1KY zPhKO=9l+xgS>zV3iWgZ0cRZlR`)Yi)J-wf8B-0|it{+1e0U~nv0Q2NAsj$V0xx@1) z+}x4GeyP^P9NE6HpLKHKf)qY$!+Rilszsv-z;+H~MM;iH2F#?UsUBQKX4`t^!Bk3~ z%Eod&UbN%Z&Kc9q1%+RLI{j`&i@tB0SD8Q8Ffz`_Elo2OPP&>|PksAe3oBI1Y`}FB zicA7|!4C~Y(>=W`*`bAde%JkaSct=x++gaN6C*Jc(LTK5vAbz8+%p_atVzlHxifF=R3#%CL%3x zal*RiLR~#C5Zn6K#3^r(M!OvBNQKjiiuI6r1a4po(JKa83LD3fL88k!kIWt%>l8Yvr-$n4kS zHPvcQsXPy^HskFfijUdj6$0#OtpEL6@|>*?w#;hiN=aLv`QWQB0<15{>|)?nRa;lA zp&6%j&rPx9&2bC-N#*is_1~wHeR(>I*ms3v{JSrSAt?KI$E;xlVe!7isJjs%;p#2S zIgWzM=W1`_Aq{&eDJam5CN(X*>qEw z>Lx3>2Y<8xRo(om#zG`!JZ~diOvL;>z87l{6UX0k`%P$eT%P`<(z507?+;DAi`w`O z6g3I~)P+c-cVL!$K?5Si0i~C@Wmdu6#!LseO&JGIfJk(K+RtG#u=vGz2~5r8i&5xd z#nl>_xKX#FUrYYF{C05rUB<6~vzN{vzt^Gn2+VU0$f@W9uURPOPFh-oR$i?CKB_bk z`N(Yf*9EO_MqTf0pc`iH!S#KWvp)zasd{W^8Tedx_k|+hn5UHBu1=^g5?#gS0ql+? z>Wh9KkBJ=J;kt6UXrde_e-lerJ@dR1@Pld(a7mx0jy^?zsQv_S7Wc@UOJ3=ER6bpq zc%sPca8RtDz(EiBixxx+0DEn?7z+5n&|FH|JqS^XXu)_Qk>$Y$bY5dkNhK==nV}dj zMQKR6EQ;=|)0uqea1pl{hTvAI0Xn`BwJVgj$fiKvjmC?y1tNzbH>M64b=;v;|;{>b(zD(^?+& z;Ih4FdL41pt(Yt+3vC{GEfo{k*N9>!vgeYWh>*W&0RD#BD)?>GmZ?b@8;AXMOkS6l z-aNnXlgoCg&~r(l$WMM*@@A z>(Lw-zvmY7)W^d^x~SImo!;MH7yl2+0_jJZtDSEGA{BgF&iCt9c}~7%LNQoo8iwhl z7^$PW6Odj~S(0!8+zKbXca!&4*K zROk8sruurNAY~AIjQoCfyvzbW{_Pebf3V35iQiDyf&I{G3rPB(a}L0}ToW%Yp0$vY z^NB>f3oUh#if87aFI;Gzs1M%E%?R$qeKRTL@(m63R76hY)jnIqLlNLKBTh;~#OaFSdhst4=Ki;rCd~7n8AkdIs75!yt z=7{TWqsF}qq8h?NO7VEI=UamPO!0RzYk$@n=Q*rnsKffJ7@B(#3G@ON$ssSldB?l5LQuc~{VVHJ(7H=>!`M};% zex~`Jf{_Wp_gZZ7lNGi<0+O!MN;Cr6|E?G%ZKq)5BstMXeJzcZ(M>)W{YGmzoY~IX zSo0~$9L3$tp(VBZ!f=fris=!Z45iCWQd0K6-;e$6g8u#6M#4dMI-m3`KoS}^_Gw*C z3e=gNP2-58u9yI{gn#9H{cht@Q(GY0lS%fO|A7KDze^8h*-W*2omDvGX%+9l$Z|fG zdYnmOTK3b%_ms$W&ZWzIi`B1@F)C^*f8oveo=DNCiM?@^JHXsxn9ZH6%71M|9w35Wp05nlePpXji^A#WfRB`ag_b7!4_hArz zx?o5hjN0%DWyX6DWn+BEl`L{Lp@KqDivJO<#dtbY4~p{ zkE;?Q+LR#`aHM9wVQrtOrQ*G_w)~#W$Yq=PKr&a{^C%&YMi$J2UtT)(3>50(NaFnF zGycg_K9RjMg*Np9HjJZ- zYlr~5LL+44Bjv=WoEOD2nz}K0w)_|s-{<(RHaL6*b(b`HK!6Etzw7xAQHp|8NgZDb zsX6SW;$k1R@59|6UsRy0A8(9IE{6=ou{6TSb+uuuoo!Oh!&jcd>l5mkYc;U$!Z+**F+ak{L2_tD}`A0x?k;ZceW8I-wnpH4e)|Zl56<-h@BvZ;QT`9NK?U%5aQ1d-=JcgGzGLYU@RGi!GDJb#arIM9k-FPZ!%20bBUO{OI&4=MoIg`;x7v0=RB&^ z=bEDd24&E%oF6dKc5AltT&LInNjBxmi*0>h;eMEC`T7;KNvle z>>#zG9)P!)1H_}F$UGWsQd=jU_Tb|E0|7dKs8jYz zCovd(aSyid9CB54WiM|LTXek2RpZNsh6b#pm09fcDoq4$xT8v!Y4aG6fMTG(TY*kZ z#7FypcrFZFz%(`w#eKbSBh*@?C&Bk58=tiZ{^OE{+H${2cW;fo-ZH zw6xT$YvRU^b9L-X%Lt6SgdIM=Fvw#}$o$@Td531D&EIi`H{ksX*r}799PO8QWR=V` z?dAfiBlcLwp~^-47avdm_LrOk_X4;@O3TV1iyLPeb?@RTt4Jf#R=?=x06l4n&PBsu zg6mm_1C4w8-RSuAAO=o4!@TTl5ggvrQeHCa%X$k$q2=bd$7pZXIww$E9$aiHa@)Q^ zTTM7%e-0$OsvFhvK&SJrPUkHrCMal8xa%ZRW28hNj1L&t4o6-O^6SMFSO|%t4zO`k zHy6Q?m=^+Yf_jFBjdW3G62DD2;nj5`$eRhqnKB%z$W^F2KTCC;%|im^oS>`z_s&fm z=^$1%_>wHnzJIcpDPNxiM|a6m1ZNe#qi4sfb717phr(d0S2Rs<`$_#p`0 zTHX-ws`sBCCN!cHnQNe5j&RvgBBE{m3u2Xci+`N2hLXBuIBgTUx+`BGA$7N-3b0K? zNA5LxS%BJ5=xe2sh)C2rDt6q=>uvan1yxu6G{Nqr#*?R$Mt66{Rs z7UPmu3wDvSCeyWxb%*~?&mMD}o6rqV;Jf3Jh!rOgNezUFsN!GgR`4)r>WQF+eusA9 z>hLi+qrlWc>pM)mPkxK zk44rWa#za~B>6}9Np@o5grJhKZb`pjnbl~X;5QCF+l3!olx~+5I5*8??0ypL&=Ck^ zRSxi*)XSw6ERV@{_imL?OOS=B;zh9ux$@P&#dI;$iHvapnxlwL<3%f%IV_Z3KtONt zn*IPNJ|f5_3JUUr&-wO<7w4B5brUo08ozveMTnpU-1g3*p10pG6E`%@XtIa?S0hv0 z{`Di^4D@oLTRYi19*k$gJv|5e15Vl#mGP^`$HwpxN8Zy@*r0>a3Ft-4g&syoUr@{geip)RkEy<2?>(Qmogcq(I}a+(g(B{lz#OLFW+vK~PEi z0VN$tQkzNmL!$oG6`QQiRlW`AV{)pUkx+(5p8FBPAw@c)y@1q^#(GFwLa*vDbAX$f zXcaA6=bSB%V`Ujuc_Eu=v`U=RsroAPQ=R|Srg`y$ZJepa0SCzDxnU0lROmuiLdI^V z_P4S47bt0jfR4K+kniVksTG(cLbVZj-<7zZ3K}tsT+o-R9gFR7lGLm#VrQ~^H|CUq z$XHLc(q#7W!3uB)0%#jGCuKFvH zuj3oJ`(xRpy0?EzP|(tduFFfTw6-z!M@8!r6szc{9AiI7X!k^dAzBm69gt8kHg>h4k4;5LD9Ffc z5)`kkwP&3)?I1%)+FG6pzW+22XoeUn7B`xQ|8(O9vKYoN*^3P4zhR$(Ra7eAX_sl>T>?2kKO`)J25~$RU;iaXw ztrr&z56*eY1?KG2(&?!wo=5=Mc*Cd|IXS^|GTph@C!GN%k#7dxH3`l17Y)~lu}v(5 z!$klQ#B!guA=?0CU9|^?*J>0;9~7Ceiq}<}r<~6R-uXLU3SR-Y;#G&3hEI7(iR+@j zbi5pe1gJ!Vu8Rlnk`k(DsU^(JGTz21>Aljixkz1iquSW**#~R*{b1K0$%b}=OC3x8 zG~L2;&F!32d#&e|V8wtpUCczaj)CsWdXdq%Zu8o-@y>A0yZM;B4Wrh)M|CObr-to>-6x4Jx*-A2K z>UMBI-b83Klv8Jef(m0zr3PfsvIl4w6i#S!=8?zm-8`!=BtX}xMDN%zFeFzxJJ3!J zbS0mum6geRW8s?A5&%KqoVi4uo=-G=2eZZiaqT&=+^~ASKeW4Z@C8;2v)00}#KZKhJb9i#!jA0F;P~^7=VLo)ZW%Muh92yALHO6@x1ErP5v4 zMW5R_81$cAe7AkSaiJfRfj=aA`ajyS-qb*H@0w^8=o&?}H;RtC9l-}&!LNw4cLuh5 z$byNKHQ1Yuuj~?42P)aM+W};5J0cPhEs^+`q7n)~iA}G0yd2(CwH6@)I&9>f%7v_J zOI}&mm2(k%P|z(z>DMU_V)%030M&?6QJo6t0^QrZIJlN=y_X0oLMXt&%d%DG@0u}DG?va!8?*Ikw9lqrPUVgp)uPy*Zg+rjGh-%MB=11oR2Lo z738UPYmRZX`{{A4R$BSgp#K}l(-#w=u?%(g68(#{^BK8pZpJZ~=&9E)l4^KxtaEF3 zYcF~jTx3a_iV~ns6n7*j46iACc}p~)Og*Txp zvh1OThB$^1yCsql1eIG4j2yT~Jq=$oqOHCs*`v@2gE?DF)=mQhP_ zC+(hz{@7;f|Fu24*ECrvgnw7>rlF2`TXk(&+XK9m;?`w3ANHgDC-ijihs(@v^MxIE z{t-*7^W6VhyH7mJ)PEy7xK|OUgW`W;P&7jVyc9omt9+@8lg*2?uF;Mf=Zpm%Ud0Ie zWBY{8K{oT5U(9%T3r!_&Huzwf}6$zCP zkW%RurBix9$)Qu}?k*K65s(}@hc4-cRRkml1_q?18yuRU-h15l+1=moJnwt<>>qoM z`*rTR?&}lRbx)#58{M4HU;6%KqHq8j)9HcT4|U$pk&Js0Qe3SWOYX1dcRHM4u$Cyy z?sDYycI-nT%HBGKaU5?Tg8=KgjhLTq~feR=1qmnpgi4F$UY%Y1QX^mZk8|<~06ezl^^c zIK+c6rXbChO^7;&hJ5?&W2G0^dfum+wwF5=pQ2|wHd|gcNUhk3-#his|BD01(h_XB znxU2IkJg*3*c`QQUh6LH=B!fIY6$vun`b4%P@pWp(4)t3+Tg*1wP+M+vN#LV!atTr z@USf}0Lqmy;<=J>rQ)R7FLB>7vr0$YdL2~L0W+kwZrGcDokYYb5`sA1|1n|R0!6*4 zzo0E=8=jq3<7}#-ITlpbCS!PlZ~?F$9R(PQl?8bA*2~On#Rp+7*6}!!lT*-M|Da}^RZ;8+k(t$f z&kF9EWlN5HDIiEktvR+dN574{i6|7CN~pfLGvMW%MBUCOo_cA$?u)sYpcHK@A9p;T zH4;{en$j2`$^M3BV|gQ=_m@|*ekfpyBT?1*}BCe#9Dn~fq|80MU5IyYCVGb z6=s-M4mwPxpvRvxSkz|3`{Kwz&|G`@A1VkZa|gSVgbGnv^xZR2bm~7crQ?$DR4J+^ z1Ekwr95$V%L?%sHJzoDfJKa%Uz3GtpT>mIu=LRMPwbbbA?ccfbkM|ZQ^YYK9!~*eD zYyCAs6)du)p|YvBdG}cX?V)#mv-+D?m@T7j{OjLT4!<`}ZEW^8(;3?jqDq)iPmGFG zUK2(M|1XX&+n)mrrfmNko{hyX%Z%ESsen%s zB)A;`E$(Vj&HmpUr~9An9-M0|>GBoR{dcQ!d;a-~ENS-UdC}eY)=vT>gEqJ74!u%h zvw|Nb-QlFNNWKb|&RX<5$tk?Kt<|}N=0q*%bl8w7r#&~9 zewCAZXWMEtqg=4aFXCOb8?kdACIm&iUs~Kn_}{~ZFNWPnkpc2-()U6st%LHK#T?Xa z*G~D~(x0zJsAGy}Y5Dc*oORP_ba)N5G?p!V|ED+(%xf?q_HZ!On5DNrPCAV?!F#9~ zO8M9KK{mqJSI=x0T)7<^$0qw~mB^8Tb7wAPyahYug`?`i4++LDmv-;_T`uFrlTGl5 z{pWm0VKt1B1;w@c(TsJK7?P*V_hfNm^ zHWX6H&i9J&L|qNm{u>eh8pIy84R~!|l^(`J6qg!o>G(&(b_;H$F0G?J9tQ%4hf_I+O<-xjac|XNltg zY}j^qV_O-2>_M`NYwQM(#ss68ul*0Ph^b^g1|mAm7O4_N}kbK6nrpp}~RawJ55| z8GzN64cd=hHWf(*IG@+QPWqQ0eB1YD$j54|(&M-lk|a(A(XX@o$X+w=_n7RYj%Ku2 zPCedP4-@n@J@E8%vVZ;GgZ8&1>UPHRj#*Cpn(FI7N>ylRk4X6$1h&PPvdMTMn&nfP z{)I+IuQbCB&@7is;o%1@AoN1#e;ofOc;j;&3S9dk6gbL*$h4qwD(Kq_j-5SJiSZU(-f7IW6;?NS1@lG4C=5!3#3_nKt}mDR9EM&amk zYX>KE* z6th-)&lUVP1^Md-@bE~C-iAX%*FAbg0?n{1tE>>j3;s0 zSL8;F#VCQN5AcIsSARcE6Y!D+56KTMLZ)xpuCG&?)+TC*Gy;e2-id}=rmmYC0Vp#c z+P5P1!6yOFqEm}NbJX+Kp@w3bu}ty3OY3?;IX2~n+2G&D5I|fKOkldV?Yp$73le5& zKhygyDUPv-i)(!h%Au)0A``__)qIrxhFKgI)va}Xk+IA};$M}eWMrgnSHE4UmVaUI z7G&Sx>iXZqN#}#z(i&HB!Q;xBp&!XLLnRr5NFl{{m*QFwV^t29_gXNR(_PP&v^)F? z=2bGqnudizcb30DIQ8{EUtiM#iLV7)ZA404uWha6@_tz3Mmk0mj1$a1Ib~!jl zDY`c~2C&-!XhB!N+vaB=XQM>N6S#cQfhyq!!uBQrA_yC45L|L&r)LO{1fpLu)T$Q-5)x zD7$uA%nB8piAVqEMEu~CsyZUY2@X{aV)hX?>VAyNn++niPU{z=xwy8#-l;l?k=AoL zY-M_9yNY=ggW(MpD#A73V`HU?tc|~O{k``l+G61~b9hJ5`}ZPAFU^D6A8m;{LY2$C zc(?c@vsS5|M1S!=&N1q0R{<>q7}fW$tRCImKMQxrx0?h7C0#TDd@T6!85wLW^r&A9iBKl+CNaRp()DBB*mtu_WWW?mK*|%*I`|001K27*i=b%jEd_F1< zIJN!>BZF*E?3$g;?VHtJ;AZ1{FGt_+XOHr=>P_{tO!BD4)C12x!Jnf>T{n~2Gd{#B zUs}55y?w&(HeI8*fP(p>7x%2~?c?b=5{(MEN?Ub5)*yV=*I;gE-zliYbc?kN{j}<) zd?V1QErt>0%qZ+*=dqQ!1dK&oZH zT8t`EVTOaRC^MMI7B<`3(gwJ+tQ^V)be%K^25$&|Gr5wpYeVtHTs@7(Gf($*;L5`L z`yu{c8YSyEmC76cP10}-UeAiZEqSB;sHp7^a0mm*-crQzaDw2=mh!s?1C9OYJ^tD1 z$27re?5lgF4pVs$;$8+wO_%c8W&)?$7`Q@|k6s z_~8Tam}eds8DQX-M&f%Pbc~ImcsmgR)n-~}-phUIpk21k<6_=qnObf3#)ICmwH`LI z-04o^-P1s{7iD*Un5F?Zal0qTOB$)Qozc_LbdJt zxwb)6jomelT!g36{#I$Y5sHojDRyZR%sxc26Q;$! z)LriyIN3+Z5n1+HE&M{+gnV)^in%~H@|+(OIWGyWE&fX!{EaVIFXAMi0h3D*weJOH z>GINMYT!akHLL}ieK9HE^aE{@{i~R^d19) zs8-r~_lJ*8AeLzOG-&}IPVjc=fm#F$P43f<99-X&!p;cl91MmLHOAk)Lj=E3QWPZ< z;9(;BfMC`AVpB^FL^!ux=eSqm+8S9t2}e`AcmF$mYWO&nA7?Yo3;F zF+mo36+w5e(yfV>mCU(BW9|v(ZDi1c4p>W-ee7I4$D})?qSm|0;?>%){->8|JccnO zVIK}c`1xIh+p;Qs$KM~Hfa`;?RM_=6@#?$PEiMCRNSnjLTExmeX0D7)f})|NQl-t? zQRwx*yedv0wvkrJefiS6yq^=i0MzqT?PGVKZ;*^r1fhrl(wkuTIF<#Y<+FCQanPNg z!FFG6&4`wW4y?t&=FZGu+xb{*i*+FrPbzr5Ob{e7@!6k=u!FQX$-=`r-Y&-pgSS7txILDh05KWmvi+Cyy-_{}Vf z?PVyn&nQPp_N zb9Z#EOU-;yFYeq7L@8g2NZb`hbH1hvlAf&zgIt5AV$dTlMp99)Zfo>`Ci^xTBR1(L zQQm@Xool>2>3)XZgK;*?7u)OKYx5nmV>?(ZH_`+NBv-(B3_Dq|WY$0rv~I@v9A`VX zI=8U}F_~&dXgQzQdfxJ%;qfyEU!^OC*YrL7t62M|F$I7SyC1GWUb_zQ}>L z)}VgPoYG$;?9ikdUv+ba=1ck6i@@~wfA>H4t(W(5?z7-b(i6A!dhu#*U|rIvr$HY!VR3MZk<9uWNhMn5D3BdS z+zDQ4a~E#S%7QQ{rK3hz7UEMP<^V| zkrR%)BvgI@LmbRGd1TZI9xJ=>y4fBD{3|QstP<%iU-`jmMVgI^=O@76 zF;1NSk#KQ>@v%2kyo56o_c-ovR&yvVLYjuv#5elwS@dFp*MNU@U;A$#V;M+BXTWK) zlN8^-&T*W-6Lz($Sfptvc7jY0?2CMzbO+a5vO>ycXUAZtN8Cs^`?QSe*`mw8iIwBr zV{D=ib_m$`qq`r@t$Ovr2Xs9+sX@`OpH7|0#oPqX`O$goGd`w&7wQ4yhNChL4(4&< z{m@E+OiAd^OaKl+qO$PH3dDIO)kmx;GHU4=N>z}tmjlB~!@K*rigGqH-&5z3DgXJ* z+dEhrqRx}M3P$F%68+3JX?yrL1x^F!=jNDozcjNbB6G8J^S=S>E+%SNEC2eQ)a8D6m6e@IA9ePE0- zC1_I(D!9`8QT36vC-&;kc76#B=`l|!(nTh4B&G8MlJ}qA>G&E;1=x_|X5)gu@aGhv zrDH%Y17Fs;_oW}o<}^3@`}iANOmLg8J=0f z#%KYQ!V8k56m67uZz}CVg9LEnR~F^c(z4}fNjEF|55y0Vf5z}O8Zc;YOYU*sk|rpT zgkpH`q|G9=s{4eEYzwR*jlPgDDHw6+dQE4i`POuRG_7B=`qmZ@ps;DMVEl07R9;!x zi>S+8uNzmt9#`_u%w$4{dk+5ct5ew`{AZpXJpZ#KVu1->fd-FyUIrC>a>}m;FdRUS z2uTp#6>m!$@QNCFMP3mRW}G0L7&qCwqO&Lf&3cS1qCK45NmtQH!yZepy2=P;%(=@COWewivHSkhB&u*&HMl@?ue_GfNSU{U0pk3gO z0w*%raL+iJ`W>>R@@9qZUt{poS7Afd{VWy0{ zDw|x(S6FJmBLi<1VW~UK6#RP!aR2X-6*z4t@2C1B(e`zp0#@H1MS4C?9VLAFYFW4YV6q^M0iJR+*Xr6`+DN zU%W^!8D0I^h>;aN+_M03FTOpC^;T8DU}yg%Y1ju)qi$LuT{REhY$$I&Ouzch7=TI6VI=*?JX7X!34lu0QKb(nUFyDz+|G!0N+%((5)@kb}C4K@o8%{x|BcFKB0 zsUDFi`JEORO>hi-hS5Zm)n@(yO;k#4*+5@(oAEwVAEv zkDVI#4&%|*DzpygFXbXf*L#-2G3Uao5F7SI(<%xk$-O<7g%tqhpilvUtvm1Tt?9Q^ zyXheolro}g?PAHe2ELcP{aRrW<=RQNYGK-ykL-CE;w>wIINXdr+~GGI`$^|l4V3CQ z&c+{}|8JQ)cPN%~q->4DQ#Dw;;>}T``5{RJ+DWbv>O925wc!gcM$G)8K1XWI)q2*=oljc9z*SggP&8~Yu=aaM z7Z^7R*ClGO2*f=l2XF77E5H>-6*+AkcK;Qb?z^F$0QJpCVuFadD-2o#&zWLvd7M&$@W&KYlDG!=k@j> zch43S*hJ%xBUln%=2`Zu$08B=rO)IhlR6M7*|3Eb;_@f%F>)5$!t7dY4=+I+TP^F-v6^VJwdq0$0(-yP2y(fA}o*iwMa> zi!Xlj2iMQl)Lpn3V~vM9vl_ep`Yr4);w|Lx{hPXZ&ReYq@5;;G>rLNiRcqH-7Q?Oc zrv9x!kk>KraHg&7s_pdj*odQ^^#0em??S(($VNql`ST3mKK8`>vfy z0B`Kuz2WF5$iuC*UIYqqlTg0OT)`#eFjE=_$I%~~0@@s_>n;o8Cd%B(r;0#)T}_JB zebi%L*=^qATYvq$uCPd^^xfjhAad=R4PtcpP`*M%`OTLuc@qBrMzY;OeKfS zLh|yMb}BE%i6yfX0+y}aWwOZ{)3@MdNZWi1RaHcM-o(Tddhf;!gSxdU-G035-*y?U z2Vr_QA~f{@&gntywDKC;*!ZZq6-vL8$n2FE`Suu=w=}_fARq5q$*6m^x%nmK0|LLL zB^fCtF(dCu2*ldq=sV&8+48||`s=}a@>g*(eL{%jl&MP+EV+BuKAyXFep4$OFyqWO zUBPzQ`^&4ip<#s^8uCOAKL5rF3C~Ig1xi2OU^CAM2?e?yJ}?#lA#O>VDF6zB3JfIe z=#c4Vwq_r*N~?{ra?crf<&eJ)bSvF|V{1VN;I0PgQf}}9VD#y`lNP83r|1i-T|X(M z3Fgbwmq^etr<88g0aTT5J;JwvvcsEDCaTjApM%Q#*N=?sae}p<*kTTM_UZ_~n?aX!A z)zJO-2{@^A8}zu#J_~~{fG~lRsrM(BcIdfS`dW6E_b*>k3>^?Kd;6ki#!ZLcX>N(Y zsI2vPOpL+3X(ibq0ESf5GR+1mu2ZSBU?yFkC`iQv0L21;x~N?rnmB)9C0ve_z|$YA zAPgW56=Mi#iiT2qtPCRi``xT2MsBw>&AS{RgNe*`IHg_AJ}jb(vP&efT!-85!KeMn zf`V2*XZ%9wA!OmahOex(@5+CBn@C@*!=sli+`1rM8q)!qjSeY1&R5RAe72*srd?=i zt6;!6z1E$I_KZNMp;$BtL=e%QIWru0)@wdHiF;OYVW( zvbds2KEmeHO=>Q&-gYNERxcr&g7<9rAB4Gz$<412kFq=tYSMwZin;5|{q$0|HTbr4 z?8RUk>)lYicN^A@yD`#nzGgq2c}DXM_lS2%%=Pq?haBd4Ip~q~Wn?=tz0riHs@qZ6psXFJ^1)U@qxi}=Dm&Ios$po0YkXA5lb!X{oq$&`PfwU zHhBzU63j{A;5aiuZx zFSRr0H<{ferxcR|$(06Oh?!&GsA`9Rot@$N-a_*Cn@=rj_lFW@BGLp@S=ngUDPA+1 z(vQTybJSC`sm@-KzaJ*;Q39w}6Cx8~-Gkg4`9cX@x`$}5dti4O(BCKE^%9BcdRXk8 ze&5yEC-=u)Kdr?>p!wk77+}G!%?CBY>{^0)N$4)AE>cn0IMg!^z&h!m0^8je3Uup{ z;UAwjnEo;nfexe>)s5Q4AXdg!P6@j)KH-|v^^$xk8g_Q}Ef<|Pg*@vu0}fV>SeUwU zH_8l)KoiyiKxop6*UHX%{$gXLF@Q_Z6+AHVm&4x8F`Y1&HS+tr9p-^3H`P)f+OrL`)Zdv?7E((xRz}R!f)Fz}qU%Jm{-APn!6nlmxQ1 zsfL*dYW2q)x_75vFCR!ptsi+3`*$YX2$T}tV0PH7_3>6*cHF<4`tf|z?h9ZejOp(L z-G8sNt^JdQVRaErc*LGFYXV}sE4B??$*()PTvbGlvoF@4vaIyBhT8tRoevG=sfrcS zqY~LdrG!kzBygmN{dNa6Fs?%Tzok6ZhBC@D0JbWd)|jYiVl}*7)DHS@9R!hvyPeE? z?Rxup2ek9mG>Kx+CN`#gdL)D%k<`NG)vf5iv~bP*WA*_;UA0g?Q?fve7MHcEil1fV zn}%uFuK(V?NqfY47d5jzl=X_%#{b1w>6M62Vb|eQdi#=pJrIRBCw z8ZUVAVZ@-f?z>q6XAygrN0~OK-6SKm_{>3K2Mj{YWag~-*RMoD#h(87!);}U42!3V z!xr)`D}-a4Z%y~OR2Kt}Oz|jv9TiRMAa}^*Pi_pUd%XYp;d9)ifWXq8!o^>iGx3}! zzC-FRL<^gTG=b{(^3@sd7Db`4y~B5bgkfg5T>k`|d`&=M3?~rCHZ>Cl-8jK4+}s&a_S2WALyDjzhlN5(ZOs!e!@4d`8^WONa~vo`G}mT%!Nwx=v$0bmWC%doGJex zBnz)L4RhU%bIsN)Dv#pySdwR8Kvr8pgC*UI$!SG-pDUF157p*!!k|0}zv*QYRAqP! zoO1WqnAB`i(24JT*O0yNZ-(2|mAbjj&8vmNSt`!jMLD|f#vT&9^m-$F!c${{*ojKl zZB$Z90@WCb4r4G)rydp^yQC!Gg!y4FmhobcWwdjE`VQZo$Pfc{nT_apE}GxhT1tseu$J<~gCUt6K;^Qna*@l8jZ9|1cv*+x{;586Zqwg)ls zNBG7GtEHE=CuzFv>Bmdui|nYYr=MP@5qqzs0JNL${=AdR=oyRF&e>M#qox*OCPYD2 zmJBiDCw0}=NAU$BH09vcbyPj_O?=!Zno2@;25n(KJ}Zd-%hPw%2|D& z`6RkK&aj`E_fwpvOm~5qp7jQw4aQw|j-hoBT+*k~kCAgy$U^lYKJOT)3Uhos2p2>< zQ{U#|enD+-W^rY)z8`xyeUCpzyHxaMKGlGw87nqN1hUzV@>GSUo1w?bNn|X7?Esql zYA8;>wl61Vv1_A3l~C1VyW{(o+P(`i6+Xb!L?Ka>zY$Snk(X zLi4J)SYlDPe3LGdH;z3UTOwZfCyvY{xtVKfWR#~Xv>R_L3FZjX8Yry*HAaXSFlkTQv?gy5K+cTj?j~JMy^ch>@5V$Evbp>BD&g%5|BDiS0w-&=J?-o_S zb+3Qy;9AlgIS|mhPQxjAyfW3Y(WWqtMtR!U3*NDfM{E*s6?53gt2UIXSH-|?3;tWQLu_sK{=L?C=v zq#MJDVE?kdOEFzn5eT0!8KQ$Kx^Uqh84nEBFacwh1g)SC1=a3J%;X-$jr`^r%L1zc zna4wE)j)?cF;MDGils_)?vPXud%;@7aSdm{L(GVMgyfD8Lv6{%G40MC(Gh!WN3ooc zaQjQNrrT}1ja0&JzHhB5HIunA;qZnBiU$Wnvu!BO2-Y}fK>@J5`!raV>8ryakHZ(h zC;)=+UaLF?c1>yT*6ARD)CS0fy~|43)m{(a#D=X@t>++H&qZl-jYJq5O^UKwPn}$r z<%yXH)=9uzf4)L3OsDbly#B4YD7_g;Djx2$XAa7n?hk#)S3YS(I4-%;re-rd!!d@V zzq1N>O;_yQCZ!R+*umx552&lNt=0akd;COPpd!fN&7_b`!p_xdVR+^4Tsp|Qe!S7S zu)p-F09Bsdm#b+m;qCagjjvx7h$MyJ2b%2rUimHc#V(xfV zZcNPTQj1S+;7w$kCEE`eD_ibu;t=}*9Gp}+Km$Z-W~lU6#P;;N6}sA1$gu}>@7I+ClfHw(*lZDkyg#wTAnZs-`C<3y%R*tZ2;~P#Q6aCB zRKcMoH;Wvjk)&5@hVP&vjm0)s-};A3KQ0pW*9{@0js>hMupg9vqrEQ69fN&`!0(W# zNYqKlF0K9bm{zN68elSP>$eokOU5jGisoVbcH=-Q%l6r61#P%qk|VvY`I*-!c%6Xz zQ8Md8TAO{~FKDpd+3^{;8Zj2(^zM4jjy=(L6Pz1`5EeeDJ2(=v*8`_-rukNUaa>pZ z($0at)Os`6y^578a7kyWaDL#u8<7~H+w=&RwoKxu!m_!}eZD^txU@h>-%eC7C^SiH z=gYYF$Vtvg&TIify93NfJXIo{qLboM7;f#LaMt!1a%I@3#gSIP16mB%14ySvWuFxc=? zJtmk@gStQyWI-hqu}GT9N^SVm)GDOIVAfb5l`7_A9p0{jn8S)!dHKc$`O6E5{SoBE zxj+i98BSE+MEoJ28ixxAUt_wsxJ?shPjGMq4(p3Gl zm9$991355r_;;|TGfC?p<}Nzupt7iD=mc(3wC*F;E@?VzxUlzVu@({I&L?gEEUgec z+}!iioc#2c=~Sg>adP1=aN2uU_BH@F2Q@GVr0+K6u^B12>NREP)och#1J2F+Z7|{D zEm}cA-Q{~==8zFI8o*_gt-_p*;x8Ngb|XFpSON;fn;TvTyKkuj)wR*1Zq?>!ifXnj zt3vSNCk-~eJ0CRQNqQ;dQ)?*hGw%hwNi7#4MEe#2&O2YG`SrDDbn0|c#AjF_lm^hu zowKzuWashumGiR)h{D#2Dw*k1T6>Mi-og;t#->T&W3AHZ=?+V8Bn~6Lp8qDR@cUg2 z<78fr+l zJ7WNh0J6S8U+Irt4qDdL0I%E0*%hqask%^9>)uB1nVMi4q2FoLwweG55~#xHC4db1 zeVrQI{WWs|(J*X*V6dxuy^xRIn?1vnq7m0ky2v)xNnuv!K6==l>~au6sV-8a<#GkH z(Kn9q1P|p&;&KGM|N5Ecd>qi3(z7`$^{JKG9M*Pru*n5fR6wKbtk<#O(lX&!9~Iyw zn5pI8aZnFbZyj;>+wQC3#S4bl9)i4&%%#LkRno1{v!gI|ptt(@RmVqnLWtihRgCR^ zm$2=!@uw4*diH8`C=wMt*vQGUxwvVoC#UFW`m{|vb3K{3s8FyF_&efZQ7CxWlBm6w z&sjB~7E}%GBhCAhOTN9ov)^O{Xqyl$OyMR0Ja>v7T9yBq$Q&SetNR|J`ThuzmEUrE z^9+npks|4qch`SJr*{n`vSwa43?tB;OpQjh2^2oE_HCbouSBt$=!?n2tTS_6IP<7_sSA zEv$pVSFCO3(sARrxS?>7_5Syj4SaQ3=s%Fpdin%1zG5D`b~OF%#o|AyQ9ekGq*NaD^rf7B z`VH3%y`L2wUDE$>)FuPVxNe0ac&Loo)E2nG~D zdAh$$KJk00hx`sOAhz_a5xb31^XC~{N?dyaJdNJo{l(Un))2kA-!X!^ZVMVbNAvQ_ zSPtHKdZc?v$n!YSw{`InbzQb{@_(F{Tq*rP5feg9OaR9M97`W4E+Y~E3u;mU{{)Hv_ATcm2KYo~Ul;X#XhIUR4#^#>X?4Di&b$PiaF{5tm8GVq}S)3 z1$J7&fXH*bLr?NRNY5S`C?thZ{uMd!KB|+9Aj>*Cz#Y1q(Zn^ zsp|Agfvq~eSCVeiFldE3CkaT}&`;{4vWI6DrRR}W42#|KBwcB1z0L!o0aoV%T&MzY z^4Qpx=*>CGpAu1_up-ykc7xy=AMSl}D{Jc^lPK{N*YDR2Th=O zz`Pf@npDeJ#?HhNQXxfqY?Rwh*9%0}&B&ChjzEPaxtXD*p58MOk)i_UrM<)Sloy>0{#b<0I z>UR=6KsDJTfl!GyJ1?|8Tr4+Kkx~y&;L^#wP2978rs$A6J^D>?d39OCh&y%GSc7-N z5z{B=x9_y?BxG?2^=k2Nmw>9PE1q2<&cZEnON|^MxslZ3x!`J(3W>79REPN`cZHhQ zb-8=zYw^k7$A^c&KfPCfCt3w ziEeH+)K|rIt7hEtG;pD>e(d)Bn-cX4Cbp)sx2?yfr@#$S@Xhypt^Mf7ePLsl-ma$o z(Z*Nc9vdq|6uEYvjic>me0@wr@ZAsw#m`MJ^bllKn~xNBVbQS`18qlfIB)L|?yz1l z4FZ)Ahc7_t?VEIs=UF?kQkl>kw+sE4d~H?{@0zn$qz()r5Oq!UfIto9zqZ)D-8H^7 zL^tv&DJhu(-&4d8WP_t!8Jm$>>q#DHcE6|{xOZYiIxI4n|^InuWWcK zT)2*d>tQiVY>yip<0+y)mT_{>fjO$RTv*cQBx=}xKGI0I?VO zfmQjs@Kb!ugm>%VF$=Yb=gZvOj$}cTO5579%koha?+@Fj#h$LwB*n}I|3?J{L9qnW zM*zMA-PJ;(My#Q#S$Rr@+_ny5@x%EW^BGGC90ue>QV`^P@A7(@M;$Y8_-k(!deZfP z1l{u0{wd6e0DSbuUqr4qkH7t zLLgnqA4rXT6W>gxkM!ehN0^k}12;xl^W&E6fmPL@WR<5%So0?E9b?VfOp@~^P3=D` zI)v1y38p~g1jfPPm6HRRH@r23UyZrlC)3K&G|DTdY9Z~%8~0K;FjR!5*#r;vXHNA% z$wWwdnS-dKP4j^P!1&%9TM<}0IQ*xHRi;Ki7|Kh=DtjHj8enr_iR~s4|7ch-*7;px zL3YW^y{6rtV~M7_soWfFXQkBbbX5M(@|?S8nLD;0Lg1%jKMA zX`@re-2tj65Tk1p!tUEq8EqF6X2B+9+KnnZ{=X~0op-n`oC=L>m2$ZDB&K?0?;84P z-PDxZLDf4;OG_lcW3lw44OF`OJ(Oz~Fb2pB%P^dQUp^)kkHTKr*wl2rz?2cP)M9yO zJ+dWod0EJ8D_aHv46)YAsR{JFd^N5OK_z zsArRSAHMsD)EQ6*)7?kHn@}0Yu7+OS1?x zKyB`?HF_#`0l)7mdiC`C$}6Qvr?-=Dp0bd~=>q2<;onI4dEIkOJBaY!TtTcOXjt}&2KNyHu0{+hQ_@*tK;<9l6U_;lI0EyP-GI?MSG zx>YRn7-DT@r=>dQHT4v5=Z1gE&5zZtse(#;;68d_tS=QksJfUUS^5ikDWjfLl9K$q=uQ{b~OlAvHVLeW-nIXE>j*()M2Dv6(%2s3Bpl9TR&#kyT_dPx-r>d9>n9qM7 zSR#lJJ!n}oB_WT7*J{h><`6(VF!#yV?%uk7&^y~)0ywKG_^HuF&4>!EQO4NJI4_c%Oc?tSQJQSyFa|YFK+;P!g3#>IiukeWIwsDIqgPnmMGd_4CJsB%cpKbA`beD>- z>2(SUN-aE2BJ4aX@4BKTZ|D%L+w!Q#^rL31+d($cQrEF4hgwtej(8ORIdfUoU3{Rs z0)~-2!T!DaZM#Y;=-W=O>>SZFrd*yW$Lmx+y`~>UxeRWjQd83q{ogayAIy3D@*Nr* z-E%0PBq5TDCy7D^&Nho)ocK&*qN=TW34s8KHUAwY`YTcSlG(KmTWIMjt>@vaY zPj-2>^hV2w+2pt9lQpBBVIGG?jB$W7-FoeKP_h_j@1>y4bAkz zB!Z8v2@@9}Vjc`MrNk%pK z9p09VpvG><-(WYA5YAU7xH`hY#@P|Mpa~&?7T&1Yp`VoroT6M zONUd>wE|lN0Ui=%xxo1BOoKv>L*Ef)f5VkMP7=KCH;~!9g-cwHw{l-1KT9eD*#8Tv z8_wae%!>FRspbQmHEEH?+R9%GXQl#-zW|wqJGLpuJB6v`M&1~)-*yDOrecYxLC;=K z^kC}%#U0ThB;IGtPj4|Q!i@zxpEnyN-m71QV`pGkXqP7Dq7)J}S_EnsG;S90+L(La zI0lh|&OKY`9#fu;fl19C^9^6b)CA?Z?HKCaE3ezLw2ZZOxHx}0 zKyc5?JYKsgODG#uFj9^mCPg3`g}kP!M2w=B&7SbiFjOeKTjOg^Mtj?Jzo6P7V{tnx zS*H7^rA;$vjdMg;KL2vBdPki}UXCLA!&M0VyRYPvZ?8O@eEapS%QyEXtGelJcjco# z{@KlK@Q?4a)Bb_&L#6sWpQ){p+Q}{ZRP~3kE&~({B%iO}x_8FhYxk4Ha8Z8bisKUmLBYX;^mgvUQn1~56?~?ZpW3~j=8}2^9H02z!O9A@?zm) zWum_Rwc!f_N`bd8u9-05c4U9~>+-TBr~UFYJ>5RHYFO6Gj3b;Tk+@n3#3lbH4C0ra z!u<6^tLwW=c8g0v`FughS@nKOE`22N{#8Mr!>cf^x(biWr+za%4t2OSR;?vzAx8E# zD+m1I-z-NJZsF!Ry1P$2&U5w_NxuFtQG#wN6Mh_{|08mGU=3fsydns;S$VpAAY-Mc zff94+IGws7R`G;GJFyZ2FO-N@pfMz#pFK>S|qwdhG`OCk@opQ_ghG^dUyUk z-L3bIPv|7$=x&|U$bAT*W_YyT`F$>eUV8dhnAEJQyT%75=6))lYcs25MelcczI2`Z z60;@$lz|gCKkK_uPETB8pkaW}nzg+?dm|E-AYKLwFBEZ0juJBnEb{g`IxjyyQO*b_ z54WqDtjNpTJ*#9OIkxAyeBHqh9kX7tC!}RD!KB3=9H!K54x1Ikk=)_p2uS@JSd$Xw zcCez|WG^=oV<`IK*RbvQJ$o8mc5{5(8yW{lcLt^RiNDhx={z#|^%qr<&BMR6CRmU% zgUQLyFs?_4;~Ev{XSx4}uD1-!0_wViQ9`;K>6Gq}l9uk04(XC^lx~#nlm_YUmPSGv zq`SMngFeqY^UXE$PrVShPwcbS+H3EvrNx_;=G@q5^}VP5BqN*|h_{x7B}I_>iL*Z| zBlIM!Q+CQH(xatr##ZPduANIST{!stIAfqqL*w_=R8(P}Tv5YAFC7~nSz$)m{OM6F z%RU>z+hR28ZV#G;^9g1>hIh=`yFnG! zZ1qQ`qTP=Vy^HiZueAlvwT&>gQmUaUNd4?Fq)c}EAA8n>mD%*l7P_nC86p#PohnPD zGKd7dV+FkWsLI@mCDdi7w}Af8^Re!-b`I!aRDLNi5Ov z&Enq0@;dCo)=#vG7)3f-`6`Qp+VUF{FdDD95;3MfxNBwoqE4W5$MIWI(g>ZPXQ)vh z-AwHn335;jo%-ABx@mzwOpp5>w;Tc0>~svJ>ULr`ZOd2+*z9a)bHtbEP(rBwdjYVD zz9i=!+Gsy46Fu5>l;E(l*T*%FjMTewF^efgFQjV&zKbmKu*cyU*<_~B#$kORi=`V; z9t#LahUNG(r^NV!t6$u@Y`Izs42wKN*<;0Qudg82tk&l&p0@7WyC}E5dqJe@t~Vp_ z%GmUDjhG)JmZZ|Lzvce42BlrSv6!H2dN6V^s6*Xie}kB>BU8xR+^GB5Q`hZghlmIJ z7c9L4hAa&WK2rUnotS??N~p1aIB%Zpcu(sEfpZ|7qD@^2{~gB{Va0G6*sqK~;(z79 z^C-H*t6o%E3}0r#K8`(74kYN=ey?uE_iP&!ED{XuC>FQ&k?`V4>5~H*iriqk&^o7{ zZ-f@a48I2v;CnN4UTW-LYjR?C*!A83wf;S7OOdcq)9)!NQOK-#O9}t>6v|J@0&fr%0G`uGKtJm@;_%Q z_E4L=>|?aLfPjt$Nuh}_tVz}7K=@h1$N@5r99bNeL!jXCB;5?*Trr2expEq2*rnj~ z-c;-6x*1%1A?d1$zfz1a@mjfvdWrqlRwZW(;#fZ!82;fQeEKaXY5B#Ed(qrXI8a$G z@AJ+*GZoU~uyiuHk}$OynIa(+w=F8}h!)s!W9!GgJv^w@e~2<_ZrL|5Cxv@HZn&q) z;b4a)Wjw6&&||Yz9X3WJvDs|w*>XDUi$crghY5OCN2_~ZV?yAyitfD+$-5qZ-_$?S z`hqwb3tS5*VtZ#sSmCxlG@;rL)TulF_#tVo@A_X210Ilr1_$e?eTgS@ zm``9(1Dkben5IPw#`KOA#1VtwC{Ju*7Dzy(Ke%u=Kc)0UB;qD9lx*fbst>qGab9uu zC@3g_?hGhU%n9wZ;oCGe-IX7L)-5UgVLrCjC*2zbADwj@dU)#HeeF?31OKPP<|rqH zSGZ08w~n%(KVnc<|G@;6Au}$w&m&oeR|WEv{-3QffxbT-4dHk?qN=ix!wK`(i$`s% zv!>WF(S}(EQoKj4Pn0&x7`s

46omZ0h4=aEIrRc)?t*RQlr1b82#FqSUZitL&Vd zFk5xo4HYLao;;%sm5skBC9b&EZr|UzYr%^BM6|Z^Ce(rtT1@C*h!IC5WdxG>q1%8n zDhFXEUJHM9 z{Bn6F+;cwH5P}b;{pxdj16D@#vTJ;XvGMVq*r*{}%Iut>k!2W?nCc14{i1fQ3#1sa z8q7f-v*`=WoAo&22ksK$Kyx;Odv=Is!8;!9Up5vv5TiG@`9->Fhdd)ii+!FCXGdoX z%JMPGuAEj)Z$rO`QO4Z34_NULaasO8(jUGhOl;BR3w=>``hJ%9k$sLHOE5OT*Pwr0|VDGY2ab}Rs!WXq~+y}Hp8`Y z7@6qG@>*JA94##rc#ApWd<{gn=VwIC-8fJf8m3(yvp;Ryy}7or+t@>hW{l4(o@? zLwq0Qa5k}b5k^BEHw!=I3sHudd7<}Ev5IuKh&V{=orkFg2Qv=^^^Scip?#b-snlYuo1ryrtJI|4K23YD_ zNG$3U2IX0^0h}V)<~e8WSAUvfvK@toZ)sQu?2gcaAw0!`34;zznI*4jX;dMCoiht!Gl+S3+Vz>gro_lj%BO~f*ZjI2zt*q-ThM8JIaBw23JwEo80 zi$F#X^*3kF+Nz+>Y3=VgJ4AH?QTEdG@j@+~o3ORJMpBH+T_~UR^RzJ3u4CBZRkg>K*Cd=uIMBfo?+tx|#GVM==?%#kXj;dIGAkE93y?6!6{5 zKw@H=;UFNvKnh+z=6K#*h|?C!tvNQHFUOp-)78l`7%X|Q#T@#vh*xSkt5FGh-gUyz{wL5(7gOipk-(5J@BRIh~817&!4{f4d(aW;__twAOH^T&E zpP9aoIPP{-)1n)k_JnUw@+Es)susFu!P1b8P$}(PVhXpH!Ofh)J+@z>tI3P+wsP?F>_C{@+jYP zhu`W-ZRt47a@eCI`*qHD^%|+v7;Y_AF0{s4Vc+kNhwF0COj(7o#4z*@{VX&gCCga( zwV^K?G$MDsc4zRQfXxpL3F9{i_w@8MwiT537jz7yj&izWkXX>nPJp^Kr7IW8;=yQUG<+uKm)wLK-?g z!Ez%szb}53eAPaH!cB|@TpG)RTZ}JodT&TWtm z9Q|r@l#GMp`gWzLX8vL)F{34BTaOD+3c%hequ+)yedM6*@$FjqemZGxlg27gA}ihQ z7wh+G<0Z4XAq(J!;eBf+N3A(R!m*2onUCRA3UT4YPFUoa&!p$k?;q(&^@>HM%$;{9 zZo+L%KEc3G&*XKMKqrQ}N)=FnBT7tLyAbC%zaMid?%@BYGuTn?GwU0Bqf zMx(BPl&}!W9vS*Ky=x~5${sLf>K|PdiPS(<=2~d>aktTjA|+{QI@3`O7IziW9Z=np z?CHAW#n;T;%d;9KywIYA_Z8I0A%H2 zJ{)XrPAX|p(OoT?Tm8MKT7y*!s)1e&Zp`eZj<8{JSxZ0ctR^!Fbu)?$mt%F;@&gyM z76~~9tJlXzR#p;zjPZ#YqhIs&{A0fgTMQqyX&zavvT|pYChmdXV3o&_rWSEZ){Ecx zqXp&4Nr=jqk9+&1*)d&QZEX)}?`7d6S3=3&di4p;BlwQ7QE0h)KYF=Tm^>gi>!Osk zwEE&C2==xwyw+yq((|lI@9hoyLznTI>4pz&e7ppa>9zPnlK9)Vjn~5j#HXESC^1P% zk%Jp1)9K_^i{78N(lv(?S64pOan-s)QA^1ye$X9`)Co#tvrKL+Y0Uf5au)^3?D_}d zAvr6XMuzsy=1d5@#okDz(H1Fa#TGcRv%0wKl_hxXG@3eJ=GOdIoZ|l`kNQmN-RVfB zkleBr(1?qs;pmd6g;mOh3g#&V`|WF3Ok&|=$CcQ zwK`xCANr-l3mXgPvUB)mV#&S^t4^o>c9~8yu1OZH z^%+$>oHHHvE~-{uKHT|>*F|^q6yJ;a5c=Gkk2SD*&}j{KF03w3a{Gm97NO$N21nVM zC$Gg_T$TE#z9&E27ayw>ywlaVa~s;=kF&sz*sv4`o8Jr>E_NL%P|#z3UDrSKGjM{X zkN%vL|7$$eIVzs{*SJzRk)$LA4Oqxm<~zcLg~>7>Ij@Gd#G`QDHdU+BBf`9|_`HbL zq(ee2ot2k&G+u??`$v8d9xF|$qys(idog&P{hrTE@A;dV1LRt(WqH$th-qEDFWI<$ zN)thd0~8r^AeHhjc_9H*14!?4_Em$z0dxPn?f4|D^Fl^EDd%T zo}~pXXHyzRt}1<^i@JunU{s%9HVv-(j*0(NvQ$SoX{xnVc6D=dQ2as|y4|mbu@xl< zR@U*Kg9Hn7Ua0=ME1o%2LZy1I?vSbK4!s=ommuwyJsG~Nluoaxw?!`Lvb1a2cGkm9 zFIZ<~kScw6BKfw?Hie+;a$fsOw6dU66S0FKIT%~gK8zw48qU7LzY6w1<-n_GdQh~? zc8L6JjVc$bh{*aexbx467~KjU&G*9ogya2wm8xQ)6IVP<4JC# z9$gmLbteqn;|aWY)TviE7y;jBl(6YV8SrBIXAGuy$To9L3TT3QduI-ml-bdWTxIkw zN|wJ_OE4Eko}_{bm&8 zU3#5Xz>$2igJb#Wtzks8 z6N?#HjhmXBcA(bvWwTn;pcE%BC8kLgL!U=~FNGleHB7ma%_`;4PLV-DMp{V9#P{o9p zrn{ql{{5S7a*B)9al1V;_Dqx|DV(p)3JzizP-|Y7Z(ks*@Z!0+H@#4RZtyH~y+)>< z;y?n2g{9d{WBd;1FVG4756~fnQ{?}K9H6F64%=`scIOyCA(~4I6s-*I<=xe)%Gyk~ zs|4cspIMgosZ!k!(+lcl3JQN{3rqN1a-KP#-*{LIv(LKoAv z-axlTOwrTTly>`A`Y3SRInf<$*c;B6U6l7_uCeNVjcExP{|snZL&-cf61{Y={BmoW z8BjII$gd@B7Tp{STIGEQeg-c(Z_*!ZIEzd`Gbf4pqC>50c4V5#FTdBAyCARjH_fz* zw}Tr+m847F#-{ZUlje~Y4G*J86O(=ymQWHe11Im*%G(@4J>U^VEvg-Zo(^Z^^7(;` zT_=2T#40eAUN(8)SW-DDNx+6mvy;4#A6Oa zL;qazmbpavr+h8Q-6Geo$z#2VC!)v5fZ*ZLTwgyQa+>+lz=Jq6ENPJziz(gR?e;90 z)c-;wl|uFRmQO^T62Zx*BrQ6saPwpOq4_Nz{tn~Yz{c?QpuJnfR%Xe|56MP#g9DSI z1=%ZJQ&j}M#yPbT5%lW~rbNcWw`uG^6jAz`jf?k|J)w^{iTEOG+rVhpeZWeH2wY)7RU5`c~rw; z#c3}Fb?E8r?!%+k73ISNLj!i?eU#RYllrlYAC+)TPebNg5($i~br>(nHMQgb{dX=|R3+3igmOleO@VX-n$?FCZdZ1f+T8Jo*Hs14qFJccO@> zFFOHMqLM)qx0~`?ti+~bU{#MO*8d}%T!R#x_7a0B^m`4HvkM}p(K8eu6k;Fjrs31A z6~V`Z*4=M^eSFIeh1L}Idzv-rAw_UPMJe44+O z;J3-o{_)+>fP#~gf?A%~WuM^~Nut6_w~(F>2>2ifrO1f)MiI>JdBIa~a0EzV^qubr zKq_Je{K^UaG5-in8q9`N?&e1Yn#89sG$I^cNh^4%I*UE;DpZ!>vHBBQBtsD0Cjn&> zQwwX1tg^xCLJMQK;-U}IZc_fWYOZD>@$%nnUQLP%$qw&Y2(T{^usr~q;Fgm!xRCjiHz zjOp*DEG(#~3uzG;f47t@v1hu@6ZsGR`Z_MXnNFJ|QI3{N@&x69K@eH)rnbR*4YhzY z5E+WR`_|WI*5p#Hp*a~JfOPdp5?h*ZrY{{86A|DO*}TW!HHeY-ed|uv;6$9b6_dGX zz9I7O4BLqJMh0;$`#(LpDClA}$o6YoH$5R6^A^BsS;3=F?h4&PAKq*%XnR!{naKQKf|B61zt0RUxsnvIvADmZ_b5GR5Lbx*2b(uLCmv2 zx95lW``}J{om(>hs;X>;?Uv~=1mVd-L4L zm3d-PBSkVRu5r%T_Yc9<5%1T6YY)D*_RNVl3v@3AqI=SJ##4?39eO{9fn_8{y}-76 zp4Z&yttAsMM_v$nob8rn<|rDl%WnUm;DjaR83k#Pd_^G|diHl?-8O^8kU#lydO-US zO5%4+R!~IF{Wu3brh$oNPNq_uc;idr8C1YqO8Jb<`=nyDt~#K-;ReW_*zVZU(l}}5 z(>7Ogu>+Cux{tH@U=V*sHtBXHouw8Pk(2XBoUBPir%}rH{v{x)Cqq?pdrkg5FWF4m z>rciUi2#oxxJyc6rY8nW(**^i&kV{k`Er+T;m~e}Hrsy#kx{ciS|f`R)@#r@@(+87 z4O;O8_ETdJDqj9kB0}edDBRbAm$`3^K7YohnS>5i>wS-|$F9gIJkR`#vD4ePm3GBG z!7!LC=w=R|d{|!TcNleI0DPcOr><|Mu1WF3uzg7=MwTBFezK~Nm=dg*3i_3`9?qXBuQdM00AgB3DGH6CXsEI+vZ`Za?~sW2iONQXj|ij)(X&=r>?vYn-?`t`94Ldp)MnTCn_8$e zXiZY8GOxGpzg_@^Fydor1rtf7{i9S%E5ja%g7XkRC}vSuf{#6a zIsWVA#Hp`8ztMho%exm$GSy+xLpZcZy_gHao39SYL(OnlWJhq9mR;>N6U%WVal^^? zpd+a9BpL1K(Rx$II9GpunWo*Nj4r{Mi+b$F$>@kD45{IthdpXW61)prI*hYR`wzSU zZ8Q%n{0sCE0YELDfn|&VV=Ysf9|Cl+)H^qIKrSgkIwq5rZmo`i@waG!%9{rYpdNmn z!v`>g>;)eCS4xBj^w|CdEq+Y*#GF3bYxfNk6;uwLGumS=#8aD?1)7FE2*QaOA{J_K%gsTyK64Uy3|x%kUCtD0T%x|9Q)u30l-)kAL6GU#^>&)goInot2ip8tK3F~_bhT~M?eXMQc{2pr zc?8VP@D~Tk=mFjicuh?%*WN#O-%3eFDW> z!baW6lYr(cn57mJygQ?)_R*(k<9|yVMP6;J3l3}u>Qu^eE<48F?ICbDw?laH9&IOl zR9;D`a-IXN=~OoSZgVkvok)GX z3?2%qOy<8;;(d2Pt~LUTu4ew`e(g!h#$%KZ zuAyOWm|1noCb5vVfat0DJ(KM=pKYx3(zbQiro;y#gQg{#J4EZFkSeA!>P}8CFXVZX zcJ5|UFs^9j2>KZXFpUN9k4E79*s|*wX_;=q5zcmW(wVZb-_Uu6)f)ZHL_iaH(3lZP zk3)rS;^HP9C<;cMBCel2+{}q76*?nIOcllSG0mqEP7@$(vz83GR9df-MfE30Os19f z{m@3r11H7O9t{CEaJ>lXmWpVBAqyZp{daPlITBc3V`8vsynwiy`;kL~-h@h)o{FEJ z!D1fpJnl3`7$p49*)*;#x)v4$++Me@Azy7O@HGaPlw5NKFzQ-mHCJNi{Q6n&4H|Kf zAocj#`}B08^&kWr+iLIrPF+1S$XT;xm!FCQRM1-#^NvH>BiaoU2YcRF>|34G~# z+b?!s=HpI~FgQG-6ZHHD{5{mS)WF?DCfs;kTG!FqQtqv6IPY!@cYm;a<%(nLZWv-L z8DfxSZPahT0##E#A4OKxvl~R+&q6h|h5K;QJ?pIUh3ShmHN}b;`YBn68%v~4^~N_* zLHTNHiQCzb@`r+KJt#b8h4a05xRw_O&zrD9i~5(v$bJvf9EwJD5(hSqNI@TZpaWG z;dC{UjbRvH4Y@33K3{e`pdc@7`1Za_V&KAi>C@f!OeaJbNF#BS{F3aziHogc_ZAwi z{?YJ%=;N4hRq%mBCCzt^dpcaIw^Q%IN$+BJGrP(0D~V^qdo}Mcx>YxHE0-EW9X?6M zOk%hT(vjM=yiGAQk{rq<%JC@w-xnd)C%w2#gv_hAib|I+_0pl{#4#UU_F>AOpTWqy z+UGJX%IItJl03uk7ZCc3F*zb(SAFCuMcqVLvk`v=buWDn1IhI_Nz+U-YDDy+5&f(z z4p!l!5%Gh`uug~C`$+j3>}VE4GeoBlA_^qSW^51FMsGZrFp%I+&f~x zW)trT9=6FJG%82KOy3!+A&x2y3IbS2U+)QM^#n>_K#Q`5ZdNP6x6&gBsKry z{S{ifw>p#Q0@6AUJmV9kh=i=nS^fiHt^h5q{F;3Oo}e?7nqqZ3p zWl!h!uYh$F0(6Hof{#`Y=?2ipDO`VupI$ichb(M9HXL*L6vqu&Gt>w>TnW^j?rK9H)-m7i2PhI@q~7+pLu9wuPipO_?V>AFvO&yFXxO=`^2UVcK#0 z1IE2rLcy$8V?1T}WO{=Qjz@-PffS-|;tQ+;T8%bN28E7@Gm^v5-C3dRcA`OmYD4w# zGgp)K$LDAa@&Jfn>{V*f7)1e6N@fY!$pyWMs~y@IC^KF zxLxa2GKd5`69pERmf$|mzMefE9g>;O*U~C`{^-wOCd#O4_6D39_Edi2T6*xEj8H~R z@9V!A3<9HpJtBTYhiUWI%)e!uB4U)An;3?}}q7r;j7cPy@kX!KR%Ii z=`y7TRocr2hl+{A$k2aTA$QNp2Op>9{YM_FIaq?2xJ)I_jpNOZmNWONATC7HQJu_+ z6SC{>bewb_sF;4ARZgQ@OcR-m>^8kN^E!cthb`jYA89^+GLon?>#$NQn~$8{Gnl`G z1emb3+za{V$ayKr=xIG1H;I^@m zUv|S?zi8upC;;`DgM*`BUC+bg)wK<$I2n0ntiG03AIH@{0!{kWAwstK>W=B6vLn=x zf|@RE(3-HI2nSfv`Z7va;)n&%Vk6g`J+3cOi0wB*oxP(?WP8k5fx%Z2*xxN}OP&pC z^6rfsyRCPjHG3|Vrpt;CE&aybR;)%<^S~w3{1ha3qV7|neY6+BUP};tx4C?d67jtE zzGD(H8E}F(Xw4_i_q9>{6s(i7zJGO1OS1SOWwII>BFW%gt;Wf}vCU<1R`V03)sl{p zm+)jmm4X^ff|ijj1+Rp(KW%ek0q`FA?+Lr%SuMe)a8d*5B=2@$W!SIb>dFMg_B}G; zrGyewy^q>gM6A&1z1+e=yw4qwuSDrXT($`7%DpET7i>_l;%eU2qjG>^+C=Txn6!|N z%*-O)hhFS=P7(3=&HDqj(&Vmwc#V6+)!xpf;|ZN^yhC!jy~W8UP{_ZWg-ZO|%G6+>tjG0|1Io@GA-AiNU|BNH zgBmttB2s&YNN$OC3p7~#c29Y&q3|`VJzMxG*AK)` z-Km-rzUz4)sQvq)!Y}XlncgY>;j$ICJsXNSuJc(=XhHu${?|!T7NWsgmVlo97koRq z`C()$0e^dA)un6s{`o!w52B5wKR=dTF2k-n#5yui7!BN7<|V^~>2 zC$9IOa)yT=pFFZYDhXH26E;6y_Ms*4pKQMj#6b$H-v=)J0a^=)L~a9i;F|G^&(GB~ zH*fJB#B$@);cOB}OL~*5`HX2+-0e;b5jNs#a3jRQW3R9APDzP|GbL@j68|l46HOBH z2c{l^5Rf5DOVcs_`0;CU5kH0VkBX21WvB0X-tC^DH}t#K)^i4O*0I4DBzu zKzY8&TgDcfK937L<;HJ@TDROU<+hFfQhL3K z6gb7Dufa5aqM{f0a#I0ef5xo*j^hSA_P;?JShKRS z_xG>Z1u}HG6nyvzwBf=evXVz_@fZ;QF1oh$tx1!(Og-VuJ1pJl*xQ8UZ}S(>g1+$pJbymZJht@)tXAz z)`hmP3N!z@CLCHhhWCU#diD<{Ub<=oZy)8!j*KX)k6(KrD-JnO2w*Uz^!3m4bpU$4 zh0dkcpAes^q~w94CGTPqT4a#-(L?$_-I<@*=-wq-SR*j4%nipy${q7>_>^K|UJstu z)Ho@q;I@>4jNw0%%KIId=P1z?wCpPn>#=eZL`KD7Y-*WnuEU6;TZSVBCje<}ev)ss zVQfmtO8=KJxPP9qtgAX||MbuTT&@MUz$WWByI-@9%+|Z1Q00UqvE@M>bgg+X+dgGs>@PUrdTt>l`+v`sAbP7bz!1~RY+-yHBBklZ zkm1=X?o&o1K0cvrZhO=XiU@$$B5&y}>Hb3J+R5=HZ^pP|p;$&jx(!G(f$z^VKhvE> z{&9i-&bi1E|J+;D0>Li3BpDCU%MV<8YEOlO)`gkx3F zNO#?%oo_L%uhOZzaEo996*LM$6nJL6rZ_+>T<3Aqjsf6TR?c@|z+Vidz1a;tZUGXg z_^UrF>>=b-rvl=_5Gd#dGrU!NHa)vR?Mu*L0>>9FhJ(2nN{NW`gn&Xm)jqH$;?~79 z@L=c=`rKhap2^b9VuE|D}U$&dE@0ct7{q4^g zpUZ;+Qva0)X)}4LtNTRO>^Yb3@AN=8Iy;~IZpr|SJedE7^}K4W&Z=9wK}=a30lx5` zLo!C=-7E%o4sUgE5#gigTcW@$MgwNtnW`%qORjw={jDEeLgNYtJ8Yh(Hx|c|;`02h zUjcxJFta^cj~V^V?6ag}0E$6)k9@KQ6@Y2rnSg%Jx7@;ghL=T+>EFmtHJiMtL9TEL zunXDddp~MiR%4P^$m#3z99kVN$Y+OldM9TY0uqt(=d~I`x9*HlyoJwUbL~@Bp{Aht zO;_O9-0zY8aoXM85f=S-*dW0i;3YUWcOQ~DtGTk13cLt@LPDx7)jH<#1;oB%G2E}^ zTG-ueY~;p{v}iycRYfJecf+2|dmr{Rnx5y)-#tIp`470u5{~VG%>x2G&Z(|r!BW?= z3@t3RdBIZzkF{Y}8J<6}7OX%nZkBt(`dvE;)Plug^y#mO&$MC<6r<}g@eWc84F_pD zJmWgT9ElrSn^N66{%q6@=O&gU8SnrG;%qAVZ1RCpB^A|amONKcg?U0=l-=lG>$s;% z*+3sCDJ%pWNiJ5cLLV$;?gJkYN@z9({eWJhV^ZDngZ9xDz`!U35+pHH*k>05!>)Wb zpc4T;h3M=7^6scMM5gh5LGkO{zelCE;=-Ts8ai_ra@~N^8BbQ$Pf( z+y-^y+l~vok#V_e%_uL!#UX9Dw1K2mkKBBB$7Fum?_Iad8Il{;H4&cnUpjC%VN&-$ zbbt!w!@nD?OpcWevJlhU!T|+D_r3LzNn722RgnR8hQ`pC2wjLJd1an^d9I--Wbk@zHJX4-?H4l zfKsb2grF}+fs?jN*T7&^x}FByUVE32_oGi|T17=QDK-eiiGCfKX$xM!LIDV=A5)LR z)w3Jy6ND$(#A220(KmsSWS*%y6sEG43g6p6aj)lB4b*im84NcP!dVw@*rza&3e%CY zJw)~+;>U_~CH0mk44b^tJDsG+pi$Fui3dmN%Mn#?Q`?;|PVr&V1l!u*x)xGSkYT)i zT8=)+@%2xApL!nWr;JJNGMa>43%m!MV$g1XVyqiSU%c!eD2IU)Su{4a$mrm0w7TU} zRW)z6gY(?jvq||7eLRCq$T*RYvmi)Ul4|RM*Q{V~U+F8gI=lbK;%J+bUCC*dcIfan zM^Vz%s-Ni2)$ykGz_AhRC$V@=NJURq!`Y?QPtar^jZ4o4ftLSySEr$yV(RuUkAohd zvbbiet_m1MR$^d9J~i>*-wvZV;n5B!7a{oQaPmGc$%vfRjQhVJ&4pmk^#Q1N5p?Sg z={>lmA5UFDf=!Gy+a2ju2TsH|rex4AMyebQouBf_phTfjuYRJ&OYEB^Kr2SMoiRDB zKOf9zk%dqcEWFmJhuCEJoKyv_tT4o&o~u9C@=a`Bo(UtT-lm? zP%ME^@P9ZW9!!#VV_g1ghv2A@xaBgvWPjUawLj~EnLp9v(e{Kc#*$rB)H>=HG^laj zo?2gd5B`yLTYR+@vhO0ADU0vAeNm>(ffWMO+{!vx^Mk|+@X&N^THjrye7Wu}I#go$ zPjCk)>-Ic&<>u48i!|5HPGO?VH>Oe?OvK!k5;gQ|`@J~-Q6nYgn$fkQ-8NmUME%%| zcQ?gXg4dzgdhp8XpFJlXEAU|D0P9-9TkxJ2{V+er^XU|`Pfo?R?0gxJiUwd@dhv7U znA&1<-T2ea_{V)@^|9K<(VXzWpYp8Y_*$%eOExYb!%Qrpec7FZfD39F&E|m+?0viT z(cSIc=jXLM+hVfc+=pgl=U{XE*`8V9#zz$1dyEi86ji%;e*P0q$#;yPx~8VeX6>;4 z0Qogve;nP1oxY?A+h-#J#A`m4fq^M4P=9STTN_1*5=bo#_xIB;ApR3X^0+Xvn>z$B z(NU*7X29%JMf1S)hQc?Z9VQc^1CsH%=rr zV`9?WupY)ZkU!yR`sa0e4=i>N zzb3<=gfhu+>DK#qp`{x;?u>7gyMa14P&kUm$DJ@aiK!F(^0@-GZt?Ax{e79r%KM#P zzcFR4;Q4&u$wKy=`(JxR_2oztXMrEj+H97{NdFvsJuVuU(GDW#sVeL*upvg$tqr%o zei1U-+pmUhxjNF~-5MN`dQ4T>*8Kkq(P5-;b5_oP?QHMwXB&5JC!)M|be;o$A)})c zNw^x};8b)gikyta6ptLDw%yLGf7sjJ&Th_*kBpQP5C558V5b3myxAq!+ghNoXtpGd zz~!wD+Wf(+52oiw7Dz#f1p#ZazDRWipS*AXiwcn%05fet0pD=$<{+@h7|47=<7w@l zdJlArBFhm40CVmQ{P1{9px(TR7dUXB{}13sY-o)h_k&Y3_?!2fE-&oBSt|*AJc~N! z6_W^}5}umsvD)ltXR>s-F00Y3SGhK{k;`yW1gC~p0@)IMawJVbG40yj?^Wl^3@8pS z6WdFL_o$#?nN6M_w$=X&nAR1DOo)3y)tR`8c3xc|R7KBRX8dwd>In5Y*xG5;@?t{GKZ`DaPWzc?qeT@U}qv)#ON z=3v-_TrQuh;9QHw`o*f8$>-Nj9Ed<__Gw(D6MG%DmB2976Qbi0rJy_k(Z1ku7~G(% z=I^aGooX;=O}V~)EtXm9Y;6^~DrgEH+ZrylDy{+v$>5 zgZ4W5-wB+WYvG`a&lL|SM^5BF;|vWAJef_f?7HNz-W4VKU~k*~7#BBvJY1*cE5%$X za>uV(xowlT$jMT8a=7H+pC>&xx*aX%pjqRr)$Lqbs(y~ypk<3kTzg+zNI98d-o0V3 z{4b*JYqV>-#Cj4|GC| zAQW@-$saoXk_sX)=8_6$*zIX-;2r7Yhz3cYGq~(`Ke-H-$U>H1XG96G5A5XWA6p@~X@dmFjM+Bq%W$q=gc(Y1M zpXp8WI5&4LFT+x9Wq%v0P{#~d{Uydy$Cc;G?Qv;t&I53P!B(8q=aM+ycXzE&kq0v; zQsNPawUsaX)HIkWz9_=+5AW8@`wb}lU;0}7k^brq`WBq4FopF12WVbS4gyFs^$vS5 zlVRjm)&qiOhxp$9Dj|K-adRa=UGiI4O9ut6HSqSaMN!n+! z2G&tgQ8kXs`wk9=sdoDepy>LAXW0&vBsSlSP~T7K-M`cGUN>v-{$apOgNXZ8%+it< zh}8`9S2@1Afp_j?|2SV0*R@wv{7{U4fk4+Gl3cKq2zs`=|0cGqzlBx!A|nTulvHMY zqR;hm?&qh%@jhx}7Iy0U{%0jj7mPmAL!Y$aj`N4&a4gsJZ8UNR(c*7nLa1I6NJARR z%78Ve(v%u@G&glBrTvBW`pmeAZ$I*-)b5L^w)8D%q5szlP_kT~k{jkXsE}r9wd$$N z1|b5oVOXJkw{`G;bWxD5=U#uN7rsmvQA8U!TtFWl0p3dR{yr}e-`%7kE8E1y1xQo< zGjZ$cnciP3{*gAN#pU#`ijZ}8TkC&uI-b>)dt3c0@j9T%ILeDpJc<^$I9*$aK;J(M zDp5lI&4iT31n9`pe;#fEiXVUGv;jzY{WO~RUAVZF6y0tbB3a*XBX*)i(@<|0?jnHkA$>R1S1%E zeJ9N-5?OFDRi>op<1YR0{+^c3;lAJ2cTRW~lff@R0AhlKF^06eeveoD+Xm!AuwhE8 zK<*F%JnMo!7j9tvfL49K%jJ^^3Z<=iky$M_Cy8!k92w3tKY(vHcIHnds_9;7oQ_W> zDk-W`_Dk%&aI6rJc0$5%4E8u@n%~qS#-)RzwinU~!k<`bWSu!Dp(XYOxq{a0gnrRv%g-*NrBk=gy|JI4ied;cB1ZA+XfH{s zdpYJ<7*N(W`<;n*dZv30Y0eqkPodSRk|rC-48*e_tYx4CclL4uUZc)0zOoCFsQWXe zxg2joza{t_`!kBX2**tyrDGC|m`a9k|FI}>&g$Yanw&SNaIIvi_qc*_3y`w2#6tcr zvsG#)!Jg-6F&$z$Qm@(;GCk)`S?~Zl@tp4jVUJFIr19Kf)dR+Ral}!1*P$k7;!VS< z_}lZJ<9F}R9!f*RT^jg2+m62VeUDiCYoiAKvMG?}lo~9Wtv5#jYdvFinRq}j!d_{9 zrYje(281Z1+7{P}fame$EkGdz5tP$rPQ+ZZCS45=K~n>`J z{v2s6Y6Baf7Df8?0t&%&7qGh(QJq|zp6Pe+)%(0y@Rsfx@KY9oyM1l6I8c2?tSc9{>Hi3TPC;;R;(=58 zkDWBYD(7)>aqVTnfnG|R@rpq@CnpTYi{*%avK2LVs5MsD%QQDY{v|rb*(I_V(^bbq7}vheRyos)>{`ri(m8iR|01bWUMLUxO-)juUALxe2}B0BAyPGg+r&8?{T9< z59+AYz3xvq_|-?0LAx9S-!mM&wlQ)SW6zcXWn}8k=CAviPV#NY!fo|t(e3D=#oXp7gwkyT1#$M< z&P=_r>5Qbmn}pSCkmaKa)Hn65K|My-6l-vhKODjs8`H}eho#?}6D-{s zu@INmGT zAuEz0Pl^bQq8=3C*(eO5xrpRO_-n5KQsN4oAvdpSeB6A!w;s2@4&1g{1oK-K2ruV+ zj$6n^cic!e-1c^m)MUf*`hS<>YOtrMcFjrOCdA`rG^ksn%uWjv;)jL*!=-*t%5h!w zXXWT>ji)rO`8E$joh5gv${I>zxem9i?dByI4tyYabZ~X{r*2CmHNG>Cmcr#cgEAho zs&DB}SXfw5Qoc`=23|W%lEHLElc-ILNTSDU`>(mQYO9fZByS*oS3)xpc2=d(Dm7X~ zTxeZg=%7Ef)7ANE&vI*UoBQg^n`UTH;XH34sHYK@NBZGY4zG?o9aWG723&blWsG zd7k_}T2{XozrP>o`f8vNY1Y~%evxALY984A`%70l#(oNl+uMJBzbycJg3hR-TLB+C zYEs60pcz&p4EdLMMW{J8S}3@$&L1yp&0!s0Nf&0Olac@BpUqFA`2qJ~;vwImm+EKA zwCYUWxOAULNPOg1X9~$ERQ${gtFD1+{RM2w&Z8AtYVbv@NHVn;fwp%TJkAR+zz=}C z5TA%ER)&E=^!|tL$v%cFw2EA0CKcU>5rzoDZirvoUCNQP;3LPf??1a|>Iodm>by9h zgo75v1p3+JCgAm=I&WWi?E6@gQ0mWEG>&}^2q4A9|I_)QDm$kJ$HJn3o{=g3$iB^U z)(foA#A{Aa6wAnWp+$r)eO=ReaQW_oq{YNE>CwMi(x9x%q^Y)Z)jq4H1O>2sHaKK) z3DPG7YOMyo8j#TT9ouKzDjOTSg)znV(ZJ8XQl1;kL8y@>C2p*VaJ1e*<5gl?}XPaUfMuudf1eeO5Te>Q4Y z``Hd-Dd6Ig_3clNT3JFf!Kqw3k*@2X1nXrWB5vZSXG{+{b(c~2JoBQgrUWO+a(8TeJoLVq zbnqJojA>4AxJ^jvJ1SJGAY`>id=J|?&5N6pZjReG8M(FON6l?!YV%I6S8K0X+DLdZ zhswcJc5XOi7c`)i-LHDzUrRUHx|2lrMHoJ8qM@RP6ReNFEg0Q#E}z{ebm2J}Fl_tW z=DljMWGwP#9o^q_3_W~ifRLWDx)$PK5v+rWFRRo3DWw3_ zQ<}!eP#EX;F#5(gIzJdsf1Er&t87G3c}Aq5417iF0=?(GN6iIVn0z(lVmv%C&Dm8? zj+4wAH@+<1F`47$;2;Cd9ZpS(CL{aZ@d`~YJjgbOzhTuH6isGj0dw0YZB>2b&G|x= z0bJz7U1;;=X3pFxfH4gX6*aS%^VTg;6d_4Ysk#P*i zDQV{hBC!Pyu->jhNy7^SajWWAolns72ps{l1ZO4{5*%)Q*P?vdB$M#d;}i@ole zTas|Y+uq8d=P6*F2;Yb(H$7v3w12DOy8d$}PDu1|b#$8UMCtp(KcA$!_D~GXWXyx} zV?=NX!(>CSsO}HxY6l}$=D64pa~%4q&Cc`3Oh)Mv$tk zQanCNL;Qe=ZNBX7Q|^n7tRJ2EC+$5;qgCnJ=Zrpz#c z+Jw7^EWvG;5~SSP0uPKPl7#7i$M=LCMmZjrIlD8#BpR2ffHDgT4jW4}gdZzODg`j34H+v7q1 zfL~OKA6klEaKsm0@xN!2x^ajw3jV8#>Z(28iOmp~^-?{1abeu_Ua&?{uvB)(TCn5C z6(ugDJAtXuQhUd;PRV(BLb#-#Iiw;K1!#!E_pQF zhrOP)EU$)*&&dF(5yAA!%JM3%e|>yc^D935OO{u4t!rj(Dfv8w4qaUftY6iZB?N*m zM+~k^>Ahz76c%x0 z)UJN8Vlw>IfYg}+M2dZ8o~`J*cB4DT<6G&7D5^tWH6kgz>_QjRcl_mxj?bP>kU^Z; zD&(PoF0td}0gwnD(&2J%aZCPfvzq+6NfMT3Y(BWuyIci@#l;ru)-~QJsX%SKcNsgk zQQS8aeOk3UM-<(D=g|&h0mo`?pkMmD( z0+nQLhwh-i)EbJo`dfrFPkgs^=auyl|F%0;4%MxDwQ`FKW|e zrvjtK8SOgh>CV*aBxgoybIB9Gp8##-ds7fhXyL{lY1Fjtaeom7G=!6wsn(`NnC(or zfsi=L?bAVE+kW4(c^FCI=W9F}@V(;z5!gWxDdQ<=x5hb$oXT1r*?=eX5~2{g*=IbL zA&^U2_SRI{5|NN_YC4f+iVV<|m&RVm;uaKi(8O<1%P{&!%+H+jp6QU2*k+KsojZjr zFi_msP|zF3zNu#~L@JQ*w*P1O+P5QN>d;Ke;7*1g4DlUM`MBAH*W~0HzJe|<)pEzo zGskJm=YA2Ei%sB8FkQqB=vB=2;}Bx=O{K=c%dZ!qmsQj$xD6X zt=BoY#kPGY-XQ zG80P1^YK&wbn5VV>chyG++8`l`IFK%^1Y~M3PGdGoO303%XQti^_ovgN{`3u92K2H zkVezw*b{`>gI^7>H}X#y8JRk9qHTIe?0n(V72ER+y@E$BNkgQvV|BM6B34#(va-n7 z4ueV{0%I%6LY*c~9-Em6f4XI1ffBW`K0U46Fd_nP)YE4v-_`Di)}@k9cRM3OSYQBi;GvH$L3`A!lAe?t)NkGQ;Fm({|5a`&SkXTAU?<5jj938_?* z&Q3ntF~gZsIY!g9cM>ewoG9gXTN>w&h1jF2;e&vj0&+qZ~(f<}%J zhIK3JRf6GtDY+?-5yZKst}aUnr!CfWA@uQzuF-UUl%b2<`nt0O#lD}rHTI=C2z?9v z?KrM0PX~XqGKqKr6atp+Fx~cz#v%~(;E84j1PEBqRl}3LhM8t_Iq)F7yP}YCy^=>1 zqvT0Qv+X$Dkc*8?CAn<)G0ph-1W{;mMk&+ zcCtna9LDRpT%*L}wmVfy>59z}t}_LXvlk)qdn=1~c%_DR%0X@5a;tLhJDBZVGW7Wn z5N=Qp3=MUFm2+vri5gmziBhQ+u|jm}ZqOR{+iyM=QPZ53cjsjqdi5l5M1l%i>VZIN zEWCAhyMKDx9;50b#Q$2;!1vUejDaCn+FQ^<P!Iqqg2FC6Bcs0oMZ-b6s^USU$Nj-PMcA#EWRH`Y+S^b{L~{!Z@JY|m zFD)x8J8`gfu}&dO*7;YxYzUPT{q3p!XmcW>wf0t zq)5E4&&gN8Cg>z){g4s5mBDXkS`DYRJ!hI_;bgR^@Y&yqw42?ah>5%RP-<#4%k7O5 z1X>#!(8!~;uo&GQB#bF7ksTVk?fnhP9;XSu&pGl)^)^R!5y{+1B8QvT=s*m3QcKyI zv9q=p7!qz!buJvdO;%9~Fnf5^9C$K|#o~84BuG8^*?2;XEG$JV%{ETNHM^#4Kub-limeb@4LcZ-u9&mHZe9elyD)?8VjYPviCt3*gF< z&W=Fok53O*VnEp7HsentUHEZbilp_b1$07^?`gSW?x--yqeK*HzSuN8P9?*jq@f`r zk46ATv3Drt9YNCly&f*!q6f+1#Dp3y{$@5P23;B>Ecx8AxNT?317rz6y!MY?Xn?OA zR-!wdU0lqw`3Psn`5E||CibHtEN%D5kc{yiY3IYlSEv&5qM#e$`6Wi@frecs=RP9rx8kX4Han9V z*tu=@7K2HC=4O=cq^_vWpgidbphjCBVrX9uJlqWqZ0co7YROOHR=scc4L9+iJNFG)*k*Q?T)zw zr-hd{;`J@#%EZfzN00mYp%ggl8N}jcM0ry@6{rOdO~-#%=J_+LVAag`0@mkZ?=`UR zV0Xfl8rb+5lm+(Tm@m&yjD|T|C1smG2(s?_iH9n7tU9ff)CmP>6scC<4vh@;8a`Yr z7tBPOm$Sr_{TyZu*3X`ziP`&EPG4sH^hcWiBqeECR8te<_735AOSp|rv+(q8GX+Jf z)yY<`?5bdK{&>x9IS91L%oOjl|0Bayty7}s)Mv@HUeH(fGJGGj@8fPh?Zco#G`Qbj z@$CHEwii1Tx7NrHqpTX!Lk*uOB{fmRK6~IlB|}7%$YvLpW~(efaX&NiB%c~9bh13? z1a92vnkK@TeuJ9XWEPzN$NoAVFY9gce6^{8A_jF!Unrrhf4qJ8J<5~OCy~aA9ox2PC7#G zMwez`M34K8>01m!nNl>?4^6>Eepb^9FsTNiyQ&00*!?Or6BS;N;Y0k|SSkj6r_#x& zq=kFlq~%E*#X+Yh#lB?M1wMdzc8uqhx^+{(tM%qTJPW*8$EtZ z?}89pQdAV(=XkewFB8u7hcyfM;{&V0P(T>T|3l0uUhuxh63=|18G`Km|n=8e5nbMd;Hz~3VMJ3Aq zyh>&jE15v?(o++#F2G5OUW~SZ zft+7qWbH`tBO^N~X~pn|YZApIB={5fRFc5Ja97tX2aVUZDE$|0XnJg)Mh zqNA^RsFKLap2o(d^5(A7p@0pg^rm7)_xBc#{ly&GU~2Y$i<7B6dV-Y=OiqwMu< z{U9nEqPhyPw;ncWJtt!2(&?l@ z(|e*9zs|z#;{aYq!juZ}>u=botB(5L$x~kbE9Xl+G|HNF##e)Kr$xE^vHYBvc)Wgm zmO=@{%C9gJNV_uq=?`!*Bx*P#N>1S6v!e%wp$a#lf+g3vs#w$Y_f-XaKD@zKK}C9t zk3DvMeAA6NQ9+}^->e*zaaagFRR+Ut4gEoW*ih!LGycy9c9ehqn+)If7c2;MnIy!- ziRm0Uxse#mdnxWNP_Brj3#w5DwuR5mnN9D>y31^^g966UW0Fl-}hqF`=ODv#Ps>w zc7}=9OMZ)o?T_Mg_Q!jw4*5tbGB2y?HneIB3cr8S#xt#-8jbAD)wrMbbw`TPIf4=~ zFyKDat>B?Lf>WkX%JBOBr5#xrno%Mur5ssVwZzmwyoE5xfhRkfkpWwG-BkJzK_IQQ zRNX4+bt86?1m`d;cykOMA5`PHZ(7J7YPH+%Cx zmR5o@2ZWd%pa%YU=6gT;wU+Cf%E@bCZ)AXN0i3uCAVAfXSwVI%km!RsXi-eihXc>H zj(~$vHgo!HKJ*JCxHMIbQGyLK!JLL_GLsdLL1JP6n6eoEb z3n3=6QqLe{vvqtG4RO5*pqS#?;td+rYTLk{!=Qc92VoFNHCGr>G zWp!4FsjcyPmzF!fIQXX!(7PjSPD=GY3A$WbgK>SAIE!8xh;ebp(-6d0kV`4Q!Cb7+ zV`hU@*GE%JhfS8k3TX7KAPkr*NBSy6^S_>Q@b6O?|G_ksaR{dG&BPwT`pwRwZW83e zKC$M&*F&l1?Y&=d28s*x`p|7qaX%Tak9}k)Ew{exBc))Bk(s@h01GSd5eWM+W9T4AkWPCcY*8j?7d`8ts!;|_Bad+ZP0rmvFn9@KJl+|>2 zLPmcstz+WQ)XCtLYGMe4$}fcd zFhT^Lv+yYQV7zyjYLgbzfVntDip~b>d^-UQH{&6`xYFKhs2tP(9>~}X+7(3e3TiP4 zstlzZ7kc~6yLE&;Vp-WQExYIf;_L6%k# zLd;z1sAB4a&jTDXBe=zlPS=b1#JN6UekIMhgN7kTl|L?W@p45fGRCw#;R;qvi8kkd zo?gQLNNs;@2TAWosM#2Sq&%U-aG6klka`|IF2VD#RUAM#M9Yquqe6+SqY);{6Pi(s zY1+%99=6*GR)~*3JDMYdEZS7KtI;<2_pWoeibku7(aBh&2gY zy(#MN;YL5|czqWn`C37)b;}v^E_Hhr>pzo2!|l$>Ezj8P)TGER@G zunH!>0LH^siXtfuN^Pj=Ku4XdkO)z?&eGsEXfW7Zwh99Z;VPO5wq|!p1hFum=f8Kc zY_r``7pM7Bt1gM5 z7|n;eF$LQ;0b^#Q>jqdzTeG&=1eR=)acrXV4_zpkb!W zt+{a~sPg;A!hT84Nwir%aRfh(JpbLeY*f(_$W#f!lpO@wZsf`EUo*VTP3;zMXn;ZK z*0VwLg#F|*Ck*}h9-7_|#+=jd=}bKe-VcBDx?4Cic1qOrdv)<7L@0`f^tT6EKy6g* zk<5IMbs0DF8h7@r>oz+W4esVP%+e$I=$-Q)wytI9BO6%qJL3F$UjZ}ev1{0!@53n9 zKhF5mU%yYq)H*^VA`uXBvflRD@V<_w$rqq7*8ccP@#k4uyGI#bWXjWmTluB;I*Ciif}{ku{x^KoIDBw%z|@_%amPT%dUb5C zMZo6*q}jR`r`(V3FQMOM&Ij$!Q&^vcE_fxkH&cf!VA6W!x-=s?s-hGWw3ZTfaF=ik zG!q=Pb}b3Zlb$KNKd=S*pIGBmwmcod%(XA87GF{C3r1hK;#j4aQ~g3T?$_$*cU!Ki zuZ>O2hU1cc@a9Xr5@X8eowIRUjOaCb@{J%5L9V?@_iN+(vQHFzb`v1Rmlo5m@H2gh z&_7LPKKNi7#tklkQt`)eG$k27=k~hc77`H&E-bcys7XLl;4qT6?T(%u>@hE-16^bT z`tBghsN*ZQ31xWLi=022VJa5ilIj=Z-p>$Vhy8w$uQPTbl~Iddjv-B<`EL=<5!-R$ z&Az>jTT}B83On6JW*NWo2LcxQ4b|FvzqNldhwr>YO5Tq)j>>IpZhyMBI0k@Z*X8cynAtLsBV& z?nl3(J>(}HtU9g=%Dv2hK^-CX>6@*OLNBjrA|b6*X+_ElfA{mRoYRPRpfFOfVWLgN zaMB5_9HvCozlW!2jT*AeU)=Tr+^kMB)|U_ua+_j)I?QuZnWkpyIQ$5+uXG?jjFa^h zw!a>o5mU0#gtE%PR5K9$uP~*Kq`_1&GF)_N#YnNbP@JlRvTV-b*d)Itq zo@A;$(gb(7(G>dM1&tl4l5uZu{b&rK%oOAfj)BA1snY>=X+Gn=?O%I@&GODjri4q| zTrxvuGH{|79wA1I0mq$2oe~)d?Pfmi*L+g|`CXudf(Rjgn-?2{?!SQcqKlK?6sq}K zKW@ol**vZ=Lt>Z=4v0=cO}Pd|Wieiovqwc`ss@I+v&d32+fdN6gO69!-J}KXPca)i z>|Jnvki%_f5l@a~4zjZML;Dzw-#AgLv0TKpx;m{;X$hVul(eb=GkoUn-`MO|1)JM= zBz)5GM`Pls5@V_Pbp$;3c=m^bz!~pNL#(DhQROcz{3p^ick*hzW`K8}3=qzw$|UT_ z7&Wy!uA~z)qBo!2D`WlVVQ0B*8V{Ec*;byrok3h88C4_+*&H-f$-+HROvM@{&WR8S zX-{ul>3cNQzXpG@-NvwaY*aFv4?aGT+3SIF9HS5VTf>*?Bqwm*pXp!|y@d)$Ke4bf zyJ;`==2b@dYAaC?IH$y{BL$qN-9e~^ZM>))!+4l~?;(4m5nBkKv=U&|emXZmq;ibB z5kgrlGjt}YX&aF#W4z|cJVVmK|M6h?!}R8e_V)sH3&BaG>immy@5?7|ME{0@7M~tZ zDw#WUzTD=c4AXQ>wH!JpZA%O|nSsaji2EBE2~Jw`-w+$(j5i6xi%x!0e4*oSpeQGg zqtQoqx{qU?^ad#)$mq~FeU>$%TTUH&fr(tA>zuzG8#6Y_52_jV)OvTN*fdBM0vy-a zo`Xykb0ELNAEm1li^ptL=pt4|;tDu^Yx2do3B8{4%@06jps}{!4%2a#VZ4%!p=5f&;Lkf?K1`FrCqb4T7r!QkK z!%~8Rjs+)3fY16nn3QJjYld@NR3DsI1%GQr4#+*8V)lu!78};kYb8;>M6rpuF?NXf zmdgjm#OMli$WVR0={_^)M!f9tVp&7A{Ji0fZlD1R(|78MbDGxkAQZhO7=7KihUyvu z$7RxNrVR77qb~w$BJ$Qi!fj+CAo=vx4d$z5rd-Vx?`NBUXgb7I(UE-7p%vajSgzG5 z%sD5hPD-gBRy3Jk37x_Wc)s_(yVatZMBYCg>>)BF*lVAvNpXJ?$zRjCQCQwhN)j?g zV;JOZn6bt!{39^`f*M+n%0EX!BNUoiJxq3=?a&FGN&qvV@C_7m8Vo60r&7gWDkFU6 z`8^vvBt5ix;O=n2NlOL$K03sly&(Z}qSx6UQ;nOk;5nvHJHt9Yp;_IV7s730ZbcV3 ze}W1=qkHGVS#U!Q>lFYaYFRR{md}UF!HTd+$?vIi2ultiNW%QirQtGfJhb%3j-LG&^I?lNb z?O(3qP*0 z>bp?~w~%#baY*ewov`4is-3%}^@t7sFz}{G}c)rKewDsPvR4mFdT+MU#PY&|8LLyoxFcQ5pDMvVhHiN3eTRQ$`Nd&IxFi+RuRkAY zL`!~={!R~k=LyN;7emhA3RT0G#bRVDs-@%zQ9B$94RRD?L@7`sLq>zz0j^)D&_8pu zUu;b5jY~T3X}`5G zeakxdCZ}y7gGG4iJKHmgIcB4XV9pW1fRY0n9DqtCO7)Z;LZ1*EFcqx+@JUHqhJtoG z`*sEs8#jFHG}4%gI$TC^rje$ZVX`}(!b|} zl;>~08?P5|`mqy)Bh`cBxj`bp~=VW`a`}qRltDnpWL%5$ouU%kHH&>Uc4+u|J3uyea-- ztCmPb7GwWROfdv!3_~UXIc~?P-^J}i$U|6|IVrD4PsX@X-9)cK;oS6xf);!m`=XL!}o86guCZ!4?&B8z=>hh-V_Rh^Lv? zJkfc3Vg{y~j!vbSEGUX3W9ceM<|(vKq0R7j_bb$*w=Y4@ltPOgbq{lxq8p{6Yehih zGOu$Kq0pb4@9*f{Fx~Dk{)1x#KNkR-m%?*NY*3@!N6o0E=T*1KLx4-FkK0iOt zZf?c}M9+-L7uVt+!FaOxQU6+oD1hi->ru((n2=XDtMX<(=cx~PSrAGVf@H}EKMu35 z*RSXyABLgBBYKS!Ri~4dFGEXob$?mvaQf>c`>WEy92~Eqi4vjpW5)lA-~Wby}Id{4P4FaE`S!c{4o4tFqK2xc^(j{qXFEVNI@a^5nZPQ zT~YXilA(X#3!mNdpyrAG4$55|TW(#IilL&fejM#Cp6N`*SPi`#E!eKwxkyVi{N3$+ z06XTz%3GA;7fW1H>%|-J_PY~=VoERDr=oyyw+xDC1-mdn+ESfP-MP|P);|y3C|}>w z*cFDbfxY5vf)MHqx?KlfZL$DTE^AC@C1f-VqhiO_-g=tX{u83~K#>*gd90t|bx}eV z@ti5F{hnG1UVb`(ar=9(k7&EKljd zLAEdW-fJ;!?~wRWx}D6Cq_5p63Pzq1<=~t)g8y)P%Gt>Y1_JKKe3kf9pG}L2o%e5= z)<+Xm#t|tVO#h7FM`E;=C%V6|0kEQx$PJHRpm9DAmkY1*;-Omcd@GM9ILWSGH(#ur zzbqhI0^nak!QbsDD@#i9oMjCyF{wN4#wR}5?D0YA^0Q8$qWbcWRl&YJgOYn`#DpKH zFeS#e{eBoj&6$0xry?6x`zl&wjC$pdoXGrL|JLR`uGR^(Qnmg4rZ|{Pr_h1nem{l0 z!0ElNuKB5VJ6~iWKs3@}W=PuGa8?6oX8Zj1(952dP0jjt%YLLG{h&^&n7)h#7-6NQ zStB^;1W5I1^QvZW|S%NQpO!DtvRpEca%G0yjr`He&e*xXMf9SzljRkM~u#r zExWwHDR;&x|4>30z)rkA1D=yBAf2u@R=}Tcqn`v!Z4Hb#lV1z^F`DAOKokdLz^Ppa zS}Pp7WUm;`kCtV$SHbM-jCeiOUN%=L&-fLCBHwXYoc;%`8hlsKfQP1 zVLjW7ysTG`nyx4!JuTnk5*qzIHYxpKRl)oRM*i8EW4X_P;EeAvtj8_9eN`St1F!bS zpoL@pRo%`{Z$N!0@J^+rU|oWrd^6;}uABOd zgqg=%?hzlKjWPQ`aVVn(sYmHU5w_5CjK{!r)HEmI($T{W%rzE4x4|weq_P z8{@lW&5N%ws~9U8P%H?Eq+GXVszlLZXd4s+C}$gDCu$TJZS@RjXE^cw4t zF>G{kG@FtKQHyqO<8f(F;mP!vsfukx<@E(28LYwxFJJbj{7igR*!>tt2Z2n|v8-g; zSeXp_KhGjU`bem7+s&K7kHz@ky#8XO99f_oZ~|%&(-vCP494GuYLJyh4=Qf&y3(ZK zj~v4LVpQ8bo%>w^qhAeuE58}F+4O+1zkFrC89#&0vVQ~gT#^Rwb6T8Fjy!AhuXLZ;Z;RiL$y zd&_p!R24srKC-DgRQ>eGVJ|+ZYgt zS6$nDDFD05amxzNpFXs!08zZepaBcgc+PVg*Hv?yzEZe%^ikdg97u1*C4Eg>+wkNKM zLc-!rW^-vt3yNLy!?(4qKOJR?okpW3L%U5Cx2-wGcfK1KN11iLm_-_|M76cCmL7~0 zXDSkzOFJ(^rwWGczzVHD^ZfgYdeWcN<d!O4J}SWn#bi>)y-&s-Cd@x4KK4psEBS zFO*VrxC9Y3sTW3Sj~R5d;Ghi;r|`JdqB){=g>UtjC^$Ls#HQJb>LgoqZpx_2WDCPO z!{-Y@IW0xnHAOm^A~Jmb0>nJ=|6HcL8~wFJl)wA_C|*n<75w1ilVygqEeAE=1iT#6 zmqH#qSZIYQFShgdMt6pPgNI4lqg+x|Cv|MT8&6m~TEh%JDbaUP42{!%zRP z;y(NF`|HrA;}KzBBtY)nVln8bWx6V=ZJTc}!8z1nmG;roi;q6t=`pWgqSiDy<@Z@@ zZZ5SnfaA8+q7u#<-zh(fl{fKNyu(Oqk2NvF+mnqzIqqJ^=(I-nvYi&jG#@kL&MP<^ zc#kctO6J0&o(?KT$@@afj)ZiIhVc}D8M{?&CU@># z{4|M@f(Z!;0mr=@w3ga-_9(7*&0HfiC}0{bs9IhFdK*97UVPoDaN)fXAJsshC&=`Y zDtI?PrdAahIRSv(Wy1X0ALPrn-)NP<9UC2!mgXQk;r1aRJPO#!@h7(%B2(f=5d zQOWK%l;U?c4cVKjVn@JIuH?OiDcK9*gE6dCFv^E1uY(eI_G2aN8~!2_2miGQVufV% z@(Z+r%IGgrTo+kWJf*+kZq&$@c$3pv!mM>C;E^l$r32QvfsfI;ccF+972YNbWlV`Hm!3Z4(b9Y_l$aa# zunhZoKk^L=IS9K)ljd<+6+s~S>u@DA&Vdj#&JCjwu2Qx2o_4B09@_aYE88Z}8ToiDr|qtpAE@arF=8#LEgYgJ!Dy z{Xr2#%L5O;Ct+-?MZW16T!&pSx%w*n{JwXM+(!9G$JKAo-$IRhwW}a&MdXAl1p9|8 zBtjPysxC>4d@-K@jZ#YWT0S49w1BLnr@B(?y=l**{*S!6X8}Pdh+r0G$Cb5y9{GY< zGruo6f@JV~PEUDw@GpO=uB4u18_$KCx{@;I&weq4n=i3@0Wai6PK+;Xb*7mf_F?&P zy$sH67okq<0FD`owkUpP8-mW7zo_ zP&~@zfy=?|e6P4Y0+<`g|8JpWzONDBlQ`v}Jbp__N+I)Xc8XOjnv^W=j3LLR4BzVQ z&}ud~>FpdGJpXFaWOw<_-k#pNWggF<;g>llB?}7)kdb^9wBplPYC>drTJ;7r{_EZ2 zm#?%rF^GsX-ha?CX(bOmZ~ZE7%_1)$p(^Js&H*cm%A-Eaq#2Ysp;uUh@WavnPLl=A zQ;44t#*{YL-1eXyP`H8EFE=JVdIvv_8g(PdxF(}1PZ3?*1QodWlN+z=stE=baQ4XN zUw1ErOj7qDJm-ZwkwMji9F{$05-LTQJ+I!Z0g?&3MF4Z*1rUD9zIh`v(EPt+bD4`Q zcnHu`>J_#5Pt__67Sne{?$Bag?Ewkv^lM1mOZ8SqE2H+1xkmWN`UQHXm+gb$m0!;s zn*^UX;K9KnJ@fE0j}9b>W2gwJ`86IYtCtq#KFLJ9J1>>%viT*%1??|U?Jqg)vxv8M zKiM@klN!b02Iy2V&jP|svW`~(p>3rz^rdZqslKKXNt26FKAgQ?^7c3AM!j}+5>^5j zje_L32BqRZxO!tRdx-mn!)#f=)ziJtytWa#CH%NA?}BKDf|h(pfUJmsML(GG`gnU7 z0e9YAzyNf^u=`0M{QoOYl-wf$b`u-#&sP6H;KypWisbRpU~&)!l1FP^aI`#T6sk_a z27M`+;zUKA!A)iG9Wc^1?XK!gWPw<=mo&t?$m4JY}e0bZ?U86)i0sSZD|Yc#2-{sG%p0 z*Vl8$=0)1AK;}F~qGF5;In=I&6dmb?3wlQPGaaY-tYwBx(*;)O{JA!IF6)wMno}}# zOlDHLTB(DAS@%au+2DL&?zF$vO?)n@Erp`+^*}^7Q)>80^Dg{8v*gNT=L>`T#m_Jl zU$+Aq(EJo3gq$A*h?04$a*eM{<~uLj%#M`5^SUD4-0;pkY-`-FDpG`8><$eK^g0SJ z4PEJll(eY?4*T)MA+dX{6VXx871DxfuqgTeJCHy*FTm{(ixBQs*Fdb$%mwz^%PO_(}ks_^u#J)i?u$ zjTEc|kSPI5ru)O!)Yu2YPS7B<<5XoxyIDfNu~i1jFaqB@d{PBE=1GID#x&2E6day5 zA)wF&fn$Xi*0Ww7!sA#ApYG*@2v}gG;INa=^`5;1wnWLJ1s&+ZKtwP$v4HRBrJ(R) zmm~$MAgdd&NIQEuIbn6=wl;c7`S9el-%!EBTbIwL5|X=6GBG*)M{MM5F#!b)2OmOYJM3S7f@5{@|=uK<}v@=6S_HP?D?7|baHF7?1#qkxGBGl z`VK`@BZ8FTZ&rn)LB&AiPoIFSTo>~z6BC_EESSEDtb8^;kfYattgyBe%dYJz_+G1M zvcp;-c<4hYz*c&eXtbKm$pIS@v?zEM!}DI0jPK-BbYa4>?3Qw_)l3WbUZu^I@`O-^ z(HXzCyIj|7^HF6!`S#^qyrK^fo&&|6^9%yP!$Y9kNlx903xce+PRWOv0|jta%-RY% zDs$B7nZqA4dlT?af`09xcx4OKUr#Ywjp5RSX8dq*I|<1^K7~Fw)+l*%2}*k}K0At_ zzMgFgDCBpBhgQ7gHri$YDRHVfd(JT2tMTrPxP{tM%F){4<3RA4JO>ZL|G~0kq$*yA;E=~hQQ(z;a}zC^6vKq&&tCb!#9|s>oagxM9bjE2|oS(X3rmlelknb~?p<4X;-QD}p_U%j#ARu7nUE=A1guZ6z4CK2gxmSi3Yo z91_e_f6USI-f=B!W(>`I#tB@>4KQR%SPi@n#H*`avRW&MD7?KyYU;{_5o_MvP!}6E zkxFGl*&OI!FCHz20!holjgBN`|EfJhr&5t7bI%-?t9aRT*WSfib~FYx%WK9`un5(Q z)g8zpd059+!34%*O~<`QBh(z@;X9_1GtD12@^y~KaM`CY)s+7aYi}J?_4mDtA6mM* z1XQ{trAt7NPU#S&LAp7BGzbU?N+U>@ba$vA-QC^Nb@%c8eSf~*@4a{C_s8!w!<-q% z(Zg%+z4o)7=UHpPm01RABW*X2y|odO;ts3ZRehYhpHb~{7o(P|J@nlb!ND*cTrt6T zt*UTIFxL57LEJOIZ2fBG)#Wb#@7^J7HE}1*j6Zrn=BNHHRWZHpT@%WCMiB_C3j;^) zr?Q=fWEBy*n~{FQn{6IQYsJ8BXF}n`+us*_;ikYcEA-6=foPsb#^t@?zF@H08@Bcp zm3f}H|D=+}eG{KPnyWUoqC$Rt{FR8HqA$Lo4G!Az0agT2ALfkR_hq?R3riS^1f|zW zJWe`&9yp)^+HxOMTPBwB0*3Xl&G5|hb!U*7d?CQggF{16pvN_yNEEW)b-P+G2;N2E zPxY)Vq?HyRcZWULN2~iNu^&u2TQe|A8?XGmMas;_R5zV#d`HCA8qr|wKtafaHKn*U zbauCx(ILU`#K6Q{85wA4AGuH)gy#qHO zPX1s_SD|G${8P`}5jfNaSYJ=CYfkRf*4|M0s^@xYYK4sy6tq+kAZW5MN8MZoGV&nl z@)n0*+>~$I8&ymhbCZ?%_B%qLiu48zud8}j|Im1;gJ>$j+3KNahPlRJdGAQi7=b7# zah20#_;@LWr`mA@bbF3VtP!o%#F7-33%&+=`@))E)kx;(;mTT1u4 z!vnMdIP?Lwy8BwTel<=#PiCtgI+TB{WuRUgi5PNdddx!V3o89R^#4bp|FUl}ogG{t zFG^896j?<3)FkYsIL^V~!QP(C+?q6CF{682LWC9IPA8yFZ$PXBiVh($vV<}-T{;vW zSAoocm}8Vx!x7C+e*$SzFnf(0dU7l^Krr|f;iCt%P8*G%P>wJ)8w#ivJw-Dlna?+% zu-3rylCn!b!n^{OkAwCX#e4dIlVXK|6;P|*ZsvW}r4?Qd=vc7QFtj?dlfKv^dc!6c zj;pJhvx!)~irB15J?s3CG`!vt_qu0Km{{yV; z&Ba*h0u&sK%ZS9ccgtpghj>nMy_z4PUE}I8>LXmXC33SiB6;X3d|`5c3u~PK>g^g0 z(kYN9EGj4KOr0hvs%>AM3tzkHZJx#< zG6kNW7N9N+P!1}c8`eA9fP^*X%?XY&{db1~4EM9~Z2_#vZW%+y>eIwRDTM_m!p9*L zcTgQYE!d^W2Uby@NK2a1Z$DYU`K<^Ar zj-rK(ge^4RVBIYp_&Lk>WTC42X#i&?SzZZpJzb)P1_LA7y9~`dKRy8fy8?o@;P~C(J^coyMPQsVAkneXv+&BNWwpasu7+1t{h<_;wRWM zIYu#Tzitowob2joM%Um~3j*#di#g|^^whfVG0AJb0MwVY8Q-qVlz?|)pB%JPN#tka9G7u{sHTs$J zBV#2QFe`KR>5%ZkMH|Y;#r_(;jBoWs>saV7zhZ5kzV(fMt|Esf?t`!^_<%c@6j;v? z*WD$Q!E|ZQL@6mp%BcMz7jvaJHLoVY)z<|GFB2h7tVROpp|-|C+V8qB&Jk0eLiFq3 z*ytBrx9CSHUU7?U8FnZ*R~6K0hfGf+x1LC^ZfnSA9D9Com>vyF;&sV#_M0KT&Yr<8 zsaHFbo(J1No5Ssi$U;4O)@J8K1J?Ix?FI}kvcuIC=GO`55yW$m#0K8Pkio?<^~d3~ zt_Oey!$EvZlxm2%BcK}1Dl`kd?wS9>Bp`-t+DFh$%`;l_uW`8$iVpv1qIvQ?=P@Pj7N6tlAaX+w*;? z|BChYZF%o^#gK>Z?!p#I4qDj8^ra;wxx&#ZyeLM1M~Pq$G~K~)3PHNYuBR^mjrEWp z1mLl0xdkpSeR)%6G|yteZ6$QtiFR|8P_?;RX*`kEFH^i5+Cjuxh& z{R|78zin)64}jm#DU*fd@neG-I9oZpGm(P{Q+<0(2+7WCg$9x^UPvrR|40%xrh~}7 zP~7!s>hjSSS4O@?33V?mP^_53b5OA7Pu(BX=NIi7bEreLwSCgE;+l*AIXrsd5wqL; z_IP5i`q{`)_skC5fYO#yv#U+ZnW8ZhEAWFsbdvq~g*hAv925@yjN*RtASNWQp^DF8 zyi6TG*u%%YEU)&6DlR;?a@wxp ziRbwftFIuoafyTRlRK0ba-5`XPVUd7XqfcKDn%s}xF&M&t;`)7}YSxL7n zzuz{CfF!3fN!0Dhs1@OR3PH?@DOz;0HS^q+w49Ti`-AbV<-<{Y=EI>Y+q`^RD1!iq zjwTX9>$3u2t_0z4Pi#4}L%z>mBmbO!mVZwF`#!Y_n(c5C+}j15L0V)B%)@odfO4e? zS^2%Y5dUgdr|2Ebdj8Qb4QB#_$s()oec?WUOZIU-J;}kcNg={yr z3Q!w*mQJ4_CM3>L27G?%Vsxe?wIJ*ok=vBtHnO>jD)5O6a`L!U9lcTuQauco4oV`% zN5t{EPB-;;+7tSNdoopj{(BXCDS|ZfQr^*<#=aQqra-w*pZt5InF!DxJt0x?t3w@E z^eR20eE^@Ls4`wTJvk|CF38UJ`>tqQvGI5s1#W<~!P;)nb&^hIV6pGn`9x#+7j#b7 zm$AEz;tRJOc#66Bayj?+qRzLFcZxk{$lwfvC(FpL$%72`1OA!#cun!#5n4fJ;YBZ( zZg&JK4+};}MnOH%lQ=@U;0>`HG%D-MrHVpH13#cnNu(VpB%2uuHJV#STTc>^J_9+Q z%4qrZ`@RS<$#_2sG@MCXo`*$j_hRZr8E~IF3o3bXlU+N zd>WO&4G>)sc#GTdi_i!{nV3tv>64ogg}xl`8$6w`SMc^ecj8HA{dPS!;d(z9z?r7t z{9pbp;jpNvw%YQJ4ukYK%8QT`+MF8OkFl-Qz5f3Mr^18YMse#0+bSI6Z_|VDb+`we2mn5R?_E8;6P`RA;)3}}`Ag@56wln7! zNMKqj%9)=N#bM;wj{Mr!()IuwZ?7wOJvQ8A-h*~w`3n zR8mijk?;4)-bZV2TdoHCi@t#usml(K`@54xV+K%fM8H7?DP1p*K|2wm$xDXrlpLIE zhHWw#9Y+>$O}K@m_N5KE?72Ym$vDy>gZGExE%{DxJ|afFzY$Ztby6|%_hVhHf?dP1 z#C3F@TJ5lPtl=5g2xe%a-20At6*khiuK5?@y`7&VL>TGMJ-P zrH8MO;8SoO#Gipz3iH82a2|h9H{~PCMi#@#p0^(QU>M;$Ct@Z5s3s4Out2+ihq={q zWd?#;=X^Ln!xloeAou~{Dm+HUFcO2Lz~x~kZr+;ZVmIF9JHi^eC(?oA)vf>yBJ)%1 z^TpezJbSh%8#zzLRga@veIWiO7c=)4f#hj9UIfuRVwDb3Ts}g&PcuHTT*0Bzf+1z4 za6$D{aOxPPyT}QJ2@*p!PtXQ&|<)gvZZL)1_g8!5U2pC`pU*pz|P zW3W=6monDn9UyxrL*Kf2N-&n7BNU?!ra6Mnp?bHFhc z7PoQR+#~Y#IK^2|kok~7{GS*ExWg-e>JCs}MlG;>qw6&~O!0deG9Z5z` z*Zp(qkAtNGO_t*1^_nh_JA1=*;}C@4^V`!-y5+}!1UvWJ;h zQ1>ppAmU=|{st)^BfdD*^A>T|d8IXyAp)E+HKn3f`O;?4rq+Ljeqib1yOVN$yy^uS z1k2Y6k8gpSXVKI~aLaB?U?|O>vD+#S38iRE=4fYu^3;nWRWX8|y2(xKOi_Gya{j)D z!~oJ)N0{V-@36P(KS_xDJmQyYsdTP@Tk8w5H*TmLhVU=fpiPc>ZrenBxPxyR0u=?6 zlH=I=N7m!lu>`i%;E=n+N(6$C=9bb1YWC4cLbt7+YA_J=_AAPmoj$ItuKrS+4c`&5 zdV17wPVuohHw}ljk$^2q8q?N>g+-IOSy(@^>uQ)R~h&Dh?uCILAHx{_z^DyPM3g z$U*Z7Z_CD#LSfW%O=%`8tDU!3HmLjB&RmF@*Yqx+_#G1hf^iJhBlUkwxF8ToMu6g( zaHiC(drn>{{Z}j^m?}EpOoSer?fNBm{vU~k>i!R-!KJi@jTdx$1ZYT9o`-BU7d==h z>}(cHF;p@*@x8xZw*9y!0w-%^=T?oK;tVRWhMZ~ah&e;Ly!!1aqOOmn700DSC?gMa;pE2ZJnf-lf=VhK_sT?HEz)dsRa8Kg8FYqpb*{ zORq1T58cnhNf^Y*1D{&bpNaiFd9jxcb$XM2i~O*uqV54NE8bwe+*$konC{H2nDX${ zEDzWh1&(Y%ghUZzQ&OM>+d3^(-(PfOjMeve4kY=qju$V$1Bn8bJQnU7Vu<4$PHh-| zDX-J;Wcl|1g)5>ZGAk>g;SLR2e1m6CkD+dGd&E)TXC~p>oX6JXB;)$yz3rz8jH&ty z|E58=!8erG4z1(2Uk@L-h7L-_Htf!K1iHS0Ol5j2|xP$9=`4CgtqqtkE>-_rm~ zD=YE-E>8lY*OIvr0Yf;B#cIytCw(+isiOHazEzYQFB&zg_@?;84#e;k0+ZuLMRX_~d%x9{&>5Fy%B_I zx}piQrN*fwfgHir-!20i!J`XCq4rv0=H!G63qwe$+!5C_C;RqvM55%G zo>lio&*!7lI^Gti8YMYI2p|@Okqa2$r3oM#;a=yQ6?WmHcA*RePJy{Y`M-QXGaca{ zB~8C5ou8&7>n$~jQ>ma~W{@9H8Rjav@jcs81$UypKfV63@ z$~p#^$^C3WnFMHXar99itJn!w-vB`}C$p&{J>Z`P#@`{q~(;+5Q(+ey5U`p1<@b z{0{ve|Jy1Mc$Yb0Funj0O)8)6(7w@+ytU%l%w3{Q zy6=h4coBuuZ59n@B(2V(n$V^CQ;;7WI;C2Oz4|O1r3mn}rMEtUQ8Qu^8~oR&txKZF zma-N-a91qn7Xqnng=$(~;^=oD9qtI?Vu~cl1reUJAwob?^~|hqM3$B>WR=hZfn<~jCSf*(qLH`zY$hukDkCA`PRFOg z^jJEoFZv>_v8%h=Xm)Zb&{M%4bdfE4_@=i6)~NK))AR>>5_&=M7>y)U<75h~sn=@pCbE4@=0F(pBRklE1QdDuQ{zPYRwpCZ2O?SuC{S!iv! z$=;|M@W!s)`q(x}sN(G8T;DVLNr;Bt(pg{YFrSD+1MVZ2WRn`7+0zVmc;{F9M=qX9 zU*c$T1TS98B2{*PDE{wP0-pr%z@0##A%GCsM_v@n;7_UIqsq>^!aBm6BxOCwbNzJB zB7(4dSr!8)7b&t^e9n}c%T3^jdZwLv@3)H!%3X5{?)hvC!H|y%4mvR$CueKRE&1Ry zm3#ld9g=pHd6}Cucdb;e;NU1vb4lUc_t8a5=K`dcW6(=o2;7|`HCnwj#HYHU?M34H zT`3LWV2(Rks*pcDayB5l{(VZ^|A3lu6!!t=L$WO7JAnnHbeCOh(%@}JrlGJ@y1m-( z@y{5p>s5D!-vo4R-CvUY_{C+5A%&TvYmd-;13!8A_Pl(SV$uKFe})K>pD$S_n03cx zxKE}3=FHAzn@nE|J&6rsGNODi%4+_|s2A=xDYT?;jIj&f^_W{=YXqXv2Rq^a{UD?M zd1*ExV+vL@Pcc>45FbdD(-chnnr2pBw|L-n`(97*>%?|u*k(xy3y(szf#1u^wO^sT zL7tgedcJ;J`mH}uLag|tmdY#LrN?+KB1;RM?w?M;e^qTY;2ITlQ8PmbhCkmF(67&( zVk8{Qb&f6>$$^yHQ2yP<39yyi_W~z-Sr>Y zs>k>T;Uj~P6!wqQKYx-J%;CWC#%Z_m zzrQ#@72FzzNQMjU{$fvIBB8qBa= z{5l7>OzV8x^aG-4=b2NEr>_dpzE=ZxQB}0ki}u~2Bisf2Amt_eP%;(7&TgzRJBDdz zJ>+R&n%oj-+~~eDszoW%%g4|PJAh@szC`{_2ho3GBzpid2h&}lniV_{bN+to#xTh& zOKu~BM+)hmeI}jnScab4ZeUNq!=P>^?X0AgUd<~gIYsp zZbZI<->J0e(rt1NYin{t*Nujq9^YD%oiyoTVgD?+wGGd>pN)A1D1mHwT)6p{=hyoQ zGKTUm5T6pkO|ao=_%Gj=j^w~u@h1MY;6rYp-9HG4$i*~FVvppEB-PVrGgIGdCXzlr z8(c}Z%YWat8jFu&Yp9jAHWF9v?fKL-)mHN8F|*vZz>%S${)b|ZogL5-AXwE5Nx3~Z zZ(hLXE8-iz=Fpt>gE^G?=Go71DR;>Hw}M8iBduA62xOFtsFJr^r!5^@&LNp#NGS^u zGw{SS6|gNw)r_wUg(M=iTTI2-AcdAnE3Yi|$vrTtMYYmUNw_~QhzxQhDel?H|@Y*3WSXXBqC+LM(C1(ispPaorO+o;ss0{k2?n>sBkD=MALs$?-bHex0w_N?qmv<|#T29{n42?_5DrLqc2 z%N}3S`RM|e*01TUoZd8%8Z(@_9GMSaMFpt6@efRdo<(RI7eM02_D{Xq7oQraS3`P^#F3Zl`M|P902c${P?GqEZ}< z5<7L1b8^W?2=fqaP2TU@wPF*!en`(~E_;5Ks=h=X%CvhT$-`H(Txu;Ut`^}JF|_)^ zwA=y|u)o`94P7eS3aumJ*Ng+c@w!6J9PgV(fZUsJrg&rD9%)%2$NgC@s8NTRXl+ zLh|uU`uGvFqMSwn>E|#fm>*Cvg{I;R25FDf+8t}ZHqVBKh)kDZE~GU>AV38q!Fap5 zSqA13Wo4F=mfl<t0{`-!~%)(od*W)C`5&Xhwn0WUFkvQ3YsFrm@6f} z1K+8;6ku=g@}b;jccYJW7GMSE7kH0hG{Nd5X>z zcdUm`m9rI|w0&Ot6j6ioZnWXV(lt64PHB{~Rm@FQC1iuWH+t>V*EzGxBq1o%Tp>L& z*{+d_DFF@K0{0iI^4X8D-*2UnR?ekL)vjuHI4^F5=H{;F^a>$6R~0mA+xNe1zT5Q>DL8aCG*|Q>f;lsq8sV zJUEKm+iW3v_|&B`dMU0(Y}&8jxT}6dpVHt|+26mNwApuCB6=uLpCxFotHmf+n)D2U9l zS{tZXudbbNH0e=^_hj?!UgQ)*qxm<%n{29#1?#`pzCP;@n@R_6op(>U{pZu(=;iw9 zq#ldKmydYqQO#st?ZoP;9>qmzSbPf`F4uA!U;1=Oiqrfhy|JIw7WhSWBuc*jknJ*! zp5@{oE-Hvw&?o|ZwzL`WQC&f&h3;%up3~7YGFdUid(0x)crixZ1uU5!zv&uM5 zmsor$h^(KJbtOw)C@RJWsrU-8BMkUW3;XWcL{J{=TeW^BSEH?;urTA*_0(IT2zAY3 zBNXw^-7z@VDCtvzyr@Cl^@ir$K0ATi8H3uFG6?n+CR@2L+Vn%`!{#>-XytW zzd}XsA`snW&(l9Av!P4DwHmsPbK)Q^exg4J8L~LkrNembqxrn<##dZCovIsL{*`!m z%U*icPhC(jS5{oaC-T+dcU^H)4a75>zv&3VLC}IAfixF9a2ZW=k=49$yORu=@O(_o z0n3YR&9+ACqT+sKq=$$o2uF8Kd}OCS4OoAn8eILVKlg_*I``9P*b11d%)i86APgPe zfot+pUTY|!i*%(s7^i}Sjy$-yOmV((?0LL3IUhk}bMqC|P2tF%$qF=io;ooDW(H7A zAtCK~xESAM(zV5v!;vh=VeU+$sv5vY4W`J0&fOA)Q3>!|KobOJUxHL#!EY7A%IZlY zD<@o^1RT!vNMAnTIzE01TDcAt9@&5@g;HaaEZhK`OhmK952^sMHX~(W1oH#V_U9S3 zx2@a5{G1%hSD!55Ccr-r5wp>=Nk3anZgfNDb%<~q!z|=h*I6U zM_K$-cPvH#X*_aiV!^3$xi;{2x_Cr^b$Re2g_r2Ow3c$})z)x%MAfPLL5IS(Ct2(* zA`bxHL;glT`W4rc>pkP2)xuzS0D-DnT-kCOlKakFl!bjF-@BabN6Y0QACV|f-PUaw zVS^d8iwta~8{ov1O@;!dlDzjOp`{_EB_AJ(D@%)DXy_xQG(8mkn7)u3^5eUl446~e zXU;4jz!G{wroX}b%p=LU0pOLDZmjbK9fk7^$ze!kIpJR_1&QEShE1JR#T~2`{Ykjp zYf9xB#>4gio>tIOvS%xBSL?{#05IJqt`X(=OY2azOBt(vE63>L0xrD4(91;lk!lBnJCmWyrZLQmQ=-C%6dX_lZ&L~+KdhhbUx$u)at*4=1d znbp6lVz$?CX(ZhB9A6QIGO_d`3mK;?B)h?cpjz_ua5Sgv@gkt1$gk`++D;GOl1v!# zl_+_kRf<9`ysbBH+os`nFCLfORks>5n2|Y*r2DVg!@Zh-=SmZw`oMnp@nqpJ_JU4b z>6-lTHIl=E4kI8#SGtMB^PM-&uqlMtykrZj@Ds*dAW+UPSL4I~Q7K?MUyF|yrZ{h6 zaD&Y48A>ds3PM3YJx&g;e{IREG&>nkIqAon?-_qETQ41x7AGXchF@#VVx^WeV6G1? zsZE}B833p8pA1Z#IJfIt!$#IaQSxR3_pMS0RA!u|>k@r!vf?0VR|hQ!+OS_sh)*Ah zQQI?L$sRO3w1kDYC0U7pbh6us(dfMRZFe$Ra0eP(!SBBLXc3iOxZXUU`>G`Cn6}zx zF6#Q@0u}^>o5Kyh@wP4JE8z_Q@@_6c?qpJB^7CwtgJc<=7e6fPny1!yo2Pc|Yd-LJ z)oSZTw=~vKfg9mV&wx=|X*TNj_Ry;9CLSk`6ns zZL834GpQFV@_x_dldn%WXxhM#Lk!Rdf>Ou!UZxig!o;+C9*5^Y3qlo`71{Z5&uehX zvLm0XOWCKWGb@NsyjZjC^>*u<^n=cz4@@^B=UQ}ZiV-t*pY~D1pB*hTTOS>MZitOf zf&lI8l)NUhdYrQN=dYKN26pBx=~j9vM0Hc7c||2b-$(ru4SleI4vPvvEcnY!*U`os zC6exrAK@&!4j`Di2%^eEbo-ilkzp2=dsO+Bz!A`%fK>~nm_yto?0}BDT9}$Lq@hbtxHQKZ`Yn|B`M~@DjVkZ#T0CXfhQ$!ifaxo zbcO!cQboU@nKdOAy?~%Cu%sRCPb(d5O1jB^sqt<)Z+nuTBb%ezzNXr`<=hJ7$Y`aO z&ZMCr<-#>T1>a_~L<0VRT$LA$494X;IcGT`6D*iJm##dwzs+VRdi%7CQj9vMGU6W&GHY>!{Q8XU?_Pxl4^~e;Vg2B8a?fXpd<$O9!>#9XZ4Z zc)s9~vmi!WA7(g=z0Z5h&9k!}vg7BhjeW}?gS-+PqpVqY6qE^R1=1Wy#80!i3nszPDTED)RC3?wXhMYGFzH#TcUlPjCm-Q4vqiPC-L6ujtvG3z~|{xVqrSJRrldHNKykCLeI;)l+I097<3I58_nS}Fr4 zw}tfdXr98vMJ4cpeSGKu;67>)ZN$t`Mamt4YibXF9^NE7bFQ+8t+@eT;S2ZDB|3mR znbm!%%MF3;)_aF7KZwaP33ouGNlu`~X#OR@wCy(4M!-LKv^KHOn|3m~Loe$u`gfG3 zwg2NjIkXKOmSO-ggbm2~d73IB=5|SAC!XCsA(Uhuw|l3mBuspQls^w%J**O^YqFA}+B=T(s=*p=nm z&Ch#(IEH+^t3SgBUrwAL6%>pm>Q3Q|C%m+7;0(mPdJh`8Ra7y?Iio__1KZB+P+zJyEbGxeIy5G$j@|S$# zM>8WNj{OuH`K)FZ+83$L&0A$ObJoOAnCIVR)s=2&7(#u&x#VkzZe|7&&Rg;^^GLqM z?>!$)JG%Gsiml(l99QvjIh{8%=HI&)=A2gu7&PQ4F4hr&#gUODTeDC(DLo3L6zC1F z3L{hrZz75wK5)NTGP|(o;ANa$cNX9fuZ5Bm8q&UcD|Cp0<<~xkmLO8Ctyd(VFS89;liF_Gz4rZ#{rDE4$nfxF8C(Sp9!67jfEdOM(5VQs1%{;WNC4qBX|| zPJ+>Ds5eWJAbz_291=*Ucl1LXHkNg7N}lyuw8ll3#z;HEwJtyBlULxyG*HmUVlX*x z{vwvNC%?35rSxrTci1)rA+Ok0%+?kGq=$fJ&=;3vQw!NC&ybDof4t8z&Eb;k%$8Da zNlCf?)=5C0MZKVP0qeC{PG-S69wehKhjsVJ>8-&INXn^qHA`bHKW#Azi*(N%*u__g zxO{S)bMO1Sgyo<>B09_-lMtI0fI_Vuz_&0Dg{jhlEF7N%u0}f_Z*4+;5&w){+c>>F z68eCL56gWaX`%@KIw%(rpa7AXjz@kk(=GxaL&8%>wKq6hh3@1iB3+d7xA!uKY3~V0 zrJ48jA3!V`w9$HgEzp%8NBa|7qxnzWW9hYYAH*x>VtS-1S59jST{k5F-()60UBJ!j z==AjJxAy;zIC0x*;c!(BfcrLxfqjcFJr@f8M`Im?Tl~F)*BN z?~ubmK++EXzBAeJ)_sc;P*W#@*bpS7XWu~T0(4DfhS_vXTidx}t;abUs)7pqUyy)$bDa=4jf*}AA0~QJIN@LyLo{P4|eWOs#O)90tv^0WMy834n z`xzfZ6v>?x8sD%(fZq&JpBSrx9*P8E{4N~*tgyHNmW(Wve*RovR^m33vNgO1EZmYM zdn$ncoB%*HaiOK>cWlC^VAK^EL`+GK5i3(phDwNAe)heM9h-dul;bR-N( z%=jb{%lgbOkEDTjib@#T+_4WNB`%90^nW#pX z&o3Wtg5htUUn@`Bg8)dbTKwlINJe@2^ZGfp>knhoVOj>)4;Smq-1-yXDS>O~-E(yN z52@4!@ev22O=vBb3w*%bt)i+)c&u~s*n5}hV;wT09vox8F zE5bNDkP^JBJe$UmOC8h=vvZ!v*UU`!(@em(7F-j__rIEtl=jh{kXPKsS{c(T>RGk+ z5^YydeYNDr;%0=PqLSZ4G=kiO*|x|q|9|Jj>7s|f9(G{Pg2 zjw|=VF@44yq{Nn@&to%w?%u zD=sh3=?NwW>MmnkkLkeO7qE@nY^rP1<#9xl68&!v;z79%RgKqsAD1!`uG~me4@xje z?HQ`_@O%)w88!!lCX}{oDda zqTxd(c9h=ldkd!04fcExFJJV^T~u%1-%hQO8_;;%z@~}=3cx*q?EUTH;0wL_RXe6+ zeqRFM1*G5X0nmS`itiq)VUC9K8eDv5vV=HRLXE_yto-&$MRqSu%7ug)hv{4KiwF=0&j zV`nd=r9quXN26A#AqX_LYa}G7k*^>WIK;&GXCeWN|5)8{ zJpF(~p2tDISgNd`G|$H8T0`CyGGzCA#bck-w`Y2;jT%YM{d9mj<68mIAtGcTg|*A? z{>%j^3*-wp^^)M^I;;-3c_#J>-Yvc|r*ARNq*3t{lAhqNM)lLnJXp&?~NjL5aLmwW8k zF=A9XcG2D(PK{kmwFtc*qMJ*v86)0N)>9=N^7(5k;$rgrNx8~^k~)fO8~S14jkx+k z)w4BB70(cOGL&{IVBiB*Hva@*Wow`0eC}&q|2YuY!W=6`R9^X@0%ir4`yOTqO-;{v zVab0veiFHJFU$Uay4j3R>DJA4=WAr9ZmyHUnop(C1C z-Pp-XU^$*Qa$=%wt#YN`jaLjO9{909(X?@Hhs>X(32aFioV{29<-;J;DG?4Sw;nKN zAPA@=g8Zip02@pQLf&t1gPXs0zy~CGy6t?Jg08E0rgE<|`S|GQcbz);Pa?K>&66HD zsj0j-D+8Cs8}sPa0->IPCTED3`jlc;FN1igEAk+GBfS;yh(B_lT(Pj?h+fFd!mv#q>5 zIUwUd#D$3CAOai7@HeLTL_;Z6Rd|qbzh3GyA|v|@bNa^1Jw`jOh>1-bF5JjdXF6J; z7G{~d2L`8St5<-Z{J}J9+-m(KvORQZF1--IUg-T*zct{UCM(V#1QYA8nNt5{SlASV zH3|(UYxd6z<&~ z>lD@7a{{CROlUB^jn})8a>9TM8GR$fPR+{7uInpn1Bg5fY$dJ50Nn=M*yO3par4xe z?0Kbw5P$&bj%E#z8#|#!6omnY9*~J8=-Uln(HS==gdZ&+Xr~>IOgjYr6DLc4(oZmKCg~CJsuQ}f1 zC|`IeyDJqq=6XP{GCBRx$q>TaIpD6^^@Aqz`K^>zT(wpM*1w2Imr8PS4|iK(}3PTmgP%TMXoi=F}Ax@2P! z%l>Zq;;E!`;=qMjQWRj#-7WeJU1BnAgo_rcB!DPu=%dGd56A6r%1}kRCtCW|RkH|w z5QRlof?p)P?a0p9=y}fw3sepQsSK=z!3^~=$#L^L3x&TcU&E|~a0P!^8KtdCspzxi z(s9G&9>;hQB4)8ALcH!Dq;08QU^l>O>n2z(u}gw<%) z2*TPVfl6js@!tP8uRzDlHjtu9 zOG=%I|Fy*2qknjkMZtC~$S8oPw3Qd!Ql!Q3MlUx{R~`gFd#DY9qj_y4Ej@T_cpC_U zRT&Uprg7_xH><~_M*&;6$w{@ag^f+D#EXCoDIFc!k8E|tO7>^y0e&>fiI+}qB#Jec zfNaT;%t|wp7@(3Lul@EwBIV{x$kq22^Rv!0V8B&LEK%&`15DYu?$6iFTw#q12X=0YI5(t8s*z!BO zm0Tfd%ENyY;jvpUH*lQ576ni$ly81MbR*4x^zfv|q)LLEJVN7~^~2|%|BrkhunNHI zFEhdNTrS*e*X+LwIE770mAo3Rcgh8N4KwDyi0JD4Q#|)CbCFd@1WR%W1$>y3X=icP z0)%kJ)Ix&-tgC9i?~@A>Ld)Iul=an_2sLLLa1I+^8LGUBsH}XH4WOHzXd?sm0MS|- zKq(+WZd$YWu6e4tAIv|RBd{fM))B%11LE`XXxFBXvJ`S^w{dx zkHThvgWeXunKKSY^*kP;LFO;&snDOCieyMr!MK5-=aN-%ch>yayO(qh`(3RYQ6v6S z0|+boWMGfytea3;S(8JQ%HEt7JpLx?Hu6=mvoaq)=X$1jl77zDVJuaZ1TAeS3Vr=o zC$P8xisZs`A`@EtrXvzr=<)(>YO|{=!mb;gy&2t|=P4YNA$+oFfbuE+f2t8>Le|&J zm&1BKV)t6(zVvE9J$zRVKjamKXJgCr+?oBajVvV@2D`V=&LB}ju`p0V49~5u;{I8R zs}a3=xft!Ct#%$Zs=S1R5D=<5bP81&!OL2F&k3sp0M#}qtL4KZYB6bP!v{UzUxc^e zt9>}!uqVqgrB1c%nm^e~;{2e%(PUl&yccUdXeEPhJwoQ;&Tf%6FL)0x!$h5GBuH<1 zik?UOo9oiOexO3|d~~AWo+A4ZCOSW!{g2a?^1X;AFanm}tr-LA_pl)2VGp}ryb-rp zCJwmIEUb612ovHpUlhMJl_uG8Gk8Q!Q4qmC#pt&`vkajjf^D-u=vCShgrh#8SW00~Si^Zv@7*eOtHIS5DRD4+u z(9osEQsK@TA$k{TizldD{!^kz-E365si!(lPKC~bqU>C&ItLl5=H^zdJZM4iVW`rh zhIRo}BUBCw903now+NP;T_?zSd8=h>P8=l<2CVivf#{BNIQy+(ot7`xeua5uxzfVQ zGH{OS;_|=rNVY|?e|J6j%qkQ|L{Y`X4PsZ8j5o(2&msxgPUZ!^kR)+J?9QX8v&^S# zx)EzDtNxV}(cDYwRc5_s&Rqq0hoT>o`3MBJlI`aaMYM^8FkJRp;0nhLag7<=C8Qw0 z?*WYaG!xay0@*W1$UcKEAX$WDo9EtvB0nthoLu@Ky&Y$Pp>cDxi=TqpW4fMA*VguL za#0k@z;?xx>M%C_FGBQ>yTVIr1pKM+&`&_kmYD8Ajs z%v78VA1yI@RP%+QCA$>amC(h(_kM>vS1am`wGj+Al1?hkrPA@<#&a(FHo z9TXY<&%~{!{K&vR|KoUd#?JK1{ZiwA!FxnIP0e%d=Kopil$iE$c0-sXYX1p;!yKv# z0nLNs7TM&_^}5w~kxNmJ{+jPdTaWkWHwdsDB+pv)XFx(w@fkL^idAMRlu6eHbX$XFAYb21+B^B;kS_`5}Y5`FL zii2cp*a=W94vz7t7v_A`U->XDk^vaAzU~SU)6;v|a;_DV{&-HrkLLFsqPUofv0|&C zezAs6Y{C|n**~h=RjR1ZR}x5udO+VbNZ zpDPmtIV-Cjd)@4U(#y^5Hf9&NH&t0$43+uS1-GWUOk3+^pruqO0t46SO7)nv)?6It z#A14zZ|#N($}$=pw@!*1;7i`}Q$Qd9|IF?>!$z13hRfd|$ia$hq}LQk__7wRM_{5_ zH#`JZTXW`mADg9^JHj3#7z0D+s#|uyfJ>xyHw8r1AKLx!S>Zj+%m`sl9Vx$)xNp%zxt_H> zudz8ajVtrTlZktj!&U*K(VMpUDQI}a2rb_XD*JgnUq)v`B~+U0^)*c$ zb^p)S?H1Q)pN3h+<< zpRoM^uL_Dk*eQWdotsvRtz)|j-L(g39f&xfYbP+(xc*x%o+w|x{$J(d4DA;Y8#$s) zA-@!p7mg}6eu50^lk+`r{p-{>{U0)ftT+QC2~?G3XFtMft@!)Qg&Www45daVmZ7TX zfwMSWQtAg&dlo7)W+9Psb1}9dH^nKR0=!diu!qx|O9JOe>tgk_qL}V+t$U-g#75^6 zq1cIt*pdpDN=9P~r3D-WLYT7AqKz0|WQfGRs31W9KrubHJ#8T%GFCPN&5~J*MN0KU zL*1`6Jb>>r){&?><|e^PX^`I<^zpbuxR)&T#*qYAvyIuwNM39mfr>_ zCUjr_ex0*1O9+E_vmB>nEGeRbCKF)J_oBD6$%CTOU|1DirbEWPdKQZ)q8dS1tg z>vKuN1n9#*ym0!=Wq{)%{-r^u_B$bbxed@l>B3?QGA#g@w{C;(F1sihnw?*W4N z?K`!H5}Y=wyB6iH=O_q#A4pbE&3*83hwxKre=wqt6;x$t=``lK_n-agHKYx-Zm&PU{3od1-oPMLE5dI(;xY3)Wu^C7RC8-r(INNfe!BZ1sVTqlh z6Zg(>{Wb4|>t%?$d&WGSf1}2H%ny$z4R2uV`y}QAitBs97x(Sjhzw^nP?)Rzni*91vko3Nd%j+jpUk?s8 zH2WDwN647Q;&E@q-Li9E7+o}G;Ui857QG8tkjf!@5|5D`DE{B`t^b_4chWV zborfTsfA#;<9$|(l(I#8tC<;ufTBQA2Q_khNTJk^N!O&hf*mf@S=$t;b&)DekE%ye?g3lL@VlkL!q4etfOo z{P0Z9jWa4(c_T6L-xhKFX@ez6{q581!bGGGUN?DDRz&3~7P`+=FO~sbo=A^hRLqdr zaANAo-q&4%*Sg+zRv&4~>hI+&bz|DmeBfaR?`J7(u5Z6;@Ygk|C*@wwG`8FSc<3wE*c{g@M z3Fp%B0^XlD)KUgK!ckkm26Z(`f0c^i%#n(D*Tq(#t%VBR@2Te{pvmxjGdi=1_Wr_E zvwOtn(Z*g731*;e(=}mMP(;vZzLpk43|=iGIJwc zk9GgNK$1@O{&mTL!I0J$(sfdrs`o=hHCNn4)pe`dY@7DFjg>8TZ^Kn030&T_@$C107fdeOSF$Oa$ zz2DEWA7w5qT^`ZS2w{!n!Ouv(oZ*@5WsYR+H$Q&vW$xlTu)IXR<%B$WN|O;xY;$(9 zTeGooy?kcsM;e`PTG^{g-ns9)5*O1dc@>z0j4m1`l0hs@##QNZd6WC7$Dj_Dsm9Bg z>(6A5O;qD^IBzVMxtpjBEWbmaTLFAxyCa`AXQbKLW1lBRtOn%f0W#=cj8@+j)SEok zSjqnMU2^-X^rg4xS2P(f*`s{2I4u?Lp<+YBPR@$&?g^Q;*Q8qgOfLISwJ=O7jf}JZ zx!BuNl@88ut@+C8m8c#rPc@{`ZGM1H*)JiE#~W>7wq??OpX!mZiSGX zU6$VtxmuJUk3ndOWbmPoE&0jevSPzT=x3;2ML$`2Vg~ENTgT*B7BgZiOt~3zriw3% z4?J3P$79i>R0_Sd(Z}8mSuzpHHvIDv9OM7Jq<4gyGe?$`F1SEdn|W}k)h>29aPmn= zf~2MjR`nx?8fW49XIKUi9e;#uxFyD=dUDGx=`z9Ws(au8~cuTfYyPXBcnqsV7e}ScSb<_Lk)5%U+v;^ld1Y zY@Jk$0NAjQ%Ncr_TC5P%)$7uYEgf+lViOJV*&!XZH5E@-%cVQtnW!5VJK8S_;i`#H z98^DBJM{zKahkgG&C_a&CPg3IH1W^#&lK1fIo`90dVu-DQYNKonl{WvdWgsU9n}06 z3Xs$#7T|Bs3ZmHI=@6^0l{TU)iqKPnt)JWbbSbHYP$C7o(`Sb}uDNif#e&hq*Yff8T`1G z-Vin%f|j1UcP)kb9_ib|Ek3KIXG=*EZ|a*z=;Zf`qPyzZx%vJxI%Ma|zEIq~T~uK< zZRv*fEt!xUa!oCKBA8>72GN~>oXHn$Eet+hWerdEjBIYiXsA*HCkQH{G;NMVsmq6+ zef{mf^SwK>dn3+ZMXD&Ah67eDop@%EYxJ>D{i6S9()0e$Y(rL56hhkGZ^vT$HbbvU zh*);SIk3OtJ7J6+do3)jd;7C0$%CH++Ci@GJbio|fq8Zc0j50Ma$?d+C!u^NZ2>EC%>XByMB;2l5{$8LxBNY(aI_&u1kgQ+lklTTozxqxQ_is_|K_t;r(arb*_?WL(|J z6oRL^UH;0SkLuw*M-T?F`kpiXnreRQE_urGNm~yz0tGBPLX2-*Pdt4PIs?8cVUc7o z>DYZzvf=BOq0chZULNr|GtCSex=-sNgX)+ERvT#zMG_kpeJ5r6tY%@KbQUv(5lr%%T)!SI5Tzv<-?jkl_<$dHJyCLI3ae zp(|ELe0=L&MGt{T2Aew(IxuiIXTscj>fZdDN2G7$!ip2I?Rnw)KMKU4pZ6+ar3^^JVHQV2g{$Sw?2K_z`MEa?wFN@e011dK=ZEgHN@A-t6-^+osrlGA})Yn(y zTnK?Iapv(IKYzijJij!ZZXz$Q*z=J~Y7Rc4?``h*PT`37UhFvL6W8b9M-LyCW3Zn$ zLTzMk@`&T6Y1VxrSL1yv9CBRq%^3J$=71c-l~i4;!%>3lo6ektEIr3zN3sQ3%Fa&J zWplbq)1^PVlVs(me4kPA6y>p9IINy)SEaG4<8G_H3vB$!&Yj7+@rxy+@s#t|Zz$|c zC^)pP*$w$t&c)$bOs*J-r4@L@rh>JGz%n{*jH+5P*b8`gEM2%?AK+|*=~N($L6u|>}<3r;X2aM=J$kR z%o0Vu-yIwiHl6A38;4$L|Du3k5sZN z;zlsB6m%Tv@;MXR+ShWeH;3d;&m@`Cr8J45&^2QE#=ms1VQJzduHlU@JF0ySM}<8^ zPoE5=RKl{=9)`#v$k`91kBDX$XP3NbguM+O%HAY0D78BhVC5`|VNgnOs+))LNg>Kk z+QklkM?+D0dBZ3^eK@9aUV1S$LjZ020p7z{Rm)zK63d8``E*SeO#L02j2}0yOOuVg z-$)LW)GXqE42p_g-rqT?#*5;;T=)2_PMlQv@WOgr7|TlaY_j79+1nn6hY_W(#6#z$9(XqNlTk}}VbiEuKq?U{9jIjR@C;qtL z#CQLR=GM)n5m74>6H0l+^T@?z)(iUv2PRZZ)1~}Opb~PeLaShEb5Q)i`_xsG!U&%F zGr85YujS`yp{i_T8j)@Jf>Uf>e*QBA{PYbM{a(%Q*NzB=%Ic>fM06WHD^I6*{-Qsb z_Fn3{$->p-ZvRnA?}<9v>6Np#Te83-lEx5YaOwgV7@hAvqkkXCNQ-dSOO|Pg z!hZLcN8TBn{H!-pcTq}tOdhSSab(lclAg$Obi3g&wCRI};(Nx`lhxfU)xdUWp=mbE z%q%uQDElGHY$D(t5){o3ybWVxwc%qnQd6dpYhw!sAXr-&Yu>wAsNV2IkR;YtSvCKi z-}A%Q2`2Z@_L(p`8tcRSvl*1Pn72n_3Bg}7RZ>-msceshS4M?d#@W z3#!a6uhPQJo(4QhVou%5C$2hq8A4A2o!MM!B&+hA-MXY!pn$Mt9%9){(k6924F9qq z)*7GttM3e%n4ha|RmI|jC!Hma&l{hEpP%WKfyT^+I8;NQ`lF4# zNHI@9-q0J_%HN*T5*U?@E>XjjQK}QmG7B zmkemPRh>W>{T%&DjdPaYon`%?26siKzt)MQ1_VJcF;wa-=I!8(e^&<1Cz~F6R0w)q z-}&4;QO-Y)pvUEmq~SGfd3+(u*d8(~m{CVMXJ$dSLURK_dW>f|@19ZY6H`=rf5P|8&%xi zN_c8>?-=*4*bSNtJykVYZsbz}M9qJO5!YZ^yMWZ&tumT0W z6N06bE1e0s_M%q0=aTj4D z#4P*}Pr<%Y*3jV2-~2k$RMn7an8 z4q01U$Hr^tAf8O6(P&Y5cE408FkXGa7@YrfV{bc;fU0Ty`tr_D)~`a3{qNmKWTbqO zFTIDtv+isd8>UJoaof3d;76n&*gOltM4(dDe>M-H{yg@XOd~SDBm_gJ(?a!;1G05c zBkaETwdp;r^;h%mNht(FSR>UU6G#xgEqI14#G~SefB2bL2R8&-DKFtCMhUnJ(n3z*U=5|x$hsIys^>&jb{jZel>{24qcH9tD z0$EG(-}&`xO1Mv+W{VvWg}~zuotkNL9YyNizaLQc4w9VpkP?>0zLy+GOZA2paURNf z?a@VbQ;e;F@;D3B!DPAD=oOOVdW3d-A)DqWMIH&kRo#(VOP^_woreANNAZviQC7Rt z?BB*Y?^lr373J(V86mO&5%E*gWLzu>!`G6U#2V^)S&{JtgJ?pd<%9+Db699?`TKI0 z*vaaH!QpB~$Q|$zgVyK1!aPO5{tW}m!U#)&{uz=Xw{Eh4;%v8~#_&uGFzhYAYnJ*s zZsvPovBa~R?z7tmJ3pKApa#oR<_>L*cdA#hcqxr+fS$d!VL=^egVre$VUzPr|~27FKO&AU0PPZI)EUwrw5uePvCGUqCg zQ&y%3=6N-ITs1BhYPnJO5S8a`P z!6C>zvefPP(vf&ng(%xx-MHc<0eS6HJ*tr|KfmSW%oBoe=LXrbjsGwtXc~ZGfN|=z zyHSUPRTEoc6AM@!p)CvB_&-#+xU`fKpXd!6dFi&4gKwxiTcq*)IadiaSiSxxU6HAj zqt-M7fX9ccJR9g4e0E*5qSxL6WyKLh1mYRb#V zkT)!nP4lf&rbw31GB4G&cMDAQw2x|QW6iwO)TJn%rgDzm2ZtqkFm97C1Q)P}f&MZw z9AXDMNdZrc4Gi*uM`45n1u-7J95jbHV-*|9Ikc^<7O@^nr^$kB;ote=eOhjc6+?SumLmDqHN>z&$!{B z32(F4N&jx>9D#+ zQ&ZDwQ$KCFt*Rdv6QAn0$?iuf&kYX&PEzb?8(aU+;|ByUlU+kh zmI$U|D}2DRp{dT?5MvTMNsbw-@YA?3J#bEpWh4`hWm0#%!_|9JuhxmF-1r$B$-MkuFqkeFfu_+0(NRazyiu-)cIVZ6nvV%t z#!|=KSL@7YvQh%7Hcdio*2y&RQU)KBhuNbyAoP?NL&DIy(0%!VAX*PBJ(flB#xRF6GFoq`_@L&`^#1RWTXdCx=fwP{8gKi zhHIlG#&1n5;S-xOXG6M=mxclZ<0So}w7dvEkznU~f;1hSz2bk~=LSHDlbNwqmdxuA^r80m(Ybba#qFSekx1bo(CK zQwJg5ZZ)R0**bW|bGig>#t?FVo;ryro4cF)N_U@{2q+=Hj55o(QZ%NA*w?l4 zq-=X%>;qT?Ad+ZvQD`fL!hx7m_M)OqH$mj#?9a;Nqp=ilzf zEE%(=vSBti!sIm#4exSq6gn)vqOn65y|_0Xtr=$;^;bCVU zcCnyS=J0z>)`xWY2swQtMBU-uyaODF5itZJm0xQnDvX>D;R{0-o%*BzxbdEc$ZX{0 z2oUq>891mYss@D4VLb7bm|eUT1gpiKnwgQI3hjjqTn1zN`Q{K1 z`fYZXxMeuwQ<{wAR@ObLDWeb;Lw;w2;h`XVyVJ%Km)4N%GQN5%o5qD9(#1NVCHYVR zh3dG&>Sts=w5Rw|8?lAQ7FOR{x;VRDE+yJrbPP`Y_o~y%lL4j_m7hOUe23l5`RUop@S$i`uVkM73RZPHy5*GE}aM; zcM#=Xzw_8x@EM+(ZG6a+Itc-(y;@)1CzCMsqS(BO75piYzTvac;~?!C2AGr4>~(7H zvbEux)a38}LruVMIhWnQ@!zH5eF8&hs;Qkj%3xT3&-9ej`mff7EA!&|b>gNe%?tfm z1tQ2MJ8VEzn7(&SqIXKXG3PX-vSuA9YwWVHHoSt2pX%N0_n_Yc6jeNfQk8qgd`DfT#&TF>WWO6-m6D41 zN$|uv{@O&jk%-$`{Gj!A%*!Dl#t|`kKPu$@woo9T{D}Da$OLQT5-azt`D8$ zbgqDqe;O^}+4y9ydjXa|F01(7Jv>`^Aulu2V084ZyzAJ*(;6-3s?53eaOVXTgxi4J z3EgmB@X`^YaNyO-x55JA-sn>0v06>xW8eOXoTSWmm2!ccv31@`V}~)bdUU&6WPEPR zr8^r-Zc{0}E_@-|krL73dgjSqJM>mRE?c$nvT$+f#a_79MN``Zf=|O=bRJXjmCUX4 z{CxSzJmG&u!ML$ z3+`lzZ@FERXi-OPoKGTVULt*g>C>XlV&`r*1_+AT`&i{?gQ|rM(@kj9NlsRZXCqy( zb(y<{nuYd==gJTSYdfqnN{#+R46|C zsKG?kb$y4Ev7RbjXY~ANtt1qx6lNo8jGOgm>?F&2>CfkQeQ%Em+bgN5^zmC@13d(_ zJs6T)q7Q_YYYQ0qf~kRE?2SaR@)y>jzDK=QK2S}Hk4NKZT(MWWsFww|y3C3U&F)3g zu_>j{j#}(KumF{(K(Gf?4tkr8dGW*1YyZB_-#(m34d*cVH%&%{y8KQo`q@IVGDJP0 zh;=7MD4h-UtW4?5ccogIutqI$Yg#DYdn5E_gS(AyL7Xk?l3Wn`c#YavZMUG?fmzDo z+aEt}7r^Ra)ciikiJBkIg$22}gS(u2mYrjEtdB+aY>H4QwWm*ut!UVo&ReAnpdxxbQ8&^1=e|~L7Y|KFhk00sIrq-LQ-ZT7 zw7*8~FCIdqX2H@T5RszXx;{mo*P(4E7^~=(Q^#ob27czW^vf^)Wy$-6U%?9Tr^@Iv zjaIHNjrya%9v`+En8fFFJF#La* ze0}=V7qgf0{_oi{`be%eUpGdN%IFhx6~+zWa0vPtlOm}(btmaj=gHKmgvXSe3z>S2 zhCJQyygr6Y3@ULn9b>$XC6qbbl{pb9o_@YEEH4{zcVWI`>9?JB;aCRyxGMzl>(@){ z^pB*N#O0#1kup47!*2*9sSTp#SUHB+} zB~tTwLc~Z$7Stt_h^vMK0S97{B!~#@{9L|eBBL`X<%lF+L63~|EqeL7)#7g=%{`mQ$oXjr5j%cC{q*poSj+@PEV5Cy0#o1>S$@*x=NSQ8D*-WH|GDr zBU=oD&>a1WOreq(cG^Cf>VNl+ALjn_=(UZ7h+iM&-!hr}@Q><+mH2vhok-EIk`2N8 zm8$t*1(b3=Ui$6)@0{THcQwU%02|pOs7bd*5}W35ulX;zR;7TPmqo?fVZZ?iTWUV9 z@#TI9Ju9taxs06{x-G`b45NsYTTTwK)nS_>2+B8i^RBM>^-_EKD<3QlMcp9(?%igI zUc=ef*x1;@JK0+d5M<$&mV*QZ9Ub$PAd&XAMBMjyv^P-;|Kw-?pq-PX2r zD^%b}{FKmta;fm$o#!^c-t-1 zu0<*&=lK;NJ%wnL@Y#iF&*TZDnzxI>Uss#I33efO;8?FA$X_~M+hPD5SZ<)k^XHbs zSuY3)PdkO8ZzrDBIgZ=`fp_d64XS84-(D@3<)phx3*odKb{^htEezxylNZHivMngz zxJM`GUX}vANECM2O3j$TF|B;kL_J z(8*3;MBldwzXJB2lX#@n#5bbgbte^tydkOu`&3S3#ROL_qpt2!z)VSrycx0Z-r+~e zu;rE7V7s-~v9SU!#--yTyC<=!LuE9^9x~9dl@0WX=OBR}zMPY={%|gvnWWS7{=gEh zQr#=>Jev!&{~-KtIH4H$DB(yi@=-h+d_8|{i`6?l;E_ncBRHbkBI}%VEQ4zl`?_e{ zVEG)Eug?ahs`MPlk8le?WExAA;x-cy(e}UDejWtV)-*ujy9L@-xC7Q5s;m2XhS`aX zdgd&OOyi@ZdBdfkX zBDm_$EdH|^@1^~4c^fLv(yO6LBD`RET{_kyoW?wP{xajoU+4_(olEiIZkDH%A;#-V zE(gN8>Qk$H1tt8J)uP!_iM>JrIY3Yy{WZe~0S86YWj=B!(niX5*PUlWe31s+z~6yx z{OM^#s#Wq?yM}`tLV1DwQPH=0=nFl0Kj0tWUz?udoxt*0Sy5#+d*8p<0HH+BrKslk zL2cFk%okWD!%|pm5?-?uzC}ge1wnmZ9i$&FI)korW6O!uK$zzET1no1S#@Shi&9yn z^BN3#_wenJ`Q6Im_JU38rTJgGQ@>m2t)Qy+mz$^=^a(0|inQ!XS` z@i;k)R8@|#d}7#%Us{-z*x||H(?D5lvqXJ;{1vq7=-N9f{<6T7 z+KWk0{@Ww~=mBAU%zN&l$97bnfza)wN%{Wul+|$x>~OM*bNQiZ|L%LvLbe_s=e}9R zFJ&jeHpB9hpElNXorTl;n0PD{wV*EH{C1k`3}4Htc$nRTjT6l?|$gp$ya-bb$J=8QeP6uD zZX2Nbba^&VcN6($9xyT;il~78OHx|*d@K8g_H89%6rnj>e^|*P!X{;2irl<=T7D8u zqxY&`G@OPfzY+af_i^XhKELOHD*IQf)Wba$;IiYo7YN^^ukK1?{5IV5Zc62H9b;Pp zDiN6Z-Z`aCMStS_yoZ!X2 zxkgFj(9&ab5IDPgAoZ@4fN@dw&Q_UMH@%cLkc?@1RBmh}(B1waRPwUiaC9fULp;s5 zYR9A2!<(z`3|!)Fg$r`4@=F;pVvXM|EHsFdHMiVgFcD9 z4zuSJ3tjx{odF>wh`JN33cg+dw70ave{a?~$)kY~z|MWpW{%#eytZ4xvdmbO5Df=a{gi!Mcku>kz7eiRo*$}zw(sIC^t^x*PEYtgDb-uz0lS2SO^;_BjvsGV11ri6 z%F6)?P9|V&{r^@S5dFE?la+E?i%qk4s8Xk=xH(-UCB$~VvHzV)#%@%swj!{m3ikK8 zdgFW8;Dy({xsG8o3kUeH`7i5Nuvq)k1Gl+Xf75g4Y}dKd)-gbpY+V$+>?KoqI?;vr9F%vK>^aiJg&%WxqHjYmw+t-;RgxyTt|gN>)g-4#mYmklKIl5ho2nw1s2RW_evhSht};2ERT@k&YPyMFLgirhsUQ zq$#Y@RwRVWmgqc2FEz2bVCUi&v*WYbnr8d|%EW~rhAoDsS+vgqBz$hdii+#8J_7ju z+!KO`LlxUmh7{4eIsz;LvU`!ru?6IzTKXZq(_UxRj*fQ)A z_wn0apx>6j)ag0g8zYNNL~FWOjzE+{jCIxv_}`l-ah3GbUtjxjl!O;Bb0E-R=Zc}> zQHT3_Cn#NW#dS$}xb zU2LR9-gBO-dw+Yq({sKh^~*vV)x_bR*f>0=M)OS6!*ztO_SsWou)QeXo^v@frlm?; zSrJ!cdIX?aM^glBodyeCN{A#F9B9uy5 z>x>S;u1?y))HO6oNhzDP6k6gukhCWE+qzsmEW!Ql^%7=IC7&D1O?VQYxy_~OK>GBP z!lZ_PS)hY0TU#TsHpr8Z@Ec5u)-FEh$!s9k;&JuK1gr;ui@!l-_SZ5+`^xG@i_TEeV{pZ1zGx)1`drHyg3Ga5_4v~?a?`^;a^Wr}> zNWK0)FSc7eeg|B^p1$0qthT@{+MkP8ebTIKlcJfH1Ckz>PKB({FD(0qbH*|EA)zX7 ziAOn95Z*J)W$3z{9n!gZiq?hgwl#ZIMa69i@^cFQZXwXw0WV%cNUu^5Dr(5Fu= zdoCz8$pygZ`Wre(tvf<_dCQg(T@IQbYN==r4pKLF_4v!?MiQwl-W%1udmf*TkxQwu zu$+L&`qhTltkAW5+ht2r;}d6N>Z5`w5+}N%{?X+_kZ(YAk4_Z6UZ?~jn{dihn%&6J z2{c`{HFf>j?XIyO#DB&|a^d{w*Km6-y!;@sVGemKmk}l3P(y5tt0n{DuY2Y{s4dfe zHsC`LAeIw9h1-qr5r!K#-f93&!1 zQXFzPTk1SKd>dpw6xE#ugev^-o6Mj3Dh0Lzg4a^@dG+!IyzUV`7lSg{?f~9!%ri3Xz9;jOnW3y33L|A7wD3@vA&;&w{P2myagWBZlg3`DKavs zy2&>y?3mqQ{n>^Gjqrq1^ViC$69XVZEw*M)4J?)bDx>U_eW%edr0uhM)|ER4G&T*b+ZEJ3VUETaI zSA+hwIRclrPD~2Lj={Oy4$u}eWnt-bykw$kTAxLb5|h>SEa!>X-AgP7hbF^ z5^TGJ4A^BH>9Kk{qXL=ih%)OnXf7UBqJLhdCW~XR=YH)`p`-o4qow0*LJfim<{j(7 zwr_&1q(y$OCdZ)A3}Qi$DaXh-*VfggTGBtciq@?kEx_}bqoOe?6m@&8j=$4SamH&{ zAsDu^ABpVN;$>G>h^o^WWWiP$%n7@SYA-F}#en)$>qh{0u`DcFH8$z-!CNE&1Pv+q zv*B>y9OnW9^RfFc%{5NWL2WYRf`Ndb_*02A-tzY4W$s3qbZ2GRmY8;ZV@l>y7_G4G z&^Ndlp`wE$2CuYlrzeAo3`+cgfu=QY5Yfqq7N|DRD;@gk{+&HV%gCujsI%V&W-(MC$!zB?%jUJihJie6 zO3lpJ2ft?o?I)OwU?w3xF}n{O?aPfAgV%U!5Qw^yrM`cWTJLw_qix`%g5R=cD(2>njMV4|72kWKfLSb z)#|dpOu#v`tJ3tL>NH%3t!7IBo6x!HXB5RS+CRVXzYkvisgkXQIK`PM1h%XdXq^04 zoX6V|5%Wy^2#c6?uGE^H+klZOIg1^Z&3pXkjX0R|1DuO%Ti{%gPznM=ThcL?$NwIvPmuf9 zG2c=ERx!IfK6kPHEosgT5RSBtU4i4hERh(7Dp9f~v-!)7NiPYV0a9gU%Tu+reSf$| zT)#%!Gs{|vlR58sX-P?sP&Uu1dVFQy*N)`-kS%vn zVT7}~NZnYT4?#p5aC7!512YnOlnIFeFII+GXLjft29EHuT8gXW?v#{%*1C|V{MX-s z!uvbsR}R1e`yG&Ty9jW;m|0@!lciQ|km*I1aOgiY@>Se)+3x5vpY5Z+nH@QK1I@Mz zsMI5t@EHa;uh-{xb!5c6vV)CqnPkp#m1Iz2H21H|Nw@FVM&;eHRQ3oebBs1##-zuu zyb)5a;xnu*uM)IIUJTY^DiCgQXjFA{&JzaDLjqgNIBp*GB$&R2MxK~gta4zu??4F>nnMwM3T6ESa zSVJ~0i zCyH&q1QF91#WRcqBdJsW?ULv;)4ntKw9GA^k*o*A$Ez6 z3lVdf3d2m6 z-_oDIBuVY_|6ezt<8=ZI!OJ-&pzXM{@Q8}1F<7m{S7YZ9k%K+cQLUz0wzi5dI^&Jj z)Vg}3>4(=b@t+$NBtW?Sn5osc{POzI(!$Ct){}7ft552OBZh=)()4Xx+nHD(>>Sip z^D#c0e6rUx|DcN68)g7XpZxRquY-e|I6e4m+IGEhcGV0JkQL$i>Zejb!A1m)M1fga zLH}B%AsYl_h7Of-azr)H5c6Q%8^{uQ9m^QbW7$&98j#Yc2iSdOSFhJy)DqmY9$2hB z0M?!Lr%p_CNGPaa9w zgRl2N=lyrvCVZ2)k>bGbBX1~e$!9J*IJzYxRrYo^Oy}bzgcA}v;D^I0OYnP~j<%I1 z1hzBmn@ulkiXp4`r4O&sOMddvhM>@Hr;i+w84UF4gt3!F&hW0zd%ne6I2rKfOoS0z5{nOZgWkI)fFw>5&+A_Ij1U={lc?Te)jg zqJ%~8dEVXrzK+C|@UX8)(3IEDJ>20P4Y`??wBsek`p*L3COD2J?0qGqbp72K^)YwkX64j~TH z@U#wP$VH)^6B4F=N*u9_M%;*;Zp`_~FSVpJm0=GCPaukv1qIQzu?ZrBmg);Li74DxWX)84?y3`4`i| zjR+4gdLVmv$khuLSExC4Tg$2X;oR!|w=s!{`9HQP7$Ib_gIaF6dGFruxcPD`^;6}c zP{kFpOpJ2YH$_?aIbQ4y4o_-(D1_u@=`$!)^TG+4T_J{T-tB9A2{qctF*p#Gu{24Qfp9938t%iKrt>TwhK;O8nFu?i99SAJ~_EY9!C zllVmq&h`st{So!|3I`};@#3Q{vVWfJfTNrK9P&zbghrasLBn$K#~t;v2IFRHg_=-| zXUa$ZpE)*B$uTeNhRc@B&8jk_1`?rrD^Y_AJ1xtMRQ$r?Uj_hkinq8Z&_^+j*Q}L) zlT}|#fZozpO4-V&>*k+n|8`#z_$~A!HlD#TaSzlq3ftRz>aSsE7f3~&G*-a!NYKG9 zl9G1HUP)Uhg-AyBC{Xc=Hh#vhm&mW6POhh3Xj24ac!X;S<$}*q1kx$+tcR6bDMrc_ zx9i9>`B?^$XM6_2(GX>~5XkOU2=Ye|ZrkdRigyDX5y`9-V2fq^T7``+RS zSuBu5ahKN&t-OSGqiD9A-DhgW9OUs31mbouo2Y1hv$JA&WE2Bb;0XRT%jKO!V|j2v zjiE!22QAN&FJQq?stSz;9BnViy?eJ7CoU@B4od2j)FpykgMS|Xr~bgT!W)uD`;l=U z`gyuR2>2r-&PSs;Vqr<(Y!kt6zsrer8Slt-lC5HZ91-2?wJ@4(d2$n?ewyIy32p&r zgzzW9_Bax!O5eq5%jt^`8hAxxC3g2XHEkS_vRN#-nbH(03o8j0XBi8I-fYtY_@B8J zow(R(GN`CTQTeFiI&XHE*Ug5jlmwoglVW{cm;jUZcsCt!8-p_6Cq6pgMs8Hg!%JHF ze+vEEX&jtqNzR{`ac}}#YpQrOwWmI_zs#Yb&j^Ry6JkBxFVdUI1Iu&gK)l4d-!i3wT#ql}eZ zd8G^5`eCORY3<{btH+--D>8!am^#>w2b=9N1jQQ|PZ3se5NNP*W|ya&N@Pg;rLgd6 zz1KXSjbA6~h|k`(soanFo0-se%mA-|dsR)vPzPinjaW!@^tqGRy5sGp)*n=~rb3Q+wY7)lF6qZZnu$o{#vHuNfCXRY?>)ns+P?MC*c*y&MVf#LNRuK}I*Ndy_g+OnI?_AXV2c!u6zL!( zv=AwwBs4`pKtNhTFCqj85L!TL{xkU7y3e`i-uv-B&wbANA_TM6T(gWZ-gh)5`;C;s zwC$xOLFsj6^M(O(rLdZ;iEwG9-S0>f$bsOvx=-V z7mb~ghiXF~!^4?M?M-#7Zd9@E0_v8V_f%ef)q8neAB_B=T3ZeEWkXuPJC-|PPXNS}UqCNd3aklxl2eXrG`PGlV*J2j$9n$7Ax#sWMf8K}8X|HD z%U&nT!DT6nhba+%#FCo-{r&3)-}f+;hjS@Skhj|dpFGKXgeioV9Y(w2MdyCbHIy`| zpuJx9dnnD$eEvJMge0Exz}fQm2rM661D5AABhv+ul#8?}CIBn$sx*Nz-5%pLAo;$qY&D(Wa_a=H21#kH0qz zJmtaqrZ*Qy=x=X)X~cQv_j_E7oAf>!S=(Cb_j>>huK`60IbOExX5aRs7&q?u)+rkM z;XX=>R`+f==Q_qrGP?|aq-Du0P%T0E@5l44)vbrDiOk5JTqI9`p#1OqK%Sih0EKSB z<=xdNmJky9)Wz(_PrPUpog0FyIx-(IKH*$1t2MZpAZA-0D?XFNU+KSelHNdAJwOPv zo`n2EoLIV1CM45xNO%rwlKpOW!n)*EPf=n|EZ3cIZsf^(O+#6i?9UL}6AwQ*y_&Nq zdALO%G2hMJ!lp2fUxOWPHkUTcT;wPS@C3-j8M+6W1syI{pS1B z5HIr9yyu7sod88`&bo>ZYD4`d2A(QN*DcIfzudMMjyTdh*2&ddbE-tTPV}GKK|%?X zJgTdGRId`Wdv`6Eo@a`RE?~(8!AeTWpgFf)l1gl^XLzwF<^Gae{prqHL}00b>)yXIaQ~q=7B~y2qO6BqT8kHt@B#5 zyx9HR8a`aem|K+w{>V|@1fNYNSbKkArJ;vHLf#ot*jQ(-%B6q4207n$@V&=90mUXH zRIU;{8RK*Ao6XK9)MDx6m4Mky^ZJe;G3ws$CEbUtLt44?pk5xx@jv`$1L+A+U{n{_ z74<5+xWo!}9`fO`0Oa8_zp3TtH!Y`O^JpOXqLe7HQ>MC~)#I-t8}(eSJz-bS_x^~p z%r^0FpjgB@b)zskC|bid{p+;<0G1;Dyi++O0YXV`8v9&rwwft^QenkER9G1Fx_Ak`l-DjddVmIfc|BRH1 z3Iuiw+>7*zUf2*f!nt)?3QEbSFL5S}Xca$6Yx`EZ6S1V}oh`me^Mm7WsRwv7$2U%D>f075=CXMk4u6~I z?zZ-$ zjZXh!01_A#W0-T+%ip+5Fqv(S@dd1(7%!^`A&*FayijjKpy@NT1ec_wo6{m|Q1I}( z`xxfbx7q0-o?{F{#^b5CRAp-y5xXGj>|HbkKixSid?(bH!<=1taI`fJ36 zo)+(|hSX=RwmEd929@>nD$Or$inbC+$NIj$lYy!%Vai6UnIzNT)H2(5&oXNeCcIl+ zd}iwu)e~VCPuQULj0Pukx7)iQOTitMAivAlg~|sdNOhW7ZtjADc&E z?Gok(m0D(-bNcq&*_#+}RmaSjR zw|u%cRo@H>tu_xP1@TbPFf-@Q&T>MvJ<1GBWV`geJ2Nu~JE}Ucayzn^zjb2^@V>-_u^GK;#nAOqLaKe`~p7?eZ z<=28O>+Og#jsD}$Gl$lutNo?dzn-Ge6DGB%{NBfqbs6BcZ8$G9!dyti5Vc&^{HFb1 z=~`jitcLV_iq(%9hQIJ?^j!C9|e>QeQDa7g@OMVT-j>5o=zc zWMPxE4L7IrLa+FrO*BhMfPu7i@WfjEY`lJv<002ID?>O*DH^ZLtAg!F7h|3Mxoq`V zS%bviZlI?JgrbI~np3$76w3OvdFsJ?b|{K;7W>XVi5soU4NOZjwBPn;m*DU!zJ)qq z%P_V;U%mvhw$LjwDr0A1?wtPhkH1B3`~eUlp&W$_|dxphx8L^INgrdU}YSUI`@i8IQdD){} z4`})PX#Uau@E`g4?1JM*2;>qo^9_TCAfPHa3$E9zPJ1K0i=$VV9oh?5f6bPdo5^e9 zbFAN-lN;OgT@b3R{mBhLRFZGBeb%m7k%Pm$=g(xFMXYb_81p!s6?oT~cDJj{eeIv^ zuWcgdUL=T?@6={71$e9I^*^ybLtMmhMXE)Ab6rwdyqhylHXR}v|B);>T1uuowCDc; zg^uw-(atlv)mgjksUTDTUV6|Q)0Y1mU3r+;m~ySA6LE?XZ&TC!qfW-JsJ`@H^NrDFWoBar*BnU2&<^!9NZ z;=Zw(T8IC17ZE6oFT!exxv~r8#D-hEXuC8Z+bQ^MezOaqV{QEw^bv+TOK!Bb$4#!&fJ%4b8F5|T9_Bcr&ouz z>-Do86?09xLcF!oFCX>38xYv+w%7RnO;Spy_P=a>ONs}=(x}R^_|vD9p^)Xg`FV!J zPZjCyx%}M*?+!YNLSe&9IV{pBZDI{Z0nc}-UHB{ym57v#zR$Q9#wF!7jb}`IYPOV` z8YK^Fn2Ap2@onQ|T77+e3RFLJs_|jvFu=)SGhMX`0h>&3;3fIfY5PO(=qi~5C(^td zHGwxxB3RJkT)4bZVm?D6Y+yS5WtZKTNw?W_^2#m81%vwFigrA;I1Nn?F_@=z&%8yX z48M+#d@3d2SjGHK#593B*(>ush}%%J*11kYqdukjd&^zDH1M^ZaP|=7TUbkP-bl^e z>QTO&oZK&So=2~2`Ue?rHit`iRB&eSfN1Dw)t>pdS4TdngHd+JcX@+kVet$s({#`3 ziOYa4;(c^3;h9nl(k%Bayz@~@@v}plKnsJ-dMU_pjMu&T`m@gQ-8uEeK_5eNB)_Fp z8%=&c4=oQbX}%qf!mrlks;6a~h#w15!<~fUyP&c=C7#zS*U%Qbrp9}F`$_+z2(E?c zpM->ZD<~??=0c}p#U(^IslYB?|E|IqYvDF_niIUlix*pr)M@mx!>O1yrm|O*$ifd3sp@{ zawJ#S8+6_|FgtD-WXlTcjZbvPYxr#Zw0q98n@)Bofq1RGZLywWroCINzb6(Ve;(L2 z5cFb>!MNUI`|7^Z*s+Kcv__*D?%pig=E0ei<%W4mY!;1?EerepQucqaD9Fwp^%{it zcqzI%4RH1fuY>a41h|H5(a)-P89O4rQtk?h4uDlDVqY=0*nHx?cE*NI2DGJA1uafX z=4HtbD!qx)H=z-KpTu2sOyIipxQZ?RkMXfImELoHZ6&-XI;9v_zsoVVQ5_`y?*U(; z)VUaS-~R-H$K&n;`gmk}KPl&s%G9W+V}U-hrfXVjG3zZ|TXB{`@5f!GuncmQY#bbUv)cKpy_&#qwz6WXkNsS6N>xu^&&SZf z%!C7!1(P%nMiy%8m?TUC#$3GlDm z<^6SksrsU?x{f;XP9oJLL!)x3=gG4c3Dcz&WgxkS zDy-AJZCP}++TDFdC*5wZcaeX-BmJ=;O^LrYpOzM%O?Fh@AcZqT;L5s?xJ_K3!L3^^ zB4Pn3p`DDnkqQH9q-oJ(Yavh^%Wu1vhu9{377^l@MWJp_i9Ku+RWa&vF!>2G07#F+rMuS9CwJ%JeX$R-CHZR5xW62S#~Ni5{r^{n*u&9DBcz+OO$ z$@C?bDF}8n$TH@$zAg8keo#G%2w|o~F7u`@>gwo}iEOmaI0M?eeD}$*wl*sB;T2Gl z-wd|?bI3-*Z$8p{L4wYWKe?lr$Xl(V123;%pP13`apoxL3Y$(*y?e0(}kqZh$D7Ra_qH{ew_7D*3E%mT9|?C=m)M_%6Z zcODHFq)HzfST1oA4SDvdeKPuk02~nF+|{OUY6a{4JnZs{>+TF<>h-~x&9~M9N{5+% z^9YTvJJM1T0#2FMmYD0$OjB#ANR|BiHqz1>{(o2iiZ2P-#cpsvOSA&9wz9>s6O$xI@icb~xCA zMItjsG?-qKK-%sb7yyRL@!Et@HRq{oqnI3P{N@D zfiSq}y|-L4?T@eJZIkp+2j-eSU7Ky0WAi%W5i~!r%;zTKHXWwonLyk_`CW__2AJ5etYt7SvyOZX^gvSVmtt$!)}hrwb&$TEdOMMSZupJ}NcTAeTmm2_<4uPxFouAu z64mUnb{gUG2%u_`Ki$6=bSjE=1};zqg*a5IxW6tO<@`)JOOYLAVqG>U47j+aoat3z zQ6EP>Z;Y8t`nB|~56uTCxtl(&x5~iaH(W^+$1A%{mp37A4b-#78FH?01q#- zMixbw^W112wRcu?u0=61UL3Vb9bSYf+jx1;zufkZL8Zb3-}f4)chR07asUM*VL&1L za~-D?{Rb6wV^dNx-mByQ1kur9{^Et#u15ZV=#%`mg86fV0!ISHE(zJHg~8mn0&$n4 zfq&I39;DIg&)GBK(57$vACh3+F*$4;rXU&YpIiSrKafy92FV*{pc+*ITdIUv2J~W! zx8eVw=2Fx#`+Kq$T4rXAB3=CdajVL`b!8a0LX5glT&1``{(o)_neew^nWe}%s>{}; z#wL70rG6$D5s-06F%-bjx=x9&=$;#SnlbcCv5AO?Y;ppFYjV;Gn1{-+I&ptv5fQID zA|RV3*M4JG=J%}|KyfrMKXSAR3e~Z1M73eMCMHO`bPLxP(Mid*+wQ~Wd$QyWj?_+R z9;mTzm(-xseuxW2!>Ff7u|fsabTFmV^X-cP2uF+=h0T;sn*u=9WXWna5s#a5vU=my z1xU8_WU?=mQOuRNN&?_m*GSJg$P9GEd>GY_#cukJ{T_{+Ye`}`#jlS83Q5mx<0&|J zA~$8!txL5*+}*HM5YV*I%{1-$$^H4IRc6*NlgV@9x zASs|R99%Y(VN2*W(T?v~&h;O~)xJrCT;kO}l^Pa)A5 zl=dsXgz$BH(|?xRZu8poG%fh3x7rRUC$P-`U8@R6K{hCdMrt=BqbS%YwY&2>+jG6t zmBTOdKTf^j5!41`7rak5&4)`nZQunnC~Ju0Jni>Z}#Ioe~sTmwY)lb!k@xTC8Pok zqd>_^0USVGy{&yik8fp1Qb9itv23Fw(8BoF3Dlix2nQHZOy}zP%am>Te zG;)8EBgT;OCvu%6Vi6{qT0qrE!C%?&OIrU8egQ}xu&nd@`H*iHDMV_6mo@o`HFo1E zX2ug;lWawIpx4-==`NSErLlF?c$IoZUF29Bu2t99M_<4G=}7Z0$VE=nH8XRJ!q04B z6rnx;2^Q)gp8tJ+ISXsRl|tM`^YZdaD}TJo#PP1BlZ zTsoRuD%j4J zDGXTC=PF4AA|Fv%VQu|hvd9Btrk!8ZsV=Q$K`g1z(N^>{Cai58zH8zf7pM2v-;_!J zb!}6S5*Xrrb#+ZZoC5N1#uoYMlf*t;*(td}*GA7P+^e6GL4m2An%+cni=e!W>Hwpg z1NSl|xGT(qaGN3Ncv#A}sp@}j34uIH?+{i4&eT7{rTne-;{itc^WPw$V-&CQzb?5i z2dePTCs!YWAfP|LLLlb_5A^V#k51B4CgNY$bBd?z-`Cyalvn)Mbqyp~|La#s=xb1{ zrBc$goIj97Y8^5Kd^8(K|zdl!Xnwq*7 zs3SA~y%qo3whl@7(=u9sPu_#FnTWstUJgzcgIGQSJyNxSS;c1mbJ-U#&0dBk`sVon z;^$Ql7M_|?;$demVX-?6Iq8)U(H@%~qORAE0@f+^U6C*VGECUDuSX%XJh}GsBU`V) zBRHT8B7$1hr9IQ05)#reb~hjmmJfu~daukvA+t=xJ+H!ZK_1sFzq

L_SpcpE2U zxpo-xym*sbp%-2!&fY70xS93(C)Dakg|;?T$ITiFmA1p;be5ssV_$W21(6seY5{5Z zwl?z0wV73SIAAvU+)7S&>fc=(Tng^E2uO0zi zY^@T=k`f~@qhANlz_I}_II8uIo4meU6u`qEa!d!X>V2zW z!Pz`_DGGAKA4Q(f8kKT>08h<v_K!|-|;KrOzXkSU8<1Kj}h!^q31QtwQ^je z(_n1u>{PgMS(G;L69W4OpyEnE4+gETBy6|#Iv)2gfP|iZ47}8&xtZ!RS8S@!*HUAL z5R@}ghU`!Y(3XYlPgPmphxQHC=}u!epcEf5T-G=X=pc%9h_tf+zUpIeGidr2!vv#L$u7!9Uay|lEyYdDwXol~~dM)Veh+D?6OBbgq z#s(6yO>9-Q6^HIadvM_zp1#2BW@?r7P*+gcw|h_D%Q3fvu{B{51BwpxJys!4r86hA z)7T$u=nVc;S3ffbu^ZBsiJ0flt&~--@S)cJSz@_M=MP6Scg3;gutjMCk61%VGF+{^ z!o!Jc5ZGkFu)-oqQjg4y#R$d%+K%G#6v>A=>ROun{J~5>rrHP z1aBiql{Gp9>%sD&FLFeml8^xc!VyzmUaCPElbPSv3t0A#9~aEb$D$8TOB=%?aTb*2 zNON#JVxh8nhVMR1TM6`MlM*@vIWBZqtKlkcgQY0JBq<3UoJl4G~o*AUoczc>yv5=hwx(u9$L5ZU zN1~H+Lt5WQkCnS(BJVwemy9w%AVZ*-IzmIX>kV5}hcXTvU8KMIGdK{oi@Yin%uYsJ zjDhFlW`H6DIjxA)A)kjVnr3FtfEnq~ybev0vabPM$bndkM@V{Q<_#1~AG|<~dHOjf znF|8m;#4-3l27BCrohJn??4#M@#>29AE>iRD6cJ+vcLya;BCP<@SjF)g2e~7!pf8i z&qn*A%9B)sE(~y_|oN>KKXW9@41wBvsgS=I&v>3JWxgl z1it%>IiH4vR($2z0A-&=_tslt9<>JfIBXp`v;KMLM3Hs6(~zz6f!yS@>~qRvnGf&)|WmI{gFWQ-{F(tYaVhL_QoLE!S0ZJggOh6a|P`& zHvzKehZ>h#7Kc&4I#clSq)$QJ$HMLHjqwp=;bwh2u#m;CM9ZYzf?U;I81?yV%-w|- zB7ry{Mz`Hn?P3cSnS2jCpF>|B8?Tvr!UO9pL=qYu7P{9SC=&u7tr~cjVGYP1wxLl5 z{UjNZVn@1;*lk4$ zJ=V-C38KlcwOen3_v@v(;JqIy8%OhC;{c^#I6O_Gw+M)Xtq^ghix+pP^&k-c#%(cK zZ7r>rZ1yAXJUf!ab4Dc$Vr5fIjLdqMRsa(Z)%Lwz9R!iX#Z^4d>|4#B`QEb{iUgxZ z0@~G!yHXBziDi5{EXc^u=9?G)i6C|^zGyK}LdlHc7XYVt17l@6$lvBV@($4X@Di_= z*A7(C^p|cO?e;cSF0Pk{T5LN~-b)&P!k8m9>`r&892`{fw4~7U6W$nI16&5+g;9Vl zKGf3I#slKlZvy*^+dkCxB;=$G&!f`D;& z=63MR!ZE1!r{$kAX(EuTv8t)=@^YIaJkDD;O#ssT^5dvls^Fca_+lntbLe+uHgd$p zC0=$L13wfkmg}a$fdfy-$*`!cokvg8H*<6!sP*&b!VS{R@xorOA6>sq(~=aJP!s24tB$t$&&y8v~lew4k6($1#*sW z@+oRaRgC!S)JuEdN1U9AvTni?pnO{y-~uWK>Vh`bWs-#;SJxvM60-*@)dc2tCFJy1K#|P(5V1`vIowhPmzKR&9=!DG{1ST) zz(#f`=x^LJ`00AS%^*%}eE%~V3z*ciUpB#m{AWg%$W0y;_W*Fe0zw~nh{9Py@=N1w zTJ1*L!d1ZRi-j!l0vGaV`u=KjI9k`#mMH*V+u`73#7%5IWsP|J%eDa;4welDiU?V= zcJ#V{x(RBT4l^F45KOh$7`FT}ZA8mzPYQ(I;gH0?77ap=4pjn&1XVg4*Uhi_@O0xG z!1*3ZxdwL06+rp9)>l@mpa2QDExX@Bjo4em^UqDo*1OiKi(gR(vYPNSVtSJj9Tp_u z2x-tH5yoq-0zwK{CiXxU9cLo9Z&Q0pAT=HQ)+}JEz2sK>=g*Vd7!GDwAMVS2F9-b_)_D|;3D&7<_`<S}8j_mmdIBPHzAp|u zF$lY>;f?vNZ=2^n+HC#;8X5cpPNHgwfzMPM`(C_i0%e07mW2yG_HwqyO19<1QI8e2 zibYXNBphI>d*Pf{kDAr!7QkSqqnO_2=5gZWG)_Q5&*>T9=2br)%gMcA52Bc!q@9CU z{tZ}4AUg`H2jL~wJpj0z@K*-Lx%-NCJ}{XsF%m$~rgB2(F9`k)0xwif2KsN zsZ=2Erz5}McgO&k;qyABK#yhhyW;qJuaDjjRc*FZ6|@Za08L9I`MSMJ9jz_H2uWgm zD(1)PCtUiM&IaI67qMjw95We#)JIlrE4D_ms;aaes z7HM6)%cFDCbCcgQXgw%J$Lv?MHyy>j`0LWSL0(r7sjMJucW9%^D>S^seLx5-qTRr? z&4t)A$!eF2{uO&DaG0P=`|CTQtzKnSs>krf_hIq;CV5{Y*Yo*KvEF{;V@^XOBGaOF z8b;G zGLPVO)z;RPh{%%|IGzEVIa2?m4nT^}4RdmCjNwRi_TAlY+TNw+ny}MIB!=8^WHBnB z7do7azIiYx0jmGJQmE{$xHuorJ==UXE&;dZ?Bnn2WMYx=SZ^CpW>>y??Z#r=xV}L# zO<`_saG&4hS9%y9soazWs(L&IjV6)J$KmL(8_1~}Z;iy)>>PJ=k}sXNWL()MhWN@Q zAjj6l(_$~dM~B6DL`5A`yM#=~^3Ti@Jnq9Pq^tMSif;{qf7Qtw?Ug^=#Yv>eK#&xg zrDOKj!4T;vl?i?g)?PieIzz?4W%l9NRzBafNy~$hHd_MOPSB5Bg!hzKUpiqizvv^I z&_2B?f$Qxpw5VOJ&;S{uBQ6QWrkI=%atE=0baZO7F2olvR1a`6fb2MA$WC#A)tb~nic3}ZpJ)J0xb*gp%y?=m@L(T~i#I5&tke*5iFa_p7J>bBFjikssOi_B zZ@g%qh7GG-b$D-h^D?+Mp$YH9-NMQl7c%qyR-KV)Zf?9N@V11ig=~ri2nXmFT>sQq z?g_03-hPXJSh7y;cnCF$_=Xf6}HIC{Fo%>6Cb>R>*A7- z=y@0L^Go96{t!XrH@cAF!&>5l^q@u^EL&%b_V?MKqWF0!mw*csQz~FIwEYyK3JM7i zQO0p8SKtd<9(Swp0&x(nN6$vS+&#nonxw)68r!4?L%1TiO z3A*27)8dO1&Xnz9^(@A-OII17J+P)Vn-ar5up8bJ*wOl)Du_jIcJ|NQ9Fp>h^mAD< zb_NEOt6iqrUS6rj!;+Vn=!=bnbc93pofMvjLtzZy_-SjiLzST~4YVJO{Efrt;PN6O z_P=R4I6VF9;sYgw)y)d0;ClwITvT8ZlHnC~7`GMlt+6VXxFY>mIP~9t3n0wn3a2=_ zphqUt%KPp?dq|8qilNS(MY>HbExrhp;mx6~^&7S5%_GpB>Q9A3<&XQ{Ir4E8#vZR| zKSPx==`l8LH`v{s|0=;XFK@Qu?%k)cZ&UNP)@<8t8MhP5P$)B{7}~rU^;+8jCuQ{r zTwC0$$gXyl%3(y=F#}zFb42ru{;}f&AgwV^yi7*~ENNF@*XM zJw)z@BE><#vqZ>bl0KhBHU~?H=V!aUA7!lEw?mo;hG0D-Tw-V2pK+jq7X=X(m3I|z zDxS*(DG;2X`#c!Hr7K%W>CB>sg)u%(i67*2AAZ&xXaV#Lh>>wX<)Kicw`CmlOUF?J zET7n6tx9F?$SE~7-saoVysg9`ungcB=GBh!1=uA>NJs?9TnxE#E_`8w52`&c2XU;L zOrLI25%ZKHrr=WEGqQtyySM-ah*+*lzIh zI&W`UkjFTvL~00{AdHgFqT0`5bGw6w==&n`@L?w~kMXU2WDk97k3#ePeiQd<#j&Tv z4%xu(G+_*;lC$fBDh0*)YawKEAP(H5N+Id^T_vSx+B_7(j3=IVy`Zp(_4yAlKLrK3 zV^!FvDUq1E6H~Y7_!xwLCf=4BOP@^d1!)Wg84Yq!Z4C|N_VyJR8yp|tC>Z5SjLr77 zHq|y3$@Q1UM_sB|UEe0|&t42;ph)W(7MX6m1@qb|`Ln=x?!QyZ9~dop9fO8I8s!Hgy{LNtFdV^x;`t2Rju@2;#8GVD)2xASgHc;2R!6ipm#?O{Dwns zb9h{Q5+~TIGc)?Nq#cdEfsuONu_ws2a<@C=3U5X%qOX9a8VC(+T}(afxCECpxpA=> zg}`Q}=hqE{n0lR6JJQ0ty;R&H;_4EznFmzpVvu#Z1A(D`co?45a;DsB^mIL`0_8<% zjryf$KAe_^>lSc3_>^yU#6*7Unq-BOM#?n&5IYaZkAat;5m_c^>oA(t9-9Q?OlZ$} z3qK8XRj>bpcGhN5CnI`$w3g|OvrdwmpRp(zl4-T^Ldh|+(yucwcTAdujwM8u*uCQr zmFV$}N25V$wtHJ9>w&Ve0vsxgSfr+^XSVvZMPI^+7?mSd zNJoXrIO&k+$D`l}@XB^lEgJjZcRUW+8@AkOt%{GlDCCXfye;E<+Ee)S0Is^9fm&sXbrejA3 zE`ji(*RQWm)igcOx_9s0itDl|*huQ>Os6+@XJ42p0!4iIh}~?Lg@5xb7WnlV{yoYlLT4Ivhz~i+LmHZ|^S`n6k4s(c^iz9cyP2;&60RDO>dBxHG3tVZoeW zL-sJVd)~OyCqK}SH+FXNrT;QtEa(4mayGnEP*9+o%$TI?_`jqs`8!mp7-YV=l!AHS zsnwP!tJKOoTynN_u&6Cf%mmhl!!I-Ty~zgehI;t8qH85(Armo=#st)KrgBv|jf~rt z&A*VA+%=cH+Au?sU}wJrG;xnyUV+m<#Z$fn6&sp8t1F;PU$s%0usMA7%ueNtoz-_^ zIvTSVqVel^_Wk2S4}EO!>W|@n*Ix*!Y}lQ`|KCo>hdV8 z-LXO$U%GN78z?eo!}#ce(WLurLc8*0N%LQu6up z?Y$rTWYzkB9UUMRd=Y0)Q}so|#qx72%$1HG(Yp4L;nhb8)!dR}?-d`E4K@qd&2nf~ z4ljBvVK5c*dqF3_KmdU8-2EY*`h`bVUolD4#n20nuRU%3k% zNb_Ss=2q`tPh4f^XyRfQzhifrFTm?119E9>#b>C*u(@J!$d|}M%P3upI&?_X#wL4C z@o}OlADqvuEK* zZKhMSV&9QKcl!M?YQk#ld6uk@pMz(B;STDwrUJ{Mqa!JQ`N@*1LGhUIjKB9dIyoa_ ze@R6h9?(}kUVcnGQThJg-SBiKtvR`H0T3t4*kl9LMJ}o^VZKBS6nh#L`%r7yK!=7N zyj*Cj{6Db@*4hTRJe)cC%Sm~l7VQb~BFa#8ftit;d>EvhuAAHI?w60+bNk!cQpD$z z^78%;s2g{?cIQd+Ub4whb8YbG&@@FYLEB^lw5X>$dqI?8wdo$y8}}JIRYSz)bv{1h z$B)PEAjigJvLu1Jp7`4=`BS5O)cIJrc8v3J9i6`dJ9jQ#>SIB>S2vFIRjL*G0wf9z1_Mep?GPBC67DP0jC1~#ak9%{-w9R|zz@ZmhFau`qm$h0(b zvnt=w+w~4lE83Iq8x&t=oC4Nt3#uY3tM{bNi|0q9s^!6C09kWj1b%S{5(v(vI&-F5 zXWR&hNXE^MZq{qAES`q;ylBVm9e;mZ()1W`j!>k^#FGbU^Z$uHSzgpM!C6MKm|>ho z2)yx`>F*^LQiJg~O<&tYp6RsEa&;ZAB6`S{N=w`xt=ZUgvWO9ZwN2fu!I{nm$f=3H#ql`bAVam^_tfhvJTX2NVK!21TC(R)baF*=P zGH1c$ol5KSVUSGOzbFFF&u^NIzf4ar%I@`RROH`tIC5NGChZ)HIHDj4eF1k*iQ3s2 zH`tE=vy(0k@Kvetjy&hNa2L{hKD7Z}Z7^mDU`c1%;S^iJ_@8jB22}sejN1Vra>+6!@mh z&z^zr0@tsrdX(Avr4knxdxwd^#g}D2e(=B}=Y~kb2d!?^81RTdkHV6YhROp64}*&4i$irr#NcJnqBgronY=?@>d!q(&BH5lM?FCWFdnf!^EFAv zL+e?z-dxhW_MgV6!4o(ti~9k%c>|KPMF|!b!Sar{n_9}sPeG}A&y^mpI0NjLhk}k9 zMGrwLfxj8+)rQ=nNa0aVJ{`D{L1vuJ2Qza=6O8%j&J*BVbxF{50HQM~F+RIFS05BF zSB&aFG>;G$hvre(Y4^EGWF36F*%o{oHy>Bs_EpuuB3*t9&OfgYeR0cYZ^-8KsZ$Id zX1BWEfWn->!Y-%@N1Wog8-Ot*2KmaV!{y2NX|eJLgNJQwI&~l1jf^Nxn147XZtdd! zpa?YY!)>odWXkn@UH<_^FdfY92-9!M#|nUWw6W(-u}4oYjE0dhry~ZTwX#^r4h3k1 zq8U$wcSJ>;)LApDNNhKIm-Fcr=>OQ8FT5TnFiE5!ft(O8wqx#CwK)TN*dRS#>@j7a(HO4@f>gbSqXT0gr6q#PqC0qb zA@lGTFE}N*KiQ3mb+`~27<|0v8Vor0I)_m~W!l<*)kFinpCkc5NkV)6MR+ISbN-r) zBQ3rYx9!E#Y-SSZl7Cfk$1wpzP8C|| z_P)6EI~uOT8KaV-F#{w{u50?TI|P$Hvm(y=tnf`)A`T zI5`~qex106AA;4RE}Ocvzcg#8D>d*Lv=tbSR`=$;dv~%UEspzDlbC>q7%yUxn->96 zHMAc+`e56otR)vp{-&a$lI-BX*f%s({N;;VPA(g#u(0pbG+R#+NevC?lIUnY(BYO- zgk4LYFx$1$MJDKZd2PBTZoW#d7tR;ineqlbF1(cV^t=GH(Q?;Lxvq3FWvlnwKv7SR z!=RHs`)4Jd$A7!9Q1sC%{P zb|CrI6lp3Mr)n=vT}mS8-#=p363xu>Zr;85hto$I)`g14qiy+ozf9ntzk?X7GyrMf z;NYTXqlXCIV{xVMey~e47jnhRdUMe=C-mL5Yix5sXDjs-?%2l3ck{3%P{oNbkqJl? zQ`Oi~K|6IX=)pJ|s6o-wUn5%TZAoXwF7_xRtwodd_!kCm*@y(0NbU70*6sp!)>8r$ zJ0{76M*_kWP-;BTq5bvde})#>0BYfl>VPV{2iLOmh6Z^xqMJq;1}Feh7OfAL=nV&D z!gfWK=T=yc_C*?cdYS&5$bzgaog5oIc%N#E9) zhWLB=!#||vgvRCHbo{R;_J;Wt{3-W9`7HG1e+xZrDE~cwzrORoAN*rz{(n3lE966` YkM8BJwd;ToMNlk2LH%CwUCXEc2l2)>ssI20 literal 0 HcmV?d00001 diff --git a/fig/homepage/custom.png b/fig/homepage/custom.png new file mode 100644 index 0000000000000000000000000000000000000000..30f3f9fb181b14ac768cff034b2a61da4a1102c8 GIT binary patch literal 75547 zcmd?RcT|(x7cB}0m7^&BEQo@D9-4IN5}Jh;dhZaCUL#!y#Re+9BQ;9zNbkf3NH3v7 zKn$S;2rZOA;O+SH-1qK#W4u4_xNkhhh$Be&zHjfn_F8kzIXCaLHI*-%V>(AgMRftH zqM%Dfb$XtPistU$XTbkFP=4%8MRkh`s_;PnS^6r**W74?tz%Qt?91t}J8$V8h}oBk z?R6z4PE-zuu36cB8%9d;>510xmkw8p?aka9nnoz_zO1JWqfxX83=aOAu{8j7o}u+a zYi2a@IakL zD&MIl3nsEJR@8*hCG-4xm{VkURv*(?kQw6+d{Oro^mNX<_|Lur1VHDSGw1cgOQC`)q zhkZAtl-=Ho`K1Qk+zVgmF-xD)__H=ok>SzNW3;je=k8IK$x81xa!V6t;MNTo{$daQ zTPW*A)(lyM;5QMqq>`ROwEe`7ORs8Nmib+cQToM(eP}o;Vz9Wp*=AM#*~X_#!q6jb zt<1h%PgrDF*DhrHB^{&Sg3pydW0IlLrNzeMa32DqxS+5K4~@pdfBq4CPmbGqnZ>$Z zWKxwngQ$Cxr_RdF#s;^ZEf|!yJv_)Nv+Ul5c$z(Y__9n{dC^Do&&8$}OHPA}jev`B z4EYs!d^o-!SD+YfRDh^sEJ-q{@@c79B@pssVzjHsmo71SZD_ycC^VlC;Wg-*%8Ihd z*FqXNjO##k54LP^Sc|Q)@{g*CT}-koeBMF3w$P!;8s=)(sXi~yELaOuZVda!b7!TB z9=&-u|LYfj@X3~cy`6U=pHX|s>O|F|*LuL~TGv@u7rL`&(V@W-%=po=gXxBC$j!?| z-`;%l5<7YyhPbA88dd2(9}C&qc-p_=PkCAvX&?OePPFP$6r3Ynh<5v~>1#VzH6;*1TheJY%fK>H(eI*!;Ss zf&pt$f|c~2WxXXz37;iwtKFCV`A+kAZeI*#ZAtazN`aDM{qqU^v4&!9=CG49Z2HE= z_4KUgbRH`Phl|i%6HMl)ahOr)&`Pf!N7)mo17{G=ISL))LrHG)F<$+9c(|T;@R!Tb zq1T(U_G{_|hHR^-yMIncwdB;>?Pq#+m4j!$R0g-hpiDn{K8J3zl z8`_Poo=z1ZgwrE9*Lw)RRv|M#WDmgd`*kUxh{t z4CRiZX}|<5hxg0tM=}}o)^c-kIeisEIfIv(Y#DemrVZR8uYSP?4xT#>8KFVM43iXD z#9r~@8`yPxB(vne-fW<4johEO`Ekyf9h;G!zJq!DF)k=o@rwYbS@$Fij*=4-a8Vs4Mss~gt|!B=nib*?o0 z!eS$T*1mbu{U5kk39EmlE*(t1Eeq(ITZCK=Rp9Nxl>4^Dw zq9nPo#qnB%ucx;-^Ch+$e$et}Qf5{bzuwOPNvCv^yE7dfX3@jNu19W<9zGPeTRl}} zt@NE^og-X$<|!R z8E)tBjg))NR7%U!jx1RID^rQZb5TIV`7n}MI91BIwL;Ez>z;t=T@7r|&5z#BjLYFO zY=!EMut4!@boxt5=B{{j7R1v>)yZ-c&vslzv7lY?WnT0b{7ld|Fxv>CrW#Cx2 z)KVNKAIa=EjIM@0s&Zx$DIl{Nsf~Lcd0uOgZx?E?Pd;eZ&YG~-hw7fO9z(kPU z+{!$Bg`OPEcPDG9`U?yDtk#(}F9~@Qc3-L{?TAG27?nkkh=<>eXY1XIO{Ttni@wnf zLZp<}fH%!zOf&;I(A=PBkforSAuKG+boi+2?SZ@ec}Z!(;NyUmS%RMR>*I2iW+F#J zefisoyNP%`nT6pseW|i)RO<$j;|87b?2ecy! zInbLX3CBA9yQKIby7uVnazC8z%9+HsM=$($6r7!dgM;sXc+F+mhP`_i%Ta3CVyW7- zcl@;1DP=Ss;@Z1~EAHK0n`Lu36)jy27m${2a%l;*VH8qg>Q)>&8^JB@O^To(hXO+I z&>ObB!OWJ!$UsIx^JaW42M5O;4i5e81))(RShQ|`H0kOj#gu8>9Yzcevnf4J!kSMu;Oov3v+by#5&<$!|e zFpB4jcG4Moy9@T7*db>Qx1YxtesC}q_4wUP5S{@W*`8X%9N{5gD%Kh6d^bh?9GzN9 z&&X7Mj$%r8a=F7?7A{pTu(s`Q|Ek8z8otv}8`d4-_I*!!k*M)!XJZ-2q}?HIi z3oR#9J{nno?9kGmk`N#sd1mO>IN4Th+4pBiCd~#C(EJ(LUu{3UP?!I#&-12D@S$n| zmN3nJdb_*3AoV{@0#Row2G0URE1!X#*99|YZf?Hwi!Q^nx_7y=;?h>!)$7+kh~x%x zLc5L1be?WJV22%9m!{bkMh(X|DJfYFMdbwSynFY~w9zGzqcGs0vgoIgpmN!5?;ym* zi@xj^v?{?6h|fA0cgRb~h{pkuD+VJiQ*@ z6JWsJ+Pkk2m2BirCUrXmZX7c!&G5_N9c?n1I#uJ>?Gw0rKxIvikDk># z+%4_(=t4ZJ&zBZ1i{mm6Y(Ox!jN@*US~NWf?tO1x?YK~K5Ej;$_wf-48W+Z(0aX-T zO`|SKvOCoxuBgZIFVLST^{?SIt#k-Ica8sPhj+EhpgwVBw6z$rZWt96W0V@ZQMJe) z!y!9SfpHztsa1CzDct^)z{^H%kDQ7i3wYkK=&f-c6Sw+mW8I?YxxL+!Eh(fT0YoqZ%%ZPAPSS#JcWXr^^;W$#dtBQXzn8WxiUE)GR9*LfTv z{1_x_VPVbuLgl|wj_*GF)3Myw?2fwc?{)@j8~(f*ETgP$dF@2)H37K%^O=L?sP?Ue z-l-ZlOzF|4L9t;{bhNKT^{}~%OL{A_c;9vMIu`3XdIoy#m=IGDz@5Fr!lGxl7y0&W z^*9?db3O$eK(l}1M@qe?7FD#Gd`d@ZHPozS{k{i~m35B|^-j_Y!A6`hY$>$om+Q7E z{}URIe{4Q)=J0lyoiuivu744h*JYb&&>7#PIyBFNj)baC)j4N$#?}$`H{<;eMtmut z%X+GVtwq2dq-+oRLato05f9Zd~yGtXPg`y z#=lRWNOC;VlaxMQyimYpp{L_8h@S z$H!-yO|qMu%(MsmSs-2tW92X-hjC#RnP|9ka7qX0_`v9OR>I zn~5pu0*_p(kkC-)&Y?t8hHAlYp2G-|G)b$6*q)wZmWE#{UiI`UXbWo&acSWt}M1{qm zK-;n-8aYt(0Tnr$ZRD<%C}3t9X9I_qe|gn9P+Z<31SL7Qo3qGV&tfh2PDH{?+PFFH z@bjB{E(i{RYl$HiY^n~Z&M^wwB%zOk$*57feqY@ef4MdjrRY7QTlnx*Se4o zSA1sz3j=5a)xmtnb7&m7mN@B`r{b*H$}sw>s`A^1Iqu6zbn-st8JX*LCGsF3+sXHS z)lNYk|GfL(?h$|*CYcSokpe5SsU}jcn4SgM|0zo80^a$Xd>SY8N-&GF&R7#6IIlOY z&gAA;k%8!iLy{V0`?koDVU0ut!=8S{6U9iOGd?b6><9z`EbJ2AYSwvuS~10Paa{P@ zj>9W|yD8lhS<$WOs*Ifjd$#wkhk;#Qk*uOKStV)3#mAO)#=E-`2Y3J7>{|&k3^#g{ zg+sl35(*k7O1pK1 zTdVN3k^AXW`ghZ&#YDN{p`caN*{`^wI~4u`{sh~c>TXCA1GP&W5|KOHQ)NQ-=?Uwgg0ph={ z{PijuTlv?Jb6sFF2nq_Cxc!gjR%^ZR`O&DWG;vqDg#+q3RXsd#o9xj9{cn6xQQ1H< zRZZe!U%x(eflYYSm@p%NonHI*hx43K4VdQ<<`K;tdQY;-EKp_0pwxm-pPuqKW(_}o zey#!YKS-TQel6O0ZjH=5N3>mvE1%w^5&1AU$=UU28lTlh{^n=*` zyA)J=bpPv!q5p5wTL1rj`kS6j6eMiIaOFxi=mI#QA#{wo_wR>D7uok`lEcEo^*|0T z(l2}>QMdKwKTJOQv5Z2{I9{W&JCc%h({8`v=DgqUggg}qF7zWTo?ELJ8*jk@{X-6T zD-)|-$O-?j!%xTit{Eqnx&~;UzkW}hj!__3`>q>W!D3okLq1UAD5M|~0iVR9i+9R3g9n#h%uW*w; zE-hUTU(?jo+}h;14TG^IZfzz3>R@$s)gzbkbX3o$lK-zHqi$~4xsoNM+rcB+=aTuC>rPX#Mckmds#l=i#JmJ>effHu-df#}5z7ujA( z6tv)-TA_^KIju#M#tJ0hyK(FVTB_%V$h2bTPsQv93o9#!gwP_*j5m>M;mIXENu+E*4<`9jo`fpN z!{&8Bem=xuy}6B2lhJwoqkx+9>hQMsf1~hLgctiV#^z$Qp}Ma~-Z)TcijA@XXR7Hs z(Wtksc&^y8WwOYSc)O~9NywtHq)p9`S8S(XYq4Jl@PJM;C9z>wdD0!FFS5yH0bX|b z`#ZFUx3@uCSWwZHHYWfb$5G`hV5Us~+gJ)iLGv{;bj;10n_JY1eAZR~k4SV$DcI>s zEHY50NM=9%WFGlRZ)pS1^bD+P2UI14uSNWOZXWS!%ltB zVJbP+h?3+L*@_F=-!uYnSE()Bj8L7ocyvIQ3K(g-p7b=g6j0m&`s@&>h0j))kx@sc zan&xJ>aH`D7O&1+l;qo<`Yt~|H&-TA6>K#3Fl1+s6iFdBuX&TVx(Q68BxXPmnz()& zDb``Sw{Hd@(3h2CO5Y}HiGc3zN|A&?J==@mGb(rb`59gVm^=R6NaQW>OOE4`up*a~ zA>V+gl&S@Mn$>KYs~MoiIF>i*UA@*OiR$>jFGv^lBQ8>C(l$9sBcltf)5;oYV%6hg zb+i9b1Q^@1-)Be~jw&*;MD&-?b8|1coc*B|7p(=gmI5HU;wBIyu-xhrbb0v;U+4+J2}xq06ciDb!(r!e~snlG9?D z?2S<(>KJBn3L8&()VkTI!Jd@u(lc1j<>`aJ1Erq69|<%nvozRCo)xBRA1kmlxM)Bx zHX6(-uk%g$D8`DJn(CRh0{G4Cew5uGy7B?b<;z8yTE2QuH$F;3I$p!_i~++LeFj$j ztN~E;_QoAAXukL_Oc+JnZv+@}W!TRc*PWcmVG$iQ+R;MdaquR^7iLDk)ke0y68MWW zQe@DYjj!bT^;OZjL7#^WIs_hh|0xamT z5CNTR^{%-l@IKFgZy_~ODea!4X{h(wxF6r+lgqF*!{!Ep9 zvIw^Ru%!u{Zxl#_FQDI zc0`Z5H2u~a>y0Suv|v|q^XAQ+Wo`oVRA}_k13=iAlwtpGY$^Bz2UIcKzW_dKsnh_v z93B9YG5(U6*|<;e;51%NHmSiyU%h(8BwkW};CXWQua!}b{#V&KFQ{eZpXv4W^|c>2dOI&?U!0Yg ztoO;VUCFR>#q7@TjDl`q{YulUUYu^-OF+T*7j<)F$(+6)&TM z3;;)AkfUNXF)>m2;8mmla$CZVg!02^Q`u{joy5u^757N)Ab#H8+nFN#nC|V#Q%Hbr zqY}fW+R{pW<-sggUen=kB)N;eBOqlUcGr_RGNS8mu@T}7yLXoc4ZNMks>dx)Vkeb~ zQr0n+pZ*?D)5SDiWInL#&pQh2>5nLBc*9$P0xz*tBf2Axh==M`w7eiHG zlMg{WiH3#@aIkevR2-+81AtxU&YsoSo1|nvTd_$QqO8x!al(%G+Ekt9-g`rVG$!f; zKu_^WNN56xmr{$}m`gyEf=p0MebQsz;6I|GYy|o>DF`G75xuwZ)W~L@Gvi)Lkap9f zHxhN2(CW}YX|8W%c*{XiWAlc3tJ5! z23j?Pb!iL7XV^^%qtCB6I%u|X)slz0F-;Wi5~yWRy}Z7GG`LrCc;Y3TqeW|gBEZ;V zK$nly>L?ZFOm%Cr5Io>l?lDP#jwr=<9^K5>$`DRU6R*qE60ZTYH9-6E@p=o?fQnjY zAfWadE&UGK_!Bh(2|I!&|6n{oeo*>k7WWT3yFu1#qefi3Mg4n`znV;mye3ZZiY$d( zP)rK~$t2bz^C9SX`oL??7N_`ijThEhqI#I5o5=gZpR^psYhDmD;xX3|<=mIvmMR~- z>ALu|%lU3bffcJ$^W_%(ld_T! zh%HAGyl{$*MF=`L2A|#)j*5~xyjSCSXv*Fn1Y`%x{C+vM$F-$^?uSv$5|>*NYxQFtO~}L*zd)HpyfU z<*MDVttE`dJ5&1sUAZb2LAR&{=9delD&5p{eDfw~Gz=yY_6pxa&MUX%vuY65EjiQ{ zH+Iya;>i#cY3+_~&Zi1+-#`)-8Hk~r*jFhJB25| z6ART;1r!Lz@kp|`XC_4{x*@vQg?dPekL(4U?-EBV;Cv~fgqgT#cR+wDlWVg(HdHln zt9$1<$VxVagbd|x(fQ0VD6rEf9Q^n6A8bYBlnKck?X%QW7*}Mqdl);^%gBafJcWik zjyJ+9j?ne7MMiaoWwpN@DluQ}WYqx=7C4EQwGJF2pwR-%N3*xB4>sweanGjK6|hRg zNF0xz{k)SI*iwu+t-hMQKWC&oNB^>JFsV{9p?{mK&JZ(O<%@dwEy`5DUL?g!bRlmf z5UAdI84y4NZ=?7voBvT;?Bjc4jrH32X~pAs^(f;M7BI%TtBMK=cFu>CW%~KR9(@}I zlAd0%VJ`=zCD5!(e^eB;hx&42vI=Xq3Ax5^p8BY?Cso4Ll&M=SscaZ%(IrU~f~hUs z%Z{4LVJa`NqS+QtFJRF)l%)>%7YbIiCU5+PO8tcHG_h-W*BgOfy zfcAMN@Qm#}#vVPbTFIn0(btQ!m7`4r7{Lz!oX9uRa{E-L)hSL9qd}~QPEp8ho!OHF ztH#uSndPpJNfc)2^Q+E<*~h{AU(8aHHutokCtKV7@in{-8JZc8QX*b*d3%Ot_@uks zvpz9pdsDFRH%uh}6)W^S%STznF8)`WN=PWn$WSgHado?VCASQ2dJ6DG&jy|$k_Qlk z-MgcQ#1qx*j)*a!t`TR0--x+Ry0i?hX;=prCusmiG_VR1#-P@Dr8r!33 zJdW!UM^z^97K~c)*imWAA-zN2bjZqaSCeUtlWXXj3#KTAg3?r(I8Sy&F~q}!WyrHT z3IJVFGBQdlrOdEYdKnK_!nDCT4na~(l~umLEN41ZyEeRu<*xBsR&6<2X-Xw2x5IW5 zg^WgxXFwz5-gCXPl+~cBBz-H{^JA%j~V{DcNt1 zZ(boa*Wq3}fv#)}0Nq6gGf{LYejDPskZ#x8N^~So`-O}9&O9l$9fObOBnhNW)&tBx zFCgSg$cpFgcQ}5@8J9DguRcc5<42E=l$dA)9`00*X9~gOfG6OQw9mQ@$l~7qjcJCZ z?~Ah(;ip$<_(&AMHd2q|QiW3Es^dR5H8ge=E4YV*gqXQzx3sjJxp4HbMeYFUzPcog z<&3Pm)8yh8Ln&ckH<=Xj_%;*UyH|+ICmc}x7@R_e_w4>*mx*8jiaU}8EmVp4u@NAn z_Qff)@tMqPWXFRaB}(}{t7pb*_G#K3wzQ}@{3^FL$g3`Is{loQ zBr60xf~_fAW+kj9RIabCKB>7-*qze>Ad0b%dqqhHKzXp8W0h~MH$DK6RqPFtuZH~b z-So$R%7pIq9iJTPKRQFK@JE6&HU=VSU?174_wnKB($QFn$lgJ*EA_k2O4F2rPiiKr zlq&6dnH<;Cc|c#F7`o==E0nbLMHD34c?4jtsYywEeMpV&iX)RiQ0nI?N2ZkR#-Z=wvYHEi0pFd+l*gZQsud@OQR{kq4-ln(&U zyasl840nn;SI8T-u@uoo8+$mn@7_OKd@w^zlV^@WKpJN`dm+B6stV`||42$o7UTMm zs1LxY1Y|`6oGL@FWv#sda7{(A%H{zGp78#C0U+UVSUnLf`S?3Ln7+YJPZbr+6Ob^e z0}|;+U!NeI^2`&E(nvcuLfE8lb4Hd3!CGCCxF>!KKgM1E6N`J^{a=dse+Sdw{|4;n z(`w>b0FM}uF$J&pPjRNItE!7lGy=cnctlG}#_HPY{LiIZ%CrcndouS3UmF&GD=J%I z%|r5aqABW9Gt!!3DOVYjKh)PhgJ1#7NCm%@o_-TIKYt4$-8Ai{S&Pm5?YXRHoHsm? z0nkbkl$dS1fUdJ^9Ed|RI2#Bw^*pnazo?#~DnR#-L*2zZ$V|E}8J_Wt4t_(>f{z894KanpitYyQiv>FwVbW{<{f zx4+7pcft%=(gB6CY`;DQS)iiI>9A=eI+}0Udq_U{wVv`$`_B}vGuEym?Vax*E2)=& z85k4>*G*AtT~1T4^Qf4mtN-sruS?c0A??lXAKSU4U%q_V*4{qwRDD_HCVa@4&3Si}e7HmTT59k^5aiFSDuKQpq;-GFTFT1c$11MCO}K0wPS(d&k9|P zOQ%fcWn-mokfUU`NV4MPb-yqFd^P3@8D?kuuUz=QLvDC(?IsDP<#Vb+Jhb@LC@?`6 zn%2D02mOJYV@6XmdEG^-jXcd}|-<2uD#m*%B4QAdE0H?Ys3ac~~CzqWoD09uwb z_XZhB>`MF#NU`_JYAOEf9~b;2Zm|WezLtQ`IF(R1w?tlpo~_O81p(cj-n4fAj;OWy zg_YE`wH}p8XB9iRJ-3jM&;iN68+ZaiX{d5+qM%ckiN;>zGP63LvfAGX8qB>t%@%&N zbZove_&VNuWq4=<{>El`d}9L(H1za|S_+tuxrN+i8o~l064YzGXjY6t!Q%&M)f#d1 zM~6ta>9Zfy;PoFN;A@m~(-LcB99y2d!P`V1=L90J0t_Jp}ef;)9QaGdF3R>$B8s{_? zWXv9Vv3HG&nrg_M)I+IIf>5XF0xLI3+LZTFo>58s7Pp~bwh8*UkE;LAM_Eq+W{Xm@ zy=aI$Ss~OGH2MM-wO{RAyLbUtL|U!7$O+XkD#wpkV9!bW>(lPi;)ihEaXG;7xpIR0 z4^TGL0}q2zu`H?cedt&J@UHl~NrKo{G^<+|OJeiO(FBW-(iDJma+CbB!1~>}cHsdc zlcIpGkIFK{z-qk|^QaGg{(R+EY5fz2vn7ho0CJAJxp=;6SmHtPSdrDYGftCYi%O^3 z+uIk6F@6TaH3idde^K2EmnfON1Kkod(Y5VP>RVg5b;XGF^Us}E&w3qY-}c9YJAB4e z_OG3pjvDxs4e=L~`6l8#dgNNQYeTE7tlTn9RwRq9BN`ePBx|Et$=MS*)e~zEZVIq^ zCjw{xG+mjJqT|@QY+MePhKiAAhs%4&yz0ld~r;6N1`hlu?L_?m&!LnqvW&!}v$VY`lNZcD&`7ahAK z41}N25NF>16 z74rJ#;Fm{lynpRO{Sxn#J*Nr=02l2fNGdh+UHTdl4)NPcnAipMi&ZV_PuK(E%8102 z53b8#ZECOl;iGR}8v|A>kY~ofN;yi2G3v z7zV{vMim@9Qf#C~O?`ZW6&VxrK_ji!k(gR@PWEyhXpo9uqL6?6#arNpnJls_4Kq0y zA2%j%_g_;~1XEZh+rEp=?*q2G;FLty;ZM}rcd2h(9Iptht1w3e$%d1qeb;q1f`Lg6 z@dQROCE!sT+_dJt%E*?_d(prU*Y&~)otL<^*qw}P zT%TtRzQ>_&^--CDfg$%wVJ2Ua5G)tdw8w-GId^UNidMCx^r(%<^3N~d0Fz{(qEbN5 z0x5a>mTs$9+cF5(UFf&no}$_6CX!NZ^^FUtK=GRC_X@EcS_NK?FoUA21!2_3PLm}% zd8k9r#eV0xuep<`6IjN*AYIC=Kh6y+y8g3#G`c1R$c(iX+ZeE9)uUxi;##^yCj*1h z$wy%getKqgE@Q2MwAG%AZa^Y1Onk%32MiuSo7D#mRLZC`ezDVrKB}(q_S{@Y&EvVt zWW+6FZmkU6<)KzUt~`jYaXk3EfCFX!Gr70VMkHlD?`5%$WM#Q{q53gdi#<}UKa=3J zv&5F0bN?{C0c``f)$biGmEQ3qoIv9MuT=tQ72%<=o>VD=%*;0a6(8^3azD?LxbSe# zM%RgI$70eB%VU6r<>b{e@WYFRy13iqyFLn`IP5Phunsaq0v96OOScy^>hAa+?L#$E zL^Qybwlki^vP>?M_hu8SJiBVV*CxM}(4=JMXRZ_g+p_?)(Vtjk*PB^hs3$NLX9`x0 zN$e9cjG;$9P0T}{h&d`b0F^D6IqdAeK|_Wbn&#|um{>dvDj@x{(T0;c!6|51^% z`pTh^CZ`v-GcrKUtCRIwj(`!uBzObk$H#?UCO$zq%k0*}+n5HsmEpo%Y8uk}(^js; zo~KPOUtKJO&r-L`h&eVe@?>>gG!Qu>wx6^$I9!M_O=wC(94!gUJI*wRq|}P9HTbSi zXU}pSmw}{rgLQASgRKC#=t|d{0uI7RWcVlGdX>arWo5&cTYP8rvF*{O5$zHw>LhIR z4U+3G0_X!RDs#?UJU||GC5zT#mc{Mg4EHx^$&TmOBFhz zm2`)GY?2R4eOnF7P3-w4T}O?_%FRw9YJ05(B5@%3PgdHRg)YKSR@OzYu`Ah68dBY#xe8E- zICSp?a1WTC!lF$-J$+{xxK?_8G5Ut^F9bL|@(>oVX&ZV41)puz+g=oqm`Y+q-tbwQ zEcTrt-^Dy`j?26q_iA&wvkY;xwjbCfMkLLLs{>Eg4#wBI8nGuAzV?nQKJ+|!l>a)x zGUqL?5xtn&3F9UA2j5!Bn!xgB*O8{qb%9x;>!0fc02tO4!>;4=`_l)V%UiNHthwC| zA)b@Xessi{!Ylkjm8AlCN5RK^hcD11&vDRL5U)x;dDL5=9 zZT($>I6y{DgUu)N2*9+QKEM9ERMB6BF%!B@cuz_S*Z?J*Ju z9wq9Rxj449w>c6r$xL!bi0pVd>vk7ebVIzk1w62J1{KVPiD)ULIegU%c_4qZkul*X zI}MN27Bc?y+_cug&THHrv3TFvRvO3#A@~~ddVjk3 znDOoof8OBi#pF8Qvpmc2x_~w(AWtg*6(6eQyW-&E;^h@_=KTKTKBBR4VQdi@SfUik zM3JxD-Q6iBYc9>ye>hJQk;_LpM@L=7+H)`4YkHA&KkB2XPR@}&EK1ugLYb#&sm5no z19={dhDML}S08~H%*}c3GFB#35JdK`tgA!AL>I2Jr(|ct3l6*C*E_sc$1MTDVw6(Z zI$2>|cjf|Va(r*G8a3Ro9p8Ruzj?RkKOk0Lp-;6>)gja^np!_l+>~=)71w53w6a^- z($+n-E1Oz6UL;6tN$B&<9 z1}-=PalNx6Kj$BIAxHFrD`~w+Sm{7MzgN3Ab);u9kYoO>&+52s3F>G|4H*5lJ)S;| z>9o-uAACyQL6?IEO%A2M@!{y8SGj^8D7ip-5*baCaLoZ6&D1_bh=27x7)aOkQ+FMI zUA=H~d@rh8D!+xii!U-Nj|x!p!_Rl3fyKRI;?6OQOWiPIvZg1hg1DU{-pRg-A1Ti6 zV0A7`6))n2K$POSG{g})pFmw{*>$RGI+mq0p?+<$mtvwpZSuyC#(~VKs64B!J;2m* z*vD*#jDXKHV~pLVYHNV>AzHG%*)eq^0}sFgMK6+(;g`Mu1Tj&%Q}X7;N)nbSxtA^5 zdPlX8a`x9^eBIqgxm)~sR~_mVEls((xxo<~tro z>VzL_(arbk^QN-?Udc^wnlcf!@rsaCaLBM&YgTKxbPC^2;BgpXs(RiG9ItsAW)VdD z%w?NXSWX^5T=}nb(mRb-$KGUN;a7TeEVU+7b?GtcSfp?D8Mv=Ee)r{Njdx0>FW$u; z4*+i=Zhpz3+7>I%!+x5MK~TX`m+HqWiM&Rb%+b?ceip1*m-4qa7Y*KB<~gWD1Z~s; zd{qr%KmKG?JrflFlVd_(3M&tT5zAARJH&tSk z8@VI4KGzn;0BqL1`gLvs7jBSBYvFY_Su!Ru(^$X?{+Mcdoi!w*m$1i|ye?boGBMfRm_?gM7Vu zr8oinIKhuitul2dY#M;mOG)A3RmZAKBSH5-*sjeKzi~E{?83E!coIS##Wi~4pB%|Q z^-U)2lJ%9NUNZ?Plmv28!f3jilIws+!Nc9%u-H(5BU|tB%op^p-`$k8Jn%n~`L+|BBoZgY2}2fm4|eHFYi>=-TrkS)b)thpz;?@x5?Ri{`_ESyU?uUhQ&CYF_!^=X;SH)8X$Ku$a^^lb?V026U8F1+7s-KK{Wz=bga+tV zp#7dSmiJvZugzoxlm5eXPvO3tNpVtd1TL}oWdu$wS%{e(*rSWH;y4AQagKo5UwOwZ zeH|apC{%SP9svChAH>`J0e~YEkxi`N@MG?V8;I@NC@nC;hV4azf~OCxry4vLMP(M; z6qP&PYwUQhU<7FQx3M8L zcvDmPY5j=3YrW%}9eSbCD+2Ba10VRjB3CX3%Q!eR`efSIJ2JA`w%HaQYj?QaCHe?x zI#q~Y+d?vFxyjdj*{cu%KHY$R4Eu7lPo>8t-*V{d`eV<)nkXhGKh}M^vr+bbs&a8)DHIvagbB2`PBt)dE1zIsifw&W;X6g?6i?b(k{oXaenvS8TQ5s4IW| zLUrH-){l}iSNaofm$^2w(l#2-03U4)4$*a_KJ8>GT22y_yhm8~od1aM=QaAE|2 z5IPo|sFn2?e)4@7$CvbCfhnk44od@Y;@FOff8%z%(@Zn8F}tcSU1OQ5 z2~^E83xuc(Y~@xLSdPemPk_gScP}Iy`n4#FuwgP%zJ}1*H@LWKX|XFRxtOQ79XD_E zJl!ohP?y)-cT{=5r%q13;`o37NXZsrd9(;T}=nb4Yo zD}|LJL?-kROeGevAn2OvlKM=*9(BOO;+cYReQGY-kVW3u!<(8dmU^slp0^k zIZc-|J;>b+{1!uFunWUKZ_i$*4hKvCX?IG8LOl`ZuS6H4(JZ|l-&Kb~^c`uk04~s) zr8bdU1g$Q-DTC_y^!DCt-O#&JXO59A($Q6g$Jl$vt-janTtu%&2>*a+=uYevsMx$K z&{vsMD^U2Tt+SNuGwGlgm&LF6_J?LI|HOs%$0b#>h~VI!o)pC5%tTLy^c4Q!*|Vd* zy>x=?B4W6_{~)MuF91bM|Mg|$F2YvcEgy#BXnDEO6B9Ha6qzb6Qj&J1A#&5m8k^Tk z+Uz*pK5MV3k$Kd}rIA`RQar|Vc#Yq<;56Me3Qe*ym7hFW?QVYco-j|R)iK#`d%=08 z`Cy`PCS*HPO{q!k;2bV;t)M`((wJZKBojgM6KQJp*f^mutsg$|=%4k2HWHI|l^zC7 zR5AbzT3GRTLZGhNH}#baA7<^ElUeh=uk6v&U_U>OE)r3`Grr@C;CwlN+Xb3yrbPyLrrraq83SJrG)NV7?GY(I2i_&-jFy`5mITR; z6a_}^5)QRQ?E^EG{Y-0J^7`cuRRG$I530sfb$tK6GBx43vwZ{;vpO;suQfX*1e^@7 zs%(45+y-G(_rLh8=;zSiRA-HyE_1~nxkafRkb54>M`oZ9|Xct2<5#uZGFnfFwVGZ2< z!@FzJBNbQyf46pJzZJrX{A9C#Sn=Q#57@knvA24uJky^8Ahs<)n=tJNjJ8jVA-}G@x85Wc?yn_?6PQ@E?dlG{*|;G$Rzf+ z(W^6aO!t0LYov=A0JJqMNeYzVTr{N z{UzU4#yks9{=`|$#UFx)$;{v=Q(v_p)n`wNomnF-E`GAkt6mC~7GHQ2V5fo%IPn}w zJ&(7019n(r1r2agxSJ4ueqpD_DYN?hSo2(a^ZJx?*ZkN#fndLChPu6m4`+tEIJi(& z@15Yajozr$8PA}15i)CcQ9798VuZ|UOD1bwai0jaX63Ckum%rK z&|&r55i)Haeji@#Q|hdrt>hqRnpUHi{||9>fDs@aA8Q!FQWhN86My5s%*a!!-$^W} zS4$SefBy0+#ZvUvt8yrtXyb~3Rf$mMw~|aESf}HW-6>NrTF;6n{SuC zx_RXaU-1Ypw8*q3j7h@f@y)|oX~Oc?kYckE@y~9gG~m+U5tNfdlc=cr$D94qDlnC{ zU((CC41(lD^~-MN+;7^KJ)Dxd(LUOpEaVJ~(kYq!7Q!>~82!6Vz65U*LeE4Mz3ri+ z`zni+hq_UjdFO^d0KyO-{(rtzL7$7Y5Q&k_7dn3*+gZu>?#rc zemufFJm0Km1Ef1X26#4Adtcc4zsy^5PG6m`SnZ?YkbmU$zg`2dBH4WS)J{iINMp%(nC&;qha+*nd6B)0x`Q#GX<6SR^HjIfO*jJh>pkNhx=k(%Du1W6V&XXH|sk2cw~PRuXvvXw;V z2j`6&FZy4)#9iZ#wacC^7`)K#^DT6YM@FXSPVnvlf1vasZT@NP#JKw76+9uLEp*Lx zFLQh7!yDEy`6*w>YSpAmsxX#L!1&X^_}C{anvjrOS?Kd94Wbu7fY z(ZDd=0PxU1hzF~TPE}*a3lcYuxww7Sr*pZHOkB{gFi1A2#)9~eH*c16b%YAcr53f` zfWIY>DhV$l;^U=1&GuzxyPmih;SB|B<2B4A;N6wm8AT5dwu&t9Oxf{Lo(p4P4VSZ- z#EbA7vDsUD+vrCZWk2A*ev8z~1T16jF_~?oSF%nAq%FY5(~}!k%K6L1gT5+&lEBNw z_3%+rY`YB|)+W7`$cD>G%0PxwXh9GMqPG{+;&#_gv}kDM ziuy97!7-CbUW=lHD7NThAc%KX0K3h=sI$&h9&4Lm$X4aa!h9@r%m@@g z@YjI~T8~zkvs+Op@KNAobNE|9y#Q3!C(A9t=HfP3gP8HQVl}xrf`_`{U^0l-8(y}#PT+U||g=iNp zst3;$efhF7zIm;ew8hM5uVM!*DTJ259TGGA#^FMa7cVEz-COr*XlKfs)>ra4)q*4; zzTBFb!>fd#r}vVwu7RFIFaTJ_662s0cjScbEO+pq+{t?7L;ye z=~#H)>pAE5oImc|nS1X%bLW|PoZ-yTWxw&s_xt^dkMxTog}HCbBaf8C_)Ndo6ZGsA z-4%dE$2V%Vy0s#0li#;gRkiI>G>5BWE@i1pogElc^t}YP)%6u_31*c1Xw{Qp)!%M$ zIbOUs$6)H0s^N7b;B9MZR`2imcA=n05*fr7yLY#$X*a5#EZa+yh72D`Jxh^yBhHj& zSUvrm;v=SzSTBFm(B`s`D|=1EfpV9HUcg%7j>?(B{FR+;4zwi{S!|x2TqtHMwRxF} zWmA}8BT`rxez|u=%@h=Y>tn8y>QtuGDvGLL!4InO3u!|>qQ1XJzHPqx&}(O5s%&F; zIEod@NVjj_XY~FRWYt$zH=_;HYi(&n3G1``Uc@|c0qW1~`FM#5lD6mFiZ-JXC>wh6 zyOHLooPtv*i(9H`O${W4+dqsKud|l&}XoB?h2$JyKBE{$bnwTvJs9M zQ$d%oH$dLLq!=^$IWnX2ZP?bVsLOnUN@6@(2WLnWq2-@-Q-~H)6ePO7^v`F;i%Qx| zt8JUs9kU{qnZ4<=9xTX}*mI4x>T{`7wk2<`fr5cCZN$rqPfP~-Dc*JT3|VM{F2ac0 z9D3sg492RxSw7$EmludbIf3S_u^*)cOZPlhS656Gw?44{RCd`It)_$n-n;`g0(y3e zPnT8eoIEcI_y%9@`tm%Ob^<%;mHFk%7f)L?Y0iuOL6?Pd#?G7}tu3o84EC|Jr;TdF zaj>!~9y`s@P2q^~A|W_d1ukN2$|)fSQc@6opt9%P7a6{qYHPF~d5vn{ra}oF#Q67B zR8$~h%%VU%yk@df)0f|;D$U*hejVuVNJB%z42UtlehtqT@A+El)Bz>$mt_Fl?C#3m zU{1>Dx>|-%{!bX!n8-i`;}HFdtw7B7aF>yaibXd#c~lBpR_Nie5htLA0rnkKj1noK zP;;}1;yqDb6fc`1XZ|~yGFHU|F$iZ|lL>Baw$YS&^lY{_{hH4{hH7nebkupe{Y;#& zlOhY*6J@LI70w-(Z*d@bSJWBmVf13o+Oj%YX}X0O!m>Z=>e;3n-d_{(+75vBxuMRE zYj3yW4*it_)4|D_yshBwFn%eq!m-vEsjBZ;vA4675X9)O{Mzq%hDv4G4UH2`k6|t~6l3JdQ zH{3c!FLG6@?_`ed1DImf`f-Qi>m@zlB8Z%jkO)Zc%@jN~?j;T-d_|S*ZU6GLJ%>O% zAkJ!7Cxw;P}*>6evokEm=L2fo7aHmUX@eD ze86q0LGJ{Z1dEHX8xf8oq#Qom@B3hqiWYRI&}zKlF&1yy+y5iJF%+V2Q0;lZ0Dm-U z3=2<=Kxp~d-u3?ZjRoS=@~*u;Bahb|%RtnX+E_GTLDCNvH!r1_-Vnt9d4WTdUEOJ{ z#_PXN5EW7k894pin^Sbt@l};QWhj47YbemwHvW37G=)Bw_nq%IP3TLUyg*FrhnREv z73b%wdR^NaL*I+I%$Mx$?y6_HD1?(X@Jqn@`0?E!v!wjq|j@@ys|op}zaY@L$GaLrgr|)uoCJ zF|Cb(pX{lCIiFP5-F-<<`9FWL+G>%8t9U1Dh+wiy+e{qr|Ebb%_2_>w=D78nw51$7 z{$O9M@3%r~gk*z7w{zqA^gn>qbiwwy}3`*qv@ewhab zhU<7@XQ)7o{0)_>TMEy6tU~FiDX_;H{1$eZC#o)J^A;$bi01?S?pyf7_h+@HI5e_O{aI3Fgu3$0_+I~Ly>BX1aHhq znIFuPzsH&138i(s0t~gu#ZXojo%3`N>@lri)4`lSU^?_CK|q^VEgOEPQ2Wm z#d51sh4`|O&}`#*R_pTLk9g$4of)i$*nh{n zl!LnGo0;GAOBeT9V{DnA6`aMqIKdy+srx*Oc*t7o+6YBq5{{LpR`QM{) zTs=s}fMHYMW#;QbC4K<4%r-Xv!vPJkr>yUj>HhTM?)u^$;#ZJX#EjBmj|~OiF-jI* z>O{>B6{5xR;^L+cTx01eFboB+d^m)7_TtUdIfT@m(T+2u^{koUL$LG(x?8@udR6fu zrlO*ScK7MNrxiUcTi1<&fXx2HRpJ2R{Ozp|Axe1j{29+xw?xnN`7QZ5%B5awQ-Lo5 z2hgF7y!mA?-OEKW8kO#a`f)sx@k&EprxDLSC4XTaudJv5+W}=SF|U*i&FoJei!)Ip z3Q(~Vmg0o9sT8%hd&KM4Pc}@FPm%Hx1;Vr|5*TJQ;u<5*Sk?i}_YlqyEt| z0cg$OSV~2%vfq!ZrW$^(t zzQC&YPmPf(H2A*eMM+?jiXny1)+tv3kJ_*vD>ZFxgV(-&=q($Jq015z7$>A&Rg%Nl zRs3{^qNzxY69kKfal#&onC|J6H^5g6Yu;_B^^aJ^ug49Rp?uro{57?;#b5W}0Mf&S z4#aM0%V+jvRQK6J{a4f(1R8)B=m2=DniylJtgKwixkQHu;bV`g%Hz+1H1ZKPfPoNb0`67H8}VdjYtn%hgE10`moT@y#D&+@Z^u+q22X{es$d zdU_5Z*aLts!oRz9i>h%Gs#nhBwV%>%GkM|L6MZ8?-U}fi8B`O(lDk>d(iO(cV;ov1 zR6+b7U=aW*(#&Gjp$iV9Nd&?{gp>-{!O~GK?!2g9Fzh~O&>FRp-)vgJOqmoMW5dpF z^T4;Jz4;62Yq}dKsE>o21(52yB>Tp%FHLPrFk8;#NycH8nV2;4`Ck}aPaIvUwEwKe zTjiL1#l)j0qmYJ5q9r|EyzCNTg_^VJ^YAb@T$=9E$Jwi0_wh*;!3BAAG11s#U%pRD zmeZO}w~%h=(FV{D4YU;B08>x+C4Sq%hD+m5Sy-iOuyomGK3Nsfgu@8aFScsEp5)fc z*?QHD?2_9Jh_&hA2asl70Hq<;ds{f8EDdKe-IJLe zDt+?cer!A6J;N~7;iHnIUoSJ%Qa|j8K7EK6^&Kt7RS<|2)XeTjR-z%bv?KsYWy5xz}>G|RRdTG`EZWE(eDR*lPfzhXe$U@_7(VE= z*)fGlvhWp4StKsrY>VkQk5WsiVnfajFHjMQVkych6ypRF%R{vgS{4EM>o zRb;>9ESJ*fJ>T)1Qpo|ErBASM-Cjz4?QTuFjqC*aS)m=775MSpX+D5SSKb`GkSLb0 z??rj<@wqt;Bmn98@gCj*Ip4oO87$}Ph82di-WIA8d?2CUq4|$3icgUTx=f`i)t*Ou zum9c&svOSb!?E7v#oj4w)l^V35^VmwR1=Nd!6iUFf*DrqzQmWugt}=~+Yd!=ZeKF7 zD-$Wib*L$w5(ru%pxbR14j>ZP2dbBz<$~DT!WW<49Ckf>>WyUj3}a2o*A&c zfBdSqDx9s{u(*U`m3P$vojPw5qvvK+uK04`Eooju3_w=fYvqK<7j}Cu9zc0@PI&eJ zLW=ZcdWLadiM&nB1|4Elt2t_>6G<;pfJzfYkq(RVx;}gH;zko_Oz!y>Q5~N=v`_Dt zRt?N%E`twZpUOZO87*S=qQ>by^ho9M_LIcjg?YwMf?7rxc>atpVb?D!XM|PPINg)( z%6ZuNAbMG0FM)htbtS2J&EFE0*@ug-_m2f*UOZ{4e1^G1Rxx#GS~E(s^UM2`sZ zvO_k&mh!cq>Ug@_r!j89<29LuE$mja&rPbg#Ybu#*;cWGx?UAXlgoq?ibn_v3Z*Bpp%zI-VV9&d{? zp@RB^ey-2+O4OXo+AKK%@Z|B{@4GLIE?Q4u8xymw)KB-QDN#8JRaqzvt~>xU6JP{S zk(7>-F>75nmI%qCFEE-`72@H@%M?W8*o2 z6?tH2H-xsw{dul8s?>EEQ!~ApqCjV+I}#_mwUcR&9akYuwLO4hSu0gNAa&L@=-K|C z896G+vSvML3|3+IcyYGmmSt@8eDzGq8`Ao^;}*Wbs*v^&M=*VGNrnId5nF0@@N-&P zf#B%l$0(vhQzj-3;mLo87V&Jf4)mD7@YSb=Sy#qo)5%N))~03Pk(})l@nag zr_q3(vGev5qP!M*tP63RJ9W9L#`TAg}T}9 zC{JD9z4u=abtl{@QL6+$!DSGl<^*X4U2Cft5CWGTC*&Ya%2hggv=@Y2Nsb7UC&m@h)|tY`ARQ z`qqm4p+hh+RMJ%+R0ruFDs% zo3%#x9uxnOXP7`#cZNBX>akSH(d-n#ttKL43kl4k32&5om1hFPxgVIhHn;S)Hcj8V z+>GLlrDekbXwlm*;YL!gD)<>+CC+cRtA|OV#C&lE_I#bd3pY z3o@;GtftVRyy_qC4tj3APfc_Ny-zp?gw`V^1l!_5!Lua)i(EaFM+H^@3o@MJ2pHj5 zDjo5NL;WoPQ>as-=fDIjrM#CEfa!wTpYp9He$klV{rfn$2f>CQ2aKkHm`CCI$87)=z~Vg(5;PX!_7~B z+|$<|(#VGNz9CXCC2M(wBi$n2yLa*t}WIoEIA4C z*sPP(JXYWRaqJ}LFbL5yWR9IyuA$3Od)V{Hb|hC$vL(*G+qSPrqe#MG7=oc;FZ(wt zRUaQ8BA2zVJ>dkdEJR&kXTA0P3YSr?EM?)W4jER|mncOm5bk zEnk|lY?+=yY2jmADV6yljwGwkZ^Ty8&TPn3-8Q5Xu%pZuttU_3QV_ICaK>1P8kudH z;E3K|5rnLt-`i*|I=XJX$j1eOr>>rfAc8ZjLZEi&F6`??TYHof2XrLK744U9-`wo5 zSnE$b4U#SC8xYUAEDa{|n73*|x&)Zc10_j*p=PJaRfcb~eLbk3j|U4qf;)m}d+za( zF*h~FXV}-u_s4<=g;oka|PCYviVh;Qa~#}^Xb?& zXKzC8fQ<+w*i?+dtjWx^;_vm|!m&snYh+L&ys|8^AC!qAg}m~b7A$NVPobJsSe+ab z##`H>W#k?E(~YtTEsBFm8R%)!_DWs=GAID^26cl}(83xD+Sm9*7d_646fOnqMW%(i zaN&UkH~}FBC}A)YDZ(99JWUBR3^Y_Mh!-l*UFd zB9xu~fwg`mSol^ZQK}sCbFO+GYvnj}RpD{O51M}}&SK+0b5(5Iv_re}PDfx(&S)Um zk7{>c`27T5?qM*P0=H&K5!4OUuNpTpAJ@#O=b1Lz><=5~J|S)^V3L}`U#O=lW{cTy zfk64+aKhy7|2%fA_8*K6uKK?Vh+wZ|)c!A=g*CHha)uSKHW%}JqHhxOY*A;FD7CtD!q>^UxoRIY=W z?(G7MZ?3HigcUI+M(`Bb+)OGs84Xw!XDD-K4{Hqb=?XxEuP8;fvbsgJLO8%1rI4p% zE1Rc%StJh(Vl!Xil&Xrl;yUYH;09_=nfxaD ziQCIYiClrdYjX^7{&9LP(FTL@bA`Vo2(75w;6q<|s$w4GI=q9o=8a?AfkPN{49)XLN6-1I!zB@o zVrK}%_z)%8Ja$%rJF;$>k4z3+LP*`3Tv$jxTQv+0w7k;YFxxL?1eM>j5sYDuGR?&V zz=J6bLyxSjX&Rq_mq&cjF|2bb=4Of6E!F!Jh6yOvMb<$l6eltJ!>uM3##k7!sAb=+j^tSN-?refGCa03vlclY%C zl78~oBm_Xyp>Qj=N>Hrob2(KNsDaN4gGISLD2ydG;*Vjh}PZkbc2Rn8Ih~r3)tIjo)Jd zzc0avs8+qg6|28BraVY|#oGl)<>6p09@DV}vq@X{$Eb(0c@0+ijcjHPk=>Vc+mni# z{o#I6DJS$O{UJQIcxg7}m6-j&0Yn4Tc9`AVVgeUlq$L!c1hj?yR>uu{%B3G*%1xf+ zoFl=Q(#!@HhmHt*-SK>vu&9#lcjmItGfNA5|_aGMTeBebHh7vMp%%+YV z;d+x&l=_SxUg02KQ);LFNVYgkLjlUQl^XPB3JxKjYN(1X9I;WP_*;G+JOc2P1=kPC zGwai%>r@35cNL!$UP)ft;ThfUY-EC=N~re6cE z{e`ZwS6xaw3Jr7j{~;aWpXD0j+W*zy?9#5?v((z!`ney^YrtJS3pOyP-5sD|u-}-? z;?^%oBTXE-a9Lkxc(`^EcLrLp22!A3Kdov2*0`28krZN3Sdusdaw8RGdPo>HFhfy_qPBi7UUHQ0l7|?l=Exh8ZNGMbm)#MnbgJjj#z^MZm&Z)Y4PBnOPv!Fi)IzP`H;NTz8FMmTnY5)sT7XIfHS3%yGBT_J>22K} zwc({~pjelR`PLeD0Kb}1q}W33=ET>_XFRBqTnOIBO=dFpCOf^mPa4>ntFm16ofucOsIOm ziet7h^^NTH#0j76EoyOIA=`{dF1@`DQ1Q@A9Am6df*Niz7=cCse34TVbZza)4k#ws{&2yM)g2cegLKkV4B>P*ktKH2VM4*f0A4x1#&&U zCX@OARf0?QxJh5NVVU!5sMc~JRblX_0OOpedi8Eq0X5=B1HkS-zTV4FNp#QY9jZ8c z4w&Eh89LXxvllqnPptm2g_qn{_P@Aix{KTw5aaOKT~5uV*X6ozA|3oxyPZ?4qDnf@ zeVBcJwd)`YD+|rUWNXayOe$jpbKs?0@~2PFE``8}FP}e`(Mou}RS5!O`Pdn@d>cLB z^$oj<+YWEGTQLcj0lI!7kqztlovCPJ4y_dAUG1}f$2Vv@tkM1w(gI0Yx`sWSUZ@P` zTVQiP$5t9qa|;xSqRM)hAnv9pWq|rkI|1jiwtuhzXQgME{7>3nT8&g??~=)@%Q8!| zAM^%=na5btP3o!2b{5M6NYVU0Q}^T!AZDPFJ)2vP?WuW*?G)djy4ic6H66zJTu%8S zh0QsODj%{`QB62U0s%x=wa*|nwn7II4_KzER2Dl6^MUdfI|~;1z5NvG@raN8Qz38k z5<|4;hBc}*5qxXF+2wUj>W}&$27I0u)@MEWmMj+}M2lsa9s-L%N&D8?*Vo5XH04Z1 zss&{Dn#8!PcT*%-`HheAoyI-s|Ig}@Zrd5v8xjN!bV}| z2VU)hS~*|=>VMoae#v=^TrMQj$;Q=y`w!x76@lsn`?!(cOz`X69 z7Rms&H2=-b!yrGPod6MB(`UOxX+c`Tty{TNrybV)ONy;}^v(0<>Y4HzlQgqC4X0(| zkp620wJ1H<#$T;;P+x93;KJi~0I`**ge_O+B8Ck?S-#`*$7^}9RptYX929ZGK<6oK zyWxuT=-Z7YK5E(*1|AsTFl9ue)^qrX zM)#PK7-zCP(CUPjn|b{6RXCi8WyGH!iq13hS$K~*?hkvlVVlg<(Xe9?hq=dxx16zw z$3EaoR^s;hw^pbjwIn?e*}1NHL%n5}Luh9VYG7R-I0p}2LBCzdDZN%iIr9tZD^4DS z5J{K)^7@bZxBN?vAR0WNvN&6bW}8b-jpuX3%IYMbHl288eD*!m;KVqWZ6X}KxVk-O zFT7yine^V!$2P0RJ>kKF_X-U%GN~xVe0M#=vV!Vd+kGB+7xJdbp#IwP0|zLM56$kN zl;}L|qSh1Oz4ZVXqi7v!L>}+j0@}$d=<0wab`6y?hY8_{{L~frhC}Dm{WqvCM*pB7 zML97-;9;InyWI8F`C!6+P*I89$!9}MMdg!0{llOtI!k)+3N~hRzR%63V&WiE4XzA< zh`(^@Qrc;W$l2JiYR3mPZz`w5!YE^hwL+lLGQlyF6%I!mExDY{rW<*FAPv=K_sSC> zN8W6`vVyT?7&jI>BJba)(oeoz>dEwN78In|e}JEe$|5LR?Mgo0DZ)V|-+`JwPT;KQ zO*(7}p)XfSmw@Nf6Yv+dY7RqsNDG+=iqqRV5HCXIc@Wqyy|C8?I2JD&{H7brq$9UL zLP52o7D`HYYNGBlt)P&XJ8R|YTr?BgdHPc9U>wU5|9c_eQ7Gh$NPX7-1gjj2q;Nlv+x*@t5wEBmv1y3 z-{N&i9K|K+O$bt^ldq$TNIr&wd{Y?5D&BirA8)ADZJk^4*&9eNb@JhDf+C-B3O4kR zluM+1ivIqA6X$j|W8)&G)=~3gPCDfsMDUhTtj?#Vi=jMRUhF@8dWa?;AeohH^yQ1Z zi~{rN6iW3KljqN$yQ2q9U%%0G>IVQDT*zj`Y5@@pgxIC|fdmcilbA0Y_X?6?f0x7w zgHq3dW!D+bNriAw_w-qBui^t7L2TeG=j)=l2DGcS!8v={2~eggYikoZ@uv|wI=P-I z-hv;cbW6Xjy>LD0t&>&U)!G`PVsKvm`EyJE&lVQyMP6|v7yqcE8Yg%qP}O0zeBW5W zH~zxH#doP(IfTEa)foa4l22mm?p&|!&88Y(@v+ZvI&<ROh5j_+^<7Qm z7HIuv*81wo^h5L0Zse);9zL3ax|x`#Z8JYqT2R}5+-Fv=OkHW~X=?+@s zOojSpw@y;WuEU}#LT*fJ3y++qRMe$ZsFm9stio10XsVwEOH8?PUb(_$X#lv7+)h4N zAGj;*jICEEQkoI*tUOz{Qd2E8#q0Q)i<(LRX2;=d4?+)iU zU4kgHKIj4a?c3kYh=DNzbOac?O}}Dc*7BP+tWX_aO;=+D@>B-RUJCF4Pno zzg>iYx*1aKdE;R-NaNve1TwenVd1}K_>g7In^sy;mlrBj*%vdv^a{lf)dQ-H&s zs-pI&5X!%jSSM2D<5ED%li5rkx7u&=($9d`@PROs5okpzGa3|@m?R*lDL;e7yf%Jq`5s2AbV72q#Q#AExaD)&qEN6hxC$W#v1 z;3QYTLI+#&7kW74C;1FdkU$eG-_g8lr=WEOj0i^%CGHbV;XSXGO-SW5TR9DiIsia+ zCt3(5Ry_H67B*~=9=&q&xLcoE;J7?4z1ye^Cm=upIYfmL+O7xEi^?j*w8Pl|UqVID zi%)EDdc}8A6(5N7yc6@%1p~kQoj{Y(QsdVrK+BV?Vpy;#7}2LoA~4puKpUv%z8EWT z`D+hu$fm^9Ar@G{Cddffr{apiey`35vVUc52n#Bhi7wVa6#y|}^!B<3RXvb4Q?v}d zi0m8l9SRZ>5>%*9@b-}U2EJd%+_stSDoegsE(S%FFO`&{H-{Fd%jU~(sfJ)ANKrV7 zIgV=^yhG~6iUS3aC(yko$D1DfrkZEO&@Kx$Hef`h3~RQNP?ht*32;X`m^f4@XOS$U zr(b*oA@;i8@HRL#X(>sH4?P~~PCussl3_|I7lG#eM^c1Yd6BgEYqTM^%y|Y=Qw2H4 zB?2`4i5%V9pPMCGMw-eYV`3?M=AIwzE%3LRC_2fpI>DLI*Z#T{<7nTGWr#H21NFzI zyV7+A1MpP^l*wnGwgKW$=D6ht?up@Hk>hz}aIo(E@%2!iSM|DDuXE{`OG?&(XT~Wu zG=2Urr+h2?79hMxJVhrcJf9!Dux_1}G`s&uvhd-?+8~6w?P(E)x0*CwzU=w=!G*Um zNpfF*;n!KO&hN{L)e%WaNzA9*){pqkQPN@xZ1nY|Z{d9Kg}S|?r*gUq(8JEc#Z15F zi_|g%A!6i~6^b5Hg$g3Wf-2@U7|P_uxkNa5+j{ylHGQvbXwqO|jtU&{YHY$+6@2E} zn{+ccX4eMFJv2^QHawr58kYKVyRQF1@R#rze!QoOlzsR#9y9q~=I9M>iQz~s)DDn9 ztGrV0(NkOoHouBeHM$}Pfpnw>)FFt5Bg}RdWRd|zzdri?xbJ;?IL|v*u^x}D{-KV8 zJHdPos|((j^w^*uQD574*o{l#YhYIsIInAUU_kAG)E9B7`?as%e!BYh#Uazz<9@{20b39BSGv?o83IckXOS2(Sz7*TYvcU%Xryj=93_ z?>d-!Pc+cmE3)|}m!(xt*2g}X$`50|99d+;+TjU%W>To~bis_a@Mvb20{v}=Z`$FX zb@Rnfn?V8Z*O4PfF1mo*e!~SUecq&_u*>`_;H@P*vi=Nja{m0o_M?dNRioS5iS{FjR2UjRtO zzf=^4|B@yBbNOe-BIREj02lqooY;qu)KTD|3~RRxQNu&C-#SNtzfy^5>s_G`gA5~E ztLQ})hHRk}sq{7uwhOi+iVdkO=hD!i?vP3U+SPYHEQer zdgZD6`lU2fRNXKa+r6M38eU)T2fu~q+j_Fo-`8{q@g=-2G`HZ(aW{W#Eg#riC0xqY zZgK3VzuqiyQO+wHik|w%IWOj4ruyA*@7YW&pPwUne+X0SpLbCTqbQfY;_;tMW4O!) zM)l5pNe*Y@N(cPM(S_wZw0DSPd_eM%B_{h@%{Bs!Kd%5Z7D5Nm3Tvoov8bs)jOlXj zX4FN(f{QOc{1gIV1pj1Kb&zta22rJYm*CjOkrDj4RCp}yUmF{tTmyETJ6Daks1b;& zhLMph)87w;Ni|9HJxjW7GR;hwm*|7Y24OThW$}K%R8`fn>Dq$jt?s_0@MdXfmMePR z-WPRRCOIfz&a;YT=RemTo+Q}tUcY&BLh(7#E@D8w29Eaw@nQbKT`rg%%*YdeuC7eW zFT{^uCeTy)a#G%k_g0L;i9dd+eM{!xpj>CD^l~oIBzx`)V!vBwf+ZBmyY+L?9f*i1tZj}8%pC!~@swGR0ZX2FVv4SsUULPa^D8WU-S&$(2uRqj|m9~7B-_T%F~pfg#lmq`BX^Et8?5m1|XjCr%7VNcDG`}ZSp;~c@c9z7{mz zF++kHins_`)=wSU8*=oiLEiyOV$auZA+6*$m||=iU07HM0evrD9?Xdxc3LzISgrz?7y# zpd4fCGJvto6TN)c9U=ApkwEHdQvRmSxWz5Hn$q*fU<^sA+_zz)w9iGOcMFC}|CJ;v zU%r6V+x-p8!hgC+&*lH_p#Q6d{r}I#@qTUa?2`@D%LgmWVnAJ< z0_Td|$g;Bqj+)}vuXhF*s0;0@joV_wnIA~^s=WhiP<3M%=MlSgv!Fdv3FL$4{=MzhwFrD3uNYPO&$f+rOOGaaY; ziW}EU04}i|$TPsGmpNYtZugq&0OCrupUqwGnY;tWIX|ZTh-JGerh(gR!;SUg`!%`oA<8`m~kaTrT#0qx$X|_ zH{n#@fr+&|vwLtH=|Nj!(Y+Dxa%IR@I$xe1$^@Ps?M|!&ePR{9f*onqmyKzQ1jSgE;`use{$p!2Y%Mb6q?RB>Tt4(KlKqja>PusB0%)J z6#}8$4=dVj>vb`t>Xf2jhIF76G*{z2RQ8KJ!ov3aig4X_f5$ykGglm-;$e|t%)T`r zohN4&dW_hhbT zPUZpq%@0eJ8;UyFUolCY$>1>WzUHpXrWmUZb_VfF`(awmpkg=Nl(j(@IrTA+?FZ{I zNpm>+#4?oC)Yj#{9u^Au_T84U5mikVXG!sC))&Ad;Z~jF@l1{4;Xxe;Sk3cwK<~uupST5Ae&KLEcq&`vphn?G}91Ud)<4zfoCj%K#1h)hB z)!F`b@Dm|-6meuKH@(_cAcniHcET7xnrJf+V$4@Af=HNN?&+n&4R; z<#SmfioBK`C=<@BTWKB#(j_y27?c4)lbbLU&cwuYCq}@&t9su~*?H9xELQa(Ay>J7 z-!(_lpMq)aZmo(oMx5m?^`y$y;|Euk>gF|Ti15MTWOAE!F=uPaFC5HhB7sOxF+DlD zhEpMyeRLBEW|IsBH>>M6W}mIk2X=zp;{|GJoCwN_gO^ty9AsTQ1mI8W^sxX?!0A(X)x&~TLS9RM>Sj!bc1 z`*!}Krd}W&Kd=klGe1}$;QCjn3%u$T$4b`b1mFPBb9L^C)^sJ`-vw0AXD+yuYm!%R z0(L{6OzIVaq~AN?OgKLpfYrb} z0{w}TC*hP%Ta9$%*EaYESx29Zw%QeQ+_0*K6D~i(`3DU@pQdSM&z{IOP;2_N@D=mCZA3TKA|IT4^MSjjK@F1n{ZQRy(yuj8?is z;u|}h4=6I|B|OzJ-4#0LjvPy^*m%)h*a9dT9zYt^Io23haU2ZuuN^9@xgz>H zYY~9CAl2oM}j2(qy?L;SQ&@0rlfexYh#8iXw zB6l=N2-z7dNybVCc!DFpkEBm#cxLbPTE{v4^?loc)1W#{@;0iv%jhYjwzE$E)ML0V zkajkE`5IB+i709&BeGj2wijoxnAckRr{w-^pd>7`%kcy|djx73F{CkwJ2LpUz);cg zry5vX*|9&VQKo)9V(zs2vW|FnzHhU4qM?u6Noq1Da@)Aq8MgRl=bZljlkQ$Ehsk?8 zCyM*r%k1sf+(v)uZ)~r8w>Zm)O<0=b>0F)R%b!;nAP<-GyDCc_IefUJ!W%42h$tl` zWM|CSn@p?lCfdzW>$;Klffl9IjE=%n&d`5rjK4CPKOeFjzAUIOh%RlkrI~jr2g;+9 z*Xr_=U2$>svU?!ymTc8aVTAq+iI$-8Y^3-5_xphiqDZjnb@0&F)}}w@W|lxJXqQGe zNc>S#-lXnNjU751zDTUCbM_dxO?PapSXgh% zV6E%e@`vI*nmon=lPdX)`4LxODZ6JxmpS3H8=grXiQ0W*XM>I#Kfk}Y)S-Zv=+X<8 zT#l8C^?*Gp;-g6fw_Z+1BAY3VQ@mG7vAg7A714c)C?2>8k|?7gR-UPrEzM9?rEq3= z5x>Q<70rkR%ZUZ=C8B$d5#uSwTWu^eajaLb>MtoWr21VTj&P2En&@b?*f-mjOh%fI z&%Pia;s62bR*R)o|IN+?x6TUQaTJ%qwnx#9P@kk@j0{p%wzIX^B|>$D1?x?!A~>D) zbV%FVbxLNQ@jYThOMjZA+vnJ3;3>Xwj>}+OCVq~U#&)nmBY4A`yLkby+|{)qC0+wb z6Z-bjVNCAu?0nzIUVEB~eW}+7=>zutquM9j<(xbH;CTru&@hOn>`nAuSA#7otwv(XwgAlwXnyD_s;1=Fjm6g&+s;VmAm(IRf*rBv)eOHvKrfXPKm009KAPn zw!x}5Z7~mDJVH*uTeP;WZ#zwTr^A^>NqaJe9z5}RAmAeQjX5MTs6kZce^DXuD6zIgs8Z-j=2-R3Q}k1NN}OL%Z+l;zY1xeD zLgCE2=>embE)HB!lfb0>&JQ&hNQy`1*&-7)_IERV8D?V4Ucfp{j9&lBro%ueSMrI7 z)cR({Z`@h5AR1^|A!r&Qb6vg?JB`JaAsg#xy46KjN4|Hhw+ZYo;?9*7a5VY~ifrj|*@# z6pF6Uta<(Nl9Yv6PX<3G$r~^6F{XSd38In9Y1fI3>P$pWJl#7Q!#NqHcY6uL%6Go(v0)&`m|H4cs0RifK*RI_cdVD?DN~L|4O% zl;qx)btyNfkq+Q9?qOq8Qh`7t9C5AO-6d(V0C*b+1A~?{D+QLMvY?&q%+T1MEj_>^mwqY@(xbUIbc1D= zWhimW!o~zgfkW5D>xEpxKePJ+nS|ZN*Ap0gDzFgvUWcVHUn1s%nWwDWHw{<|lq9wy zMbK6}@Z1UAAN4?Boq-;XGoK%Uy=HB9#Y+Pw7zA?{IX97GY!w{H?X3titHD)WB4Ht? zdA+L3j!NeKtGTTMUOgo$hqku1EPLm1t9#wSEJ)bpvZju$!Wl<2VlV`oM(B ziHs|#Y<~xbt&A8`bxGd4)!gLaWQM^U1HHh@@s9BYAOAdQ1v8R%zqh<@JF??mpvdc* zpY21XBQ#1zEs?kMuE|uyrmnz5>&r0Xqr5nHrK@;ti+$n--$}=bu#b^Y%uch|IVCWw z0wVlQlVFKRGn(EU0y~m+chsqDtyl4eL0OphmRZ!BWbag;J+mlp_+g%jN@eU5M~~#d z38I!;r0Uz#$v^9+3M^N8gp(K=jj`#D~-}j%L*`3{)ot@dt;4qk+ zlXK3yJkRTSJ+IfovC+FAkF0s=l9kTzyyB4D2Xpu}e805*8sX2IelmmFSqhlUBXpO6 zlN=uWAtR$oWUa3tX`eW|(3_K`lAzA04J7}$`sxxBCYSxLMB)^PJq`pZV!4y{DSTM1aNlR+)Q)< zIeJeNPWZRiu1Q;97fk}RXf)TSII9_^YRI7OhIJ?Cb(%(281UAksbfYrf~y7D9AObU zds<83nfl#=x4gQ&IVKXKy^W`eW(Br-Y_EDXZ#v7!rK54n0!T{{e784R%)HG4-DA-Q zx?npht@E&+s~)M(zn|C(NUZ%xbq-t5)j-=+fJ%f}f6ovn@`4w3_1h(ZdX7NCU_l8c zkUc>w))gpFT^lB_Id;Ailpm`f&xqkfYb6ThMAXK_Tk0)zj%3_5yW#hiN~IPlM)`{@ zR;@GdE@7w{kfKX(sS6Xn{X7!5x^lM4eh7#L&p-{`uMvNj|Fav5WWbl{cfjLeY$riz zJnl=StFjwIbYBpp7^Ghq!MYoKr)LUyp%9}9^Tn`)GsUc;zF)bV*<89v8k)VbOXDW$ zl@)@5f)W=pa#Z7lN|y?LF7!%Xy6Sa7W|nyea%&lXZ}E^p>Z6~JQzgA5yYX$^KKvIi z-U2z!+sGmmB|}Mn1N#OfANKFGX%n0G%~FYERfCeGy`NGGLlq4Vx|@jA{0ndu3A^KI zHx)ECeD9yT_BH`rVnR=f*N(Eh9wtzH*7RUVcO3I-O9G5$-E+ds(AmY)={c`IzQ%E= z-#|8KM+bvj-g%0#@2i*3#V3oXvyUupt;BB2Hs|d^V|>|8?~lDitWR9FWJXm)*^s-x z@JGWSxieucG1J+C))hn3Dcl;CyW+?WA+zclbsr5LQEdh%;Wo`u$n9@$x~2U?`tF8r z?a0k3pntEYOhR1!Uf~u+Mh5U67HEN zQPeG5m&QMiUIX>Wi!{pjZtGNtk9aiQSZ35M(URyRK<3lmyk9Clh+16BADLXj5cDiE zyJ5ki*pP>%8)F-j=hj7@R#dA4pG_J6@_4Xdyt!XN^GGe!Q*;OCt@o$nwb^psPt`bU zT|i8|oAq-!DwxIX-%Y28ZI1irl?5#S$dESYmYa~s8D~)?yiIygkvs2Ik`kg%>YF`P zWjoH%M{qI{7msK6U;lXjPd}@+y40_l3NBMQPcutD+#Njo=!E!tCEgy5{yLT0m0US#n1C|3)oxTA*5RlD2VS@zY ztiUvoZ(8oU)n8x$^e$~a*A57)WhRAhsV^QTNs($>h=i_JJItjn793-UYeD^Nnv}?H zR$>e|0B+2M5JslvccRg7j-~-$LhK zbW%RKa=Sg{R(-q#=INA zUR`SWrTL}n;#ShZTFnDMuF%+J zKAV=b6u%_#_*mE)9Ra8bMMN?w_FR{G>ev0)AjZQpm-S8;9Z26on6YoZ8I!x4>tU)U z5W!fgsBE;qfV))czyZ?8yRNeL3L(lS!i7d;dy8!y&poN(bN3#mQ)|z9V7*S`i_oQW zEm~H5dD5@Z?P_CRMlv<>d?dH#Cc90!V!B^*`)}HiQ$4&Pi4EKwi=ymN*=5oaj(u(M zY1>|kck7-0^i}nMynF`Nl*j6ZRG)}VmpIlB(oo`vIT`@YW53Wl8n{sKR zKbu?PRah8ac=t%12niINGsQ}buL3%R7^8?!?EQ!^cjZ^k`z=;^QA^yU)q zi0u!Ln&=xvWBM4(gDk@PlEm6|qNt>vFoAVkdC!u=FMjOoV*&l~a8bYwuZ}xo>as8t zKMaT6RQ^hmh+8F-xIHKy<_?($l%UQFywT#$^dzySFmyFfgV2KvDupA#?UR7XmVi0F z!6ap59Ka2*SppW@w@dv_ZI4yt^(tWYpY2|2THV6pPRG&tSYyh zBKY$BQ2ANG_imm&&>9qsbc@q(Kc^Io(cu~CRx25c!p_9DTS|}fD<5x0fz&I@-V3f+ zMXr3DG7$NXG>eSd{t$N=H0YLHd%+}X|4_#y$fz+}lX-hOm95UtVdtu8a+p~{SCriW z8lkl>E)EYLrX$e`J-Jb%c^`AfJh15&v$)e6j!x=~ziI6m`q1*b<@ukZRXzBzyH;c9F!THo0pC}4CONlvs9+8*gP8+ zf5|JeKYKc;-phLQYunh#M=wk42Rg9y<^chXpqLjs zDLDS#uT#?EbQ&!C3%i?G>S(QZ*F;8z;Ov(Afv2=jR%-?(Cd{FkTPJ~^wd%>rX*y;> zbrm9?++x*7XYQ6&(dksCC2Ho${5vmKW&3C@!9^l%nvX0x)RrpmffNPCap*4uV;49X zfVCQ3YIIMdwZ?ZXa(j$6Y(6bBDa$162nMOk*bA72YimaP5Dt`zpU*tUj{33Yb!t|! z6#%aCv6`D6d!XCv{xxrb8PfRg273>OM?mMzFeAQ|)x1;nfd1*Is=rd%GJLQ0lgf(bx&$+`# zPqS4&`eg%09`PG-GW8_zrG}F$+16rA{C=iLoSA$>@Kpi3IHO9X#C0O1CP6zZ`^OrT zj=);`1Kqseg9LYEiOnlZZ(Yz8iMFYoMI-xQO@qw}YIMal^;uIw3Kx0;IEs znkvU$gr4mbAB3ESRNe%1KCF9X@+&L=^KKqL-ESm!9-}Le=nN<)#g$@Xc0Ecef=N^p^2F@^OHt#= zLZ)KL)Yq{M$JlAkir2j&9^Q7636|AE&_3FeZ0^wNpZ9bAntAH@RkVTNuEH)hg2cT2 zvVb5{!;Cw_-Gs8bA^;G-xy8E~1}JE}hpv5#x;ro3fjXr9Y4mav~TFg#GxPg_*e&3AuC2OU;Din!RVwhqX+HbQA{fc@43h@zB%QV0}0F z*9VLlzT`--ozyADw6+0zE}m4 zO$e=~Ot^`l&F7e15-lqz%z;q<;M+~P-mEAY?YYT4L@Iz$ z=8M7p0%6;4wcwdioqGh*q7DT_hZgf9k>!?*`t3o|aE$=f3^<7W(_GcFn>7PJU2Ufb zlU1%g-3#7JS(J%NMx*8+(0&xKg+vQbN&4`zE@Tac?z;>4qsNP!c!Sr7n@Pf(QV5Wa z+uy=DC=YAq)psn25=?$MC3LYOF^`tLz_$OA|9w<8$_?m#H>CWm0v=E3`r(h_m#Ob${3{`IB(AooS8%-V|wq2H?m^ZdvQd7#s; z-49{e3jP}1g_~dA-O`IuIKiAU&r)KPvw1RYdnkc2H)&|p;Ee29&-^y$(Jf&Wq%sl@ zv?e@ii~@Lx{Ow9$mFv&$;Y!~km^AmI9-f)m!35P#?T~gRll+Ls+U4e zuC!<^4M?pBq!J>g_00oKk@O!Vtn2^+CS-)7I&>G;U)Jm0juSx1a({%?K-i1McO!8e zJK>FsCoT-`KH%H)EjY;{4G!Rah=du}{gljP$%(9&VL=-ziMu;HXV1sW?%<5iO!R?x zCZbo)XV!SH$#1i7QQ%m!Vm*2$=jruVt=MSjYw?@<$>!(M;mJcTLsd2mKnTBEw$o}7 zm0<*EJbBRf>&?|xAsg3W_CH)(x?iu}Z6Lkj9Tzv5X}={Th0fOmS!A+xr4Q7vV; z-g)6k*4rcS5O4_`%+A^N;VADF&rLG?;-!H2YU z$FwN@@4x?sGL%J$Y0U#T<^XYAv1)yPhB@|Q0)X4v$sA&0;GVuPVORWq5 zY34=X!*)8MuTO*Ao1PG&8iQc=f?sr!0Qt=i*mfU=iK5>Td(IppN!UZXga+3>lAMFz zA)&s{gF`#J5%@y~4^~L)?*|30Yj(3GnxTQgZ^lf?l-+q)Uh4E#h@`LiaAxffT3XeD zbFaePjYCA9-IiFI+El3Zz>XR&Cv&eI@8t?FSb^(8)1S@sjYY%UD3eq zY%o5xW&t%-0e z{=Iv(n0KNw+KzvoJlRXz$5qEV>AfGc`FX-p=bOP8W|xlND?I-y{*m3CI~}%^(Aa_f zI_?UVuuVWx<{DgNyS;?QCA^RPv1QZXg58DsC{6ZWkeHpjQzRA1*YImIMLwuK{=4dy z_^vLb6fUxfi}Knut*ILef;BE_twpiE4yWc^L*3q8mtU_MlLO732_Ok&YmgRozJk^N}?qg=etl1lfXshlRW z25&XB{fF+knRm7% zp>-Omlee`Ujl2Hi4`Kb+2mxmVqdnV|st=~@=h)1|ae-051Ok`$IY5M=&*2)vQo-A# z`*?3HKVX#%xOV{BuGkshSXqoPq(IsF3aHM0oxh-)uW2Ugpr>D;#d*BJfaZtC>gkcS zwGsQ85wE%BQ>-^#m8NlVx=`kqz?M%LHH7FD(VWmYhH z;sH4Gpi{%=QYbRpcc<-AJgq6(4hX2(Kx(YIg^z?9E(o6xMz`0>jh=%POO?q)boB^P zGU!(}!hlLozcmmQ9*sh^vTbz7LN#=_LR{r!=vg+sR*My=yU%?uo0N22e*bp(xTHv> zUa2CZD>o01GA%u8PkU^h&h~;zp=m=dE`BVscI)}#G$M8Sg zz7IaoDPt_duFkmWyE|XO&|7h$<s5~=@4zi8^ACWE+fdlCBefUgBsGXjg5(D}^q=6^Xh9Dny8 zp&B^Of9cck>;D%Sj>~Hshn@ZhJruYh0Fp1l0hd2j zmI^8zSB(NVtCLwywpv9MdxzdtRu<#r`#M8fr9AlvKXlIvCC>iMFEK$EXx6Hq3 z(W;MH5cX>G+jrTj2{C;FwlQC4-Zs#m4840&v19)LO|2HNj1%wk>;u1bjDPmq`3rM- zz+nD#Jv|g5waT;p_uM?y=G8A6|K;lfx0Qo2iYB`RBG>Ay?v%~w+v{)MX z($Rk-awI*!h~G&oF9O`l)!bHI#jIGHH0&daHrX;8se^4*M`%qR(D@)&SNgA_&UiSH zEmcKq+6|=YMe|1-|L~%rVTpXJ_@kFCG)zOO{xO&8?}Ib`Cg`?+?_JiP+gmU0NwA!} zk`K&nPrHv}Kb08KWR+V8JKO+%4!w&FCnoT7;z(JyW0A`v z{RKDnsu91BX*+*^P3u-Ad4bLbC3FcBiw;~4W#z=w41;lY>7l; zHUFi(cj;T?qvJtf#e8=Dg3%8|X(9m>?Akd<Kc_wS1BKUds7dLXnjmX7oR+^N4 zH~RrM)&9{}>!?op<21L*qR&gk1{OcMk6c9XzxOiXN;N}cXh6A3WE7Zg#C^g)Ez~wXqNdnrj7%mp*K%u57L1!SZ4W_&%mE0rr+gZ0RW|r@2h%JyUQ6_ zNs{Ed7EKrb+y=5tf``1>nNr}Y9XC;+dFpcc5=sMziA-Vvm1RMzQ#@+EE{`+fp*sv# z!(XqxC24-B{OJ30yJW=ss;n#s-Dmn%eBzG`JHwr2IwFYT}ORpS5YCs!5Vt`Yl;J`i_ zA2F~B%*Sfv+O>9#9XoMU$H2h$-y`^t0vsPPT*3#`vB9kn%1S*+Q+s;L@s2EXS-lU7 zz7p2*t51dsdzI)cI{L*>xTAD3Mk4Fp_uQTwQ@Gf44yFf1$OfrE(>F0+c zH-(0u7Dc!O{-8#wvVjVUkt|-myI}&&-iGw_MG)L3sTMSS$D~2mqbGSFuD?#37S@H5 zEr+=C{5#4I{5J~#Ve?x2ZbmQD|M{jOz?x9N&dhJ3t@Lqu3t4nQ_7vz>hw|_iN0J&R z#p13r#8?>y8#Ue&6cmgBvo2?yfxV~&PTAW~bb~~ohW#Tp;#jUK`X{;1v;DK?w zaIDB!UwoqXmDD!u*fyJxb%zb3!FYY{y>`I(MbQbN~oJtS1}o^qnS-G3OwMPpY!NDqUO( zkGlx$xKKqNOW*5Mq2y->1k?Gpo-9z@(*+P|AgWP;RkAu(BCO8T-9ArB3g(4a@kBA7 zagE(tZprJYD1DIh^K|-rGez!$d4$lQf$UQ-~-s zi#dMJ_9BQc%)QAvhGxihC@si~=Su?iw^?b|Io3d#ZV^=n+an7n<`p%<_WCeu=Wq+%qB(mx}Fg^8N%$|Dz#S8K;2z5b$;B$FS zS*5+T*q{J-w*A19hQ$97)VxjcCTBAIem=6F_<7{;Ve7qs1L?IlU}z}bR{zWY(zr*? zoAtIUfhJ4@!}(z{zl_`uaPepfWALJ~j&UkO5=&WrzPbL9R(PG34(9UQf1YRw_&7xk}p z2!}S*cOi8CWA9QrQ^DL;vx&`L>3kSxUp4fV+kI&(8;YczNWUFOcVRjF)8~b$BA9tk zrR1S;$FK>S>wLXxIp+7+HbIUdRK%_KEa;89*HDd1J|>B*bUK0`liQzugpwL^D=OHk zPJ=MWC+gXD5o#LO`l?fg=SdxD#@*6mikr{~nmbuC_%M*#$)e^*Z3<}enH_z`(Ihjk z7P$ZUa*08i5UyMrk4D$~G;`yCV{2N`yrlw;?p+%)UQ6S;8kLR?kuvN zX|#AlTOn6-SIVoNU#J926|Ab(ZrfX_q31{FdcM3`vP3vf6|-IQb$ibd3jJYzeL%PT z0@ss=zy)HSSd%vgB+0`Vh{9(}Z1+p3c>fOPyX>g?%`s*tEe6%nfefUQ}|lw6qX9#ksE)1wCWn-^sH< z12I{ zf~AmC?yJA zGsmPXo(#oe1V3^zvTNmrLzksxRW5fKFsTV8f5ygvF!iM9f%L1Zb(b7KSD;Y!RYIi~ z&O7j?jEt|0*MeojZbwKhy9yU@omKJ(e2yrdE% z>FtO{h%n9u`G}RsA=;DnJyFfJdpqpm9{fg?1~UE|uKrDp2AF`C90iE@*7l6LeQ#~X zuV1G{JZ2pEsAM0DX7=-Y+1!3|Mn4F!g&ijm#&tmj-fU&X&2K*E5tbl;`DUKsWzS4_ zc3?h=zqN(awm)Ay;s|PhsaW4E=z;j?H$7+%fUOzScZ>rig*o5s;RIB@%s2f(*D~KN zaw3TOd3uKW<(0)irmwKF7F0$DOUKfAv}HA+pku})t+{$j23q=lU9+DEvZ!;wH+b%P zIUx%#=*;%L0BYk1V)j!-7-SY48k%ZfY8d?o1d$Xy_jfn1UA>_RVSABY=^*nY*Hz${ zwMMezHNq0|2{JxY1t~o}dE}Yo*l=>Sld(fv(Q}>hDcz2*k8Ji833CpKapl3!heG zClDr?l!qH^(4d0mve;LI{|p!?EqKLFx!$aF**_4t{P6@(>&;xkS#y3o zobY}92Tf!9?CI4U$CGX-j5WwP0z-k&bzgZNkZ8uq9@XCbH|PQcqV@tMAVR4_NtuP>LJNa= z97iBXV*$yl%a?ey(-MmLY>lLyqUjz6Y7akOy$FmcgVyMDSdp~l7IKr z?E8##%p0$hM1jGoSN8bunAdDcp(Z9J{U-}LWm zO++^NMQ{ISIP#|tv;Bm!P$4y0AZU6LN=(DavD_Nxw~U~DxF9Ar))bqs;YD^&UPJx+mgO;AE5dmuawy{bJdd96_8f^ z-aKI<}Da1gKQfxX+|~>B?7$Tjy9TI1yXI&B0N7qRbueb4&kJ;7T*$pkt~-n^C~5 z(U#Xn{14M1cdLl$SZ;Uq%-PMRhRG$=?!zNB6KQ+fDKb6?c>`O&HhbaJsWhv{awT8E zNLXwzfp1p^7Z^9ZU1p4Ls2Cm3l=sV&g#{_AH5@-L{Phhj4deD`v}oG5Y~{};i*e2n zkizHg8dp2^g^-!Y)Hj%zsVL}8nf<6$e@#0#99Q3I9Xn*gv6KLyQpqH@`UM;e?LqX! zTjHKg*Ov=8prr_fpzMZvt2PIV4LvGte?Imi`Xx9wG=+5}Gz7RIjmF2^?AJhruHDMm zv(lk=!||&`iAkL#&V%T&P!_bf^@?}XgIj2o$>LK#C@@;qO&rKe+q`6jv#c|Bw%i+jfnir`GL?VF`)5T6f{b z%QtUMVV9e)>lOFI!1KbJ)__HUvqvIaTw0)k%JoMyjQ5}A-Kgt&*M&Nw-+zGu)Vr zmB|S)XFsLhPcP-{pnuSlSVh-AiE#$gS+Q+dIhwPFTZE1WfEf#!{f5Ehgx?4!BN7Ed z)v7s!-d}feye)7N67VPLckbN5MA02R;R-&=O@+A{s6t>&xk0ysvTDL!ZeE(7`WnXY zhtLL~d&{LD`ERl~ZHE1hid|loJVsySPz>`?1(yE z_JHM5l?48{A`8_x-o!iygn9h+`%gbu#gYCsi`IQq6E)JMkg|VsPhDmHp-aZs_;{@i zwu8Suz5K;+>YT~ZWB+VQA$5OkKy6kKKJn(x34mJDGpxfd_UeHbUbtGXLUXAZGlNg6F z*Ha`cq;dn@M|eP`OsqSP8|_ALyNu;^u?A6LSRpUZ0junp0j*<<~2g#^gkqC^CebrjjMiq1*U!Ui>Y;N8< z8JC;jlO_!ggNa7fj=2!c*?miYgmoG_ex8>L>_%^$NM1!g_gsk-qUETC#9FINl`8qzH}b(Sgejk5O^H z%3Gy69C4{x-;%HgUPNe6LWEMoJk{-r=;N)wyn?1Vy_N>$vlJq~{c7<*B-?APgaC|q zP+`+482|j>^@3Z^cChZ9MgtM8$ItH-;PhFd%7Sl2~RiaO%kMxZ{4z^+kYIr zv#Wi2k09>YQN4TMsg%dai-4cT&ant{@3WWiuf?Ia#tjEMj@&eH(cdx)MT3V&_*(O{ z*aGAjL#F8ai48gx)+#ouMQBC1ew;U6P1SS#aPq58=X2-J+gGqSc!l|SS0TWSJv{po zW4Xt?0tQ=o)k{aa;{%~{;(P7iJI`oI+Dc0Afvj4>VD?0>TYBxSegDx|bd*N*qEyqb zU(PoJoLXKFU@gfp{R@{y!cMRU&D~C(2LJK4H!s8DzgNbw1!n7Vsits%S7d9X1m@ZV z9~p^Mm!#YHRkAaASOZ#2qpxf|B`^hs&;*1XAgAK8KJ2?YNCtVA_a> zzOn1i$E4YhC{$Q?$8iv&Zl))g2dTm2O~0mpQfn_@+jD2@Pk?<|-sLyx5t$0jN(wpV zXO*?9mNRLUb7@=0QT8Z)>&yTu0h*on4OB&0s^ek zu1dz$&J~7B#2WwHq!$B@LIvg%W((iSVpm4rmB}rj;)-+|^wiodd&gy{T29bcGu{D% zCl}ZF=(BYvX2_XJ#wSudR)j?DdX;oP7v|qv`6cPjp%u%u|G0Pwv=F1CI}*p)X`Tqx zs3B(GGStOBisSMu)5L1VX(;{THJi;s zHsTof_JWY49v%(Knwn#b=l5HF=1;`v<$g6~5~etSGuK4z4_BZ^4L3=Vzcs?!nQo?| z{{H)5g$@bs5D;#wX1aFof`Y&=`G&glV~&b+?0bpEkv;S@r9Z0g4l030!g$d>wVBP^ zyn)TfltI9>qu<*fGi2KB6EH|}07AmR+!97_lVo~UN5bI%&z042A%)bT#lc`GGzxKE zOs`Z35+0q2_jG_LR)lS~Jyuc1ELit$=m|~Yuqdl@1*l9i3C}n*ZT<4VdOUk}Zhbo8 zF5*D~mB>yK_4Qf)Vs2zBtrbhqhpj7U+Qrz#&ge1}Z+4Vhc$bXZBr!H+ixc4RFK2e|vWh&0Cw@Vrk7^rfv zX&~mnCz!-M)6)Vs3PftOp>YlL*=(yEHswMI(i?MknfCiGKnYjM6m^B~<#C4I5f}0| z`ibDYh9&1;?@RFMw`C$&scP{W0LF6*8}fP|(d~SS)7OeRKI14ThzD%(5_sG2#VXem zN@IXStdk~f*zjThSBok#Tp^jzYLDSO%ffOM)Oz#cm}#D_Uo*H!>`J^|ce%QCqBf%~ z>y0FnuL;;F9An@q{?QQ!I)sE}nY`|l9$4u-LHrh+1Pg$B0<4qdPQA@7AuoQ@ytXSxs=Hw12 zUstX)IP1IqE|OL1QbmH{0V|cF$1SUjgBe z%D)TEsXbuOlAUbPpTK7v5=MXEiHHrzAt9<;(IY)}P^a<%C07&_MP$f2il((s5Pk2Y zJwo;fR}OW9+4ck)gh>z4F-v}s&jIm4X85qW+V9HBQxg|ZLcqRT08=A%{JCE(ug{CJ zF|?rq_1_+%A+h{N%j;Gt12eOQbfh#>0SF$8;N?J7A)a>^XHcpq!)uJArKuJEuNqjL z*}%7J*h!044ci~-k0gEe0t(5Xs6bvyB2$`c#`S@x=qkslau$CjN_wt$^&pKqHx|m- zALwp$@zT)L9WR3xV>ZxW+z}g@se|68Lg*%n$p$T1hkeW;Nyc+ljb@k0Fr6TneA2WhwmEXg;Xj zyj~sq`v~9ubMN6fG{7H)3$FhztsvEv|DkKf@IPeJj{o;Y0ZrlX|By{PsIH2vz&1{x zZ7ub8L#w(EQtcyQ$1L(15}>~56U$mAjQi8KLZU$P3siE2xFo!}TD5#`BUMTdq)Iu( zKXhuOA}v~+lh*q(q+^V4Sfo0g&)-GNKrBxpCC^b#smK2DXI9O88&kAT!_hTK=T*lL z9kVA)K4Cc|J-~3KBZMvN;61*LF=B7P`7L}EN_nkHk*V!v{iaBXwv7cTW!9;Bf7vKD zzqDi(4`ue2nDq-_`IVNCV0d1&_ogT5yxi}zviy72hQ|%s)louiC?h9J1^Z2b zkMXp@1Ak$mbowCp=153|Nb+1liG)CQb=d|^Q%ffxPyrSq&i<;A8Ke-T?o0VyNi{7O zu9o6FmaN9Ly~UBrqjbOWpI7ky_uZB%uj68-CSzL*gsY^^AZK2`gw$0VEPWh(hV9%? zjpO3217&TFGmdZp8}B-m%X-1^1^z5ynj(T6yH@5um)7c?r|?j2q7mA}Mh`-nm$kj7 z5eNa>+W-vx%03RqrfL26mP4&wJpu!&Z9y1i8@ib1QZ*kB(2zX2Vu@QGL6~u&WSJ|s zPQ#&^V;31zzF8U0g!u|C%vR;}r zNn*&@2sshQzgJQ#S#6_w$4s32)vKoSHh-*78UF+ym8Vi4OWeBQQs5i2d5}|Nppg6_ zHT?GpW3BENE*v8s|%f5eM>oTEybnJstHX=$d8{l`4WxAAQ&&T*9D&Heja zSN?l0RYxD+|J@nnsZ>v@$7}eWzh6&T?)~roL-T)XPW>+_Le~TTOD(LH1`$2{_hs@w zqzL~nPuc(f@H9{V57)E*_ixTc4Rr>Z0+4D|16AgDjyV3ngpPQ=*gpA#(Z?Nu2e{(4 z64Rc2giM(7bfdBS^zm-!xxl@@IiP0!8hVNo;C(bSdO(aWKHkG%DU_^Rqpp_N>4Wu( z_VHtWi8=o?{DvX#c~y0W%Tm0i)pu}v6h$&&K(8S$4VsDE+k-#}D$lODt9PJC(2q1! z4XUztLTXMi`kH7|2b#rr`t&Ix3+0w%6Q7_`ZD_mcq79ZMiGwDCZ9 z=eL-g>r>NKy%pV+acHzlWZX*%ZEa-3mjDOSx<5qk(Wh2cpef5Ygk%0AE$IOWk9*WB zohDwVX*;j3)~509rmo_ZDEP0X&Ymt`Xm$Y|1+;uvLZRE_5&NFT0^G*#Dz1ATh@h{V zyw&#cYofnT?7t~L*U%%U6azJY%#Pb}?!~FIR{=YquJUUmK zM>wYJ%nv+WlMuZWOiH1eyrGb!(1bA`Fj5REdim0RHM@*|LLge3ACiPV`5T)1GD(7jSdiMT# z9+M5YeFqMwdg>HymFveTl)j;fqp5DA+p)U_bcJ<&!M^ zAjNPNQuXw76;S?p&{SVV#LPl(^Kh+Q5zHN49S9*Gn3$wgJB<`n5yW^v={bUFP`k;m zO85Q41CxOOT^Q}mnjfQ|DB9Y3a#;VQPAnH47%jXU37V;}>9(6M5B8i=vh4ML{`~Ce z)7gGr4(*mgR?9~(UpdJpS&U04wcSi>!g@gRsISw{V^OEqxqzLJtW%GtyPBmEcMVF2 z>cPAQnaj&E9m#lOpNTQ0&SX)+aeuIEM<9iN{`m~uC3j$>JIBJ352{BJJIw>1)_^C{vF-GbA_KYjoww|8AIFF_Pss6u=F(N>pbdOM?3kQ_a8J*A|ZW1)X}6qZ(fxSh=Ph1GX6fO zoXa*2I$i{}Do<7(YjF*4)-$r}?Nk`~Z?b($6t>Z=b-kopsAEPyI??-R5%&9U(2^1t z{Vx6B;ngDV5+-lFl)DHRj;2O)450N}b7dR^upAs5wiB2`Fk-@BKFw}ZFK$ltJC|7HzxbdeN(w47E0)V+tlaKpr^BF? zLFL1cPd>yRDto|VA9nV@<@n*6V4=?LS=*5%%vwDxc_{nT+S(_9doAKgyHs}Xnw5ao zjigo8UQ2$pB$0lfEZLd{Gb3u(;_T~)rd-2{S%=ZuKTo2A95;2Cn>L;4r#qAJGuyyp zUwFkWr4$o#f>iWWD2F@*o3a;*B4Wl@)2tWv+d~;uFc&S7YM!iD>$j;Vil{$&q{~qxxqY8n?bk%TzRFoMYd3FR9dKw5ejWhkvBK6bYG&S~{Fe(r*^!f?Uh!9;+} zJ;<>0s1;I^-z5nysMJ!@!n< z?t+L3kj%=cm>)kbk}1o4(C=~ObF-qI+ooTN37*5QzwHslX~qQVa9jg^+dI3W0b933 zJ3egfb>+8KUZMF=VFD3HlN?+2x&2~wJjFb4JCS7PR`d4RMqoKOCEI>@+Rcg$#Wyr( z>PzN0-%XYASErZhRj*GxbYZ{mYQs`6A$45gehQT;AjU8|%47jJ6nR66eI$ZCN@iGP zmx5VZ{sr!6-V06il%WWXflqhh@Z4x%(_H#*L}2YB)h@e8E+`0h8qQJzCWFKi0K4Rj;C&o68!ZgFSH6Ml-(S#=~!}$=Oxp4Nkkc zS#xHauvA1j(WPnZx~>q$w&yZ-a>_D5&vL`VD~PNXxIN$__N?a4w!0YNYl^78oJLao z%wC>)at`!+k2znFkD;uy76;>t0bBO07&#h~ZPrA)@abh(L0XLg?*e({R`M+tc*qcM ziOJ5s4%Q>Y@yj5*9BYZ6el}fcx=T^CxspA471r;%6wSYVj;dB2P&#__szrsg?ebBl z@p8UCtDE3~J04J!Fs2aB0lWDF_=xo!b+UNLY9i=4QOwzM?b<)xa_j|+!ZY8UMwNQ! zg>A^~d7pcE2?OsgG;L}5WIcOASo)lCS8;u@BGn6+P9ilH8rD%I!uuOTz9$4sF~tdH zL1Vek3>x(4bdMhwjc<))Mhk=lCtEWvoi!|gD-|8)Vr2hIQz3$RKzy9xl$+)i#lR)8 z>8hF6e!R*u6Fc9y7+p*g(AY2eRKk)yu|A?^tgEDam04NpAdPe@6mRFrY)j1TTktgI z5;vU4b6F)Um{zmXyd^Ny+O@6cK1WgV7}$5S-b&?CSku9D<(MqqRxBC?^@&$i#VjHUv7haYi^Ag?_yUS(3tGQYk3YsF%#sa!WO@!RPmH@ z)YqWWyb+m8%J$#VOM`aym`e_90$acfN3WZjh!r5f@1^fHqYyrz=om*aw)s99PjD6hSM<0X4!{0h1d&HVL?72S< z9I!&JtTRRM!9*0*NOdp8N#r8bb~9Uo_u^hCNlcLble?F(rZ^#NCK& z)&n$Y7Td3Nky4Fkg{iRrpe49dBr}Pm*I3_2J*+%__wL=wM`J=xLzSOMuKadYOPqGp zNY_9z+16s$TcfI{Fzz^+y4Y^i>|G#B`N20?1!;%f#?o2^*@HWHMv*P(mNr*JA9CNO zUmvdD9G}5T#^z;Xlt{hPxP=jcY5or%KD?#?M83Xh?oqXhI-CS+Xz1rp**WH@O=MOe zJ0;d*f74n&+54-u6H1AUDH0TeG7BLk0*XV1NBr`&Q0?SYaL?mID z^U)!^+*5%80Xo@AkIt3X^R-E#YhIt_aooY7}JjNNpvot-O1BrlEUV;?Uh(V3)q<*j=6~1o&b{s>v;1Ez%ifYCe ztQ#>d=J(>OQHv17Q>>?MT5Q8wGEm~c9mj8~5;HYBTXd3j*ls6k!-W%<|23eLn=SYr z@|@i)Fgj!PRa1myk?KTfw2^EY?Wr9j!Nl2E97~wln+PI!k>__yL0wcp8T#@v@%4?C z6>L6`RedTAoK&3YGnYwY@Kc=Yc@?rj{`MW?w>zeJ(X6teHc z-245l$th139!rIcJBM}Qo@^V?Nob}&Jdj+ElE$o+z*Hc#vMUx6_l=66{Cv%%HhRU> zf0l?B^&jMb%2D+-T3w;)2gL>tfNj=*6+`HYK+L$G&7?P5`O9oc1t`Zd+7Q_4x_^G)h#cGo{E zS){_kZoLqh`E}4@_!P|#37{f5N|VM2tjTUA*J*XN@TIBQ$?X+DInAppsT|$};*4ueg=Pp|yzXlWXvJicX;llqZj@ZfwLujRW%hpq-!0$$xnJP2mzWAQ_Ht4wfyN z-EkTgY;WyNbHT7>;>`jdKm>;apU1XTOI^KT{q?FjhaUXF-uNs~RpK+cH&%~XTqZ+g zE5%Y%^5}Ctb}~O-IFbJ|a4U4xn8YF2OΠZG**ZM{YtcxF?u)ucf(mXIm@*Dh& zx74$igS(jUxq<@vSJugS8WOT3A91H_hBOVCprA3os_<5jhWE=-Owg&3G1A!Z7P{6k zQw8}uV~qr6V?ouM8~n}i#zfQ}ieuh(O3<`9XzY>18{zSwqK40cYrvke=P=i5(yRQZ zb93!Jm-Cvc*pEIlZ2SLK-g^c`nMKi}AS$B_Ff)iK(S%4=0g-G11j#w0A~{LUU_eo_ zM9GqxOoN0bqasbFB@0c{l0ySc4o!NyXXf5J_3GCB@&3Ge^}Z^rK)Szl&e>;&wb$aY zrDg6{7(Pr`pC5@6YK%9osrv+Zs4VXWg-H^6GTy{jK>M{kYF{8#I(n!98twaU0*GeTjbpF3OA37Yux5fHgs=C2|-xv!N(-LS|kQ=)7Se4J_x~LOWY5U+J`yLXfTLWv`!Q;8&w%pAP2h zch!HEnYfdPb?$G72o^62V?mwkK)QSNEbn0@SKm1W^?@6dRNJ1iw2TPbP|TUtRHiV1 z1@`9c+jAW#I-txpY5#GZ*og0Q`29vw7mq>K^U4@(YLEdgtysazIE?H@kPPmOtW8aI zr}bZmn&{MfLeUebpx=aK*dTET>x%P{ziLV}Zo z_=x-B5M&0>lOSnE0DI?-Qm^w=pLBxs5*-!2vm9ZNKF0?KJwfF@+tP1>h&?^6W@vsy z%`RJa>$BT~tj1kW{O&D6S@NH83tw>Pxx_u~Ec{Zr^eH#C(UsfJypdePE30K;nWRaT zc9U!W>HA1UqB7xtyu?R8i%f3|3Jd=n$5*?~*8(J;4x(7so#Ex<;1B{eIiYOkYOD|g)7jev?0w-B6x-T$o;Qrc zLVI9HfQJJStbC)C`+f)`-^VTOq41II!S{e@P5k*guSFLmm{g^Q4JI4XE+_!==Oyf4 zUExnduf$$6QSxz=Zg8HK$;3H{Tt!>F>V^p^{_~Y>ML>bLOoE4(4miSg^Q8Vc80{YT_N0V;?fDO>7&16) znpekWpmuaLbd znoFa&ynb7HPq$vLmYE?F5=}uK-qY^-MNEFu{<_3^ylSmGms<$gBQKpq6FvlS{DAzW z$XjWr_%%cjFTv?lf%jsk#P7YD5!Uh)FD$KkvtE99W&)fY%MK&0Gn>sJn&=5K$R&T< z(51ft)$m*4K7Vx87|(@Y0+Xu=WeJUQ0=%r#*D^3zl?m@5V@_i|o)%wU2YlsoF0vIn zzxN6GM#3ok>Xf79cil`8w*^PEGi);zcEd_wan?3Bdwx7IoweG%xzHYa6xw30Xt|s} zws;ouZp%_*74vtNo`Y^(*&8lcME-Yg`RF z4j7H9qb#};9bLA@*rqCp*_$V5hn>Fl{poRtHROv^O8HcetD3IX*CEJ4OZ(;|J>F^r zStdLkj}OJ7nfOf*;2Gr|^rl7n6s$&DX=BscHg z*qXErj{I>`KrR@F!~jKkno?BcE(PNdb7wtm0v`A31RI8*tIZ;unb1%sQQ|Z6B@LaT2x|-j;Z%7f0!}0@%JzkOJz4vlA8?@T~^4`X=+w|-^WKQ}4u@^3~|z zv;0r5yqWDv;Bh2k1Ainj+{Wc4s|))2`T|o4TYqU&p8<%X`llS$uCjex#j=vC$U*m#^K0{{AU(1(OLen8;*!q& zFr34vJt4i*h_SY}gVZf#NoyUl(Pm$o)SzRtQSqG1xg&6YxV~Oj!&8{>@BbQ8LKSR>YGcVlT?=SD8C&3?)Qa5$~MBY?5rKL>_QJkE= zq8L`&PSs|}hU7wO-+Q)bSznRa0CVu!(QzdDI{WSC-#v{!KWpDBjCNoWnCg6EU&(d0 zo9EV6qUH$4a2s2AlF(G;Pi4~%X_7%z-|Z#WsLi5FIiGLSfj%C$kg{lPZE4HVud6iI zqHkUua}=888T-mVRa|G^Hgb5t&9E#vM!JY1JRrm+)N&o&Y#Lvjf6hh_8UlM`X>C}$ zoh7Wo?s%XUS2?X%J?~QuSq-yJ2Zel1iPvSnn@7RxFcelz(6j5Enin&OOO8wZm}a~= z0Z!mNSdQy)MaZePcA~mNeha8P4&m_DXxiRsI^@5;`Ra1>Q z$!#GbQN}Sbj(^wq^~sIyHO9IM8y!)0;z+$N)*}#WOawe5`ayhDf3A8v(QW)2ceEQv zm?Yx+_wQQ7#^>W1kXepe@Unz$l^hiD2^y7uAFmO0xbn<~V=&r0RVt~%KQ`HOG`A44(!oi0Z!dl0=2F}3R4wFD=`Yo;6x8)cd#!Y-JM~6e zx(n|=17<%9(|FFDPiD)C7qF()S(2$MsDPh5r|KS$|HZGrhQg5{BraKH6N>ZicVVXl zX$KuBjzN#*!&(9din+45d|&_%4)9v1_kP60gq`X#GziQUAX|PKjg#{)@i9SvFTV;~ z(en|HVy+=1z1FT>$@+Y({kCI+5a@JOKq8=2(su_{nfQJ4SscpJ zPxlKt%b~oup8fl9qV>Z>k0DR~9jf65# z0cx{HeJIq(K8>Cv6MOWdWaiR!w%hG)+4jvW0SqihNqHx-Z-3u+U_ER|cw;@Q&8h{m z(9l#;pk3e-+tTq!qRGSdTVS5p)-VtC3FZjP?!taMrT=cyp#~nG;5anZwyNPn)B$KEO zbt2_RiBpJy%vNN3EX|K*;K^`w3`DCY=NB9q4~&X6R!uX0>yCu=IPCqTR1bKZ?pHWZ zGlS#sFvZl%1cCsy>Gqewp^}K~j>8-1Yl4t*fvSdM3@>k^e5zgq*P)~~q=PkbRM@nD zOG(zW>Jxx_$sSmkoAmtl%Z{sc8Dt}tT{TcOM@3CL#GopfaqrD(<~E6FaOrK$lJ4lw!oz}|m;3?T$&bI&JX{5^U1K=%xN zOF>(o@``SQ2*(i#dr1Ik{U$g*%sV3<76(^wikICGC|KNb6bFB(GlK zBsVAr7C~U>WAwB^IZ3|U0(Ww9@JsdjS%f&lc~;h}Xs!&S!(q_&{`BL=AHXec8YS%6 zs+%|Hwem7f?9j{@mLNG_v3cbB(Y35}4V}dC(|8+|u)Z-)?O0P4=n+TF0qU&2HxzZv zo}K|zZx zarp^GxBO;Y;a${ZCgwWFbxU{tnW*Y6wsGfMS#Yqpy^^BX-nNV|{97!pbsk^_xu-`= z@jpa*S=NO#1;M)5|8-m@EG;7qty;R5DvfgeSw`}iQrOfJMf%Gydv#tT7S*`z?HiCpJ9 zm`#UN8ZEy@Dr}zFIClalAzvZ-ExPR#L9&&MIDK+ox^3lFiqqego=-Ixw^4C30(_?{ zDk)J_O)g7|hv!Vg-rsOwYbPw(OQwk;Bbw|t7kgycB+6AceSzk4!-?o2?A{rlug_~O zDQ=q9k6m4VRI|zD_3oVQJpcWh9C$1;ddz-rXJ@Ws{B{52W_O+m`8gp#Rf!H@&n&S? zpC7aI-cHHqL-`5-Hq%ec_Y-)?*9L09@gxI4i!Q$2)Au`zE&DKN$>qnIoLA1yBq+s- z`?mRgGGZ>iZL&W@RVZtnzl(5qXbhnL=ml*1fTXE_6>_?Jy0q6sP$ zC2HitRjTLypS9+!;#FxTKAX+hWz1M*Y@4?dIcfh0_R4`gm%ST>kkz*$g${dXKT=If zztr2;psG`Epsl<&OmGfuxE9Y(7lF+g+e_U%>&^G6ZuJxs1Zgw>2j z^N|BFyQwgU?l;0Ga{%o@oEH{;qk;U^Xuoa(?(uJhpgld!Z%=NyA0Rq1k{o3}3m zY$O9@Pv$4GA3?2NbJy9tPij#CTBF0|*oUBVLW@+a0qCDsU94A&UH^e0TnmvseTvT6 zV6jAxPu%OKqI@`ev;$pF{D9HRe@wRiG&k8~gEGoJhdBO!@oK+P$r*Y&3|H8bgXwp7 zZM;;4f(YTPoAo#7Tghov7O9%5LMVq$$t^pH-TE8R^ZY$G`17%w< zSi(zy9x6492ZG86AY>j3E2BGEIx|4081{SK(Cy!ue_*cy*)_$@pc_+vQ`J;8ErNH8 ztEta~47hbM6EkxL=A?fi05+-$?ck0r{)LNGQ9%yPnDXrN$-@T{L3NPzYI8ybL{ zEtp5QI18XYTA6Ym>+0)WYI5Ow%>RX%U3L2h1N?u7*8cZ1|KETCu2p#lojUct=0jv| zAj>Jma=n<3R!(kzzF>XwHy74aY|A6By3Vz=8kg=Mb)pkj=CbBCxG52yDw5O`p3)xg zXRiXU|5A@|&r@|@-&Yfpj(^m>OzO0VuB%^f#bO^>L_^p&>Duq}MS20~h2V>H6fb5w z@;pyzU))$@-8!1U!z9Q-Y8Y8M_6P`z>&s3UK*G2)*pFgNxvs1aWlW-bFa*6zDm z^5=EGyFdVjY-#4bcu^hd4Da6`cfJ5%-{=KU*vmoPHN-q3wL&EDA%t`=1*Zl$flJ{%cJ%x%51RdzI~h^N=%T^_$wk8y;i<>E%}A zNklD~Ny}quu_|FY&=+nIaLG3lz9+bSc^`##iGkl7bWyq6cec|1*F!ce2G8oNwBoDu z&y@{ZuQwg5;;uHqJuiEE6JE8+fA5lQRg$3Fihqu|vHD{4J8mJNHkP`zXtb!f+7Ph0 zE5`vr{tqzym?atmwXkT*go^9@62DyAHrt2G#4H*AcP9i|sHw-xp%WA(aYo1}LPG7Q zF{I-CvNH;4G{9$Lv9; z`mz7_?FSjk`h_tgQ*e$r%^Nlier6mrDMDZBo#oGFEdtQ*c0u8E(O-oN15M5~4=FzD z!|pTFZFo&#ZEX#>m&>CfRL7D9pgrq(X|%e!L7BztD|uEQF4Yhl&N`Ix-yQ-14Nb^_ zM`eBbq6m(U1t3jd1VQKG{=$GQ6umikG_p!k{VG$SjnCZM5Ey0pQ220>IASC61nf;J zz?^haIpx;(`#dVNDE zN5)>M5kk4;)IMK@mj46g@mlu+^NzDNL(2*Y$_OP#$FJXHDlEFh6?!%8v9Mc3T00~} zHiCc!Ktmq$r2%Kv<;LAh@*My%g+r3c>%{&RmbGigzVZs0>H74k;zCI9Pq~&-e35F2 zlyM^!cCCf8jf?5PTY%%2ek<>3UAczxQ(7>qNQL zm{woTQ6WQ1rWU8L9>9bE^-6J}+V-AF6oB)z>J0{x1wfS-j+Hx8jB3DQ70aW4Gbehy2dmY(Atgvhdp z+6_Wg!fj{ePx6L&QTr7%!RGHR*UL9ZOAGLcQCL0%WL}~O%f=AC!)@iuMW{Y%Ytzwb zJbY*_Vy9;c`6rK^u_-(~gofd4BS7po53QUB(v}UC4#S9C%^dJyYR|x?os;HJ5X`8F zjth;57>dg7sBoP(75BaZN}r5cR#uES7gZsEJ6tz5-(FJ`Kjk_8?L?7&tvVJ({4rRB z)h@Y1sZC1*!c-;fzMr!9_xF>ZXT2Dm-bfGv`jd2W&y?ARmgt4Y13dYE{Z*(#k0Y+= zOaiLSW*1sJKtRU^Xu+-na?c84Z2*yKu)VvTD>>$M28w$0qX9^E?>T=!!m(iY>*H|gl)GUjEN_&Y4(6a+LkTO?zn%cwI$H00e zOUu%)tEb19!^#CVp=+iKTkeA~Q2^rPZE}le4Rq{6!l2(Y0Qm!Iv78E!64B-_%un?v zDF6r$SbN>%8t=@v@+OA{_YU$)lneRByW7$0+SwJh#|joCisW~+fIM5eN{Ruhp5#A? z=rX7Rs3-taflw^}G-F4B;=_`W3Lgz$ZR>ZPY& zwCTx6Aot-HFC_zjjE0H7XUkp!JZC4|yT)`l4{xDOF1yd-xNt;-!!CdF;1gFpPXDr}Elh{({B~)&{ z?!I}nAryoqnYVkMI36GS-f==o4^&`{fwMf6CgX4eODpyCaHe>q^v>dAy zPoFMw6oBnxmduhtsZLZb$J98M>Is{-yB-K1S_TnfpfexZ<+arwXKNvgOdAzA{QGsI zHk9^qHul#s3w@OLfjIkYEtcFW3dP@`D`eb0X|3Y}-L}}X)|`B`bff&CVqlACF+d5$ z$^XugA?7k`3%Y)EGELyM1C1^hTLT?MYszvUG`O;}@9PI>7wX*wdDkD%matUJ0ZZ>| z+9pJQ;d?XPi%eOux+M1k=zpWe$@_4CfRXW3jF?#i1!@#`>|5hLZz7Y}LY0$Y@5pA!1f zX9YCXbH}}H`MHW>$1I!Yal9;#%OFK7>~~|-={5K8yYb89kM%~TUv^#j@x+aAh-i9N zcLh2FT;Wa{mZam6dU=#oLAS{tf)@sa!lD2qJFLV~IZK2zXFO?YP>c-~RL`h_R#`i~ zw9sIU*={7csovVbu@Aw&FtVu})aFShz@jIslHvnoAjFsUO3Q3uODe{!F8kE?l247o zhxP4XUEfktthfTmyifqyuzVfhapd0FDegvbU)=D)G5JHIwRq*;K|Ee{yW9$a4(n@> z7##XK&a3R{Dww)=rx4-Y+L)A*(yn0m!>EH*Qt>0lb;Q!sReirfEjY_IQ=aEVHp>k3%v}8C)JnLG5J~Y1@`z!!J?e+v7 zO`C!nVZ4Q6(pC?J0E5_^O=$fddh>K`n2%rQaZap*fel8ne z)BMNux{^iYaW(GU{j0~ELjcS=I3W|y+(9OJ#E8}u=#^&i&d{-m=TlOhWJ&2cHOV#K zrz+!KB}M2C2%^J)T>K1=enyo}kxAyCmo=h@kd|)oG(xG^LkS?zW-!0FwRI9Yodd(! zWG>>ARshJN@w{Xb&uqIia-d4Sh!t=X0H_osOzff#qpvCtK*M{Em+S=y2kg2M-FmyW z_FXU)x`0{-N|*CXKmRPooD6U%-wrtPOBHwXHaS7F9{}KaChe3Xr+q1*3)dAhJgH=V zr#f<-D3E>24Bo>GI7`UvN9g|e(%-iXO9y;D`>nvxmnMSnqz+28hLlYO`D$c zYNfh`1p+$1WheFKsc)9gW4jjIy@nJkHdFEVJl^6eBYuPL3twI;`R%vKZ1l)K3d1n* zdEUp>xN(bg?7ya$9)x`*)OivewlR14s9cRZEr#Cuc;j;a?8w;ZQ>WCsS0|&gj3qo0uF8k9NhoH@(?&OqM`D)-Q|qn0HwR9P zlv|gjCf?$FJTHklT**Ak=h;;5_$^Svb&~dA+EqOE!-s4T>0~)VQoaz>bAP)JVN}JF zCb>*j7KY68eL1Vfn@eeK?`*%>PGgdkIacq} z<{-GQ35nZ$pv$sd{HZRNGIT_8PiR@|-%B*{_S)hz_Aqvol^xK%cdsrrVf8_tH*H47 z#>l=dyj6P}Mvff0o*rhOy@#9i%LbXC*^MuDE}1={mB9|~EjXd+FdTos;WX^fyi=^-L% z!R{b^0%gYhKz6=bzt4>*<-wEfC-EjQFjpZO&AgFB5IL}3$0tX~?8?7sp`!TG{uFXQ zj>~o+GNGiP@IDym{0oX}G$QurFKX5sOKiuG9F%cPG3EOOvFOUa^G{BrnGW8zuYR7798OyK6`R>w$63~0>;6-_rTA=mRX#`HII zan2ooG`F+>MK+nScN;Hyx9KfoPB*sHCH|J?pHLSXMvF@2xvOf1a^Y84wW32z)pT6tNBRWkd8XBM;;Fn}sG*x__hP za1Z&Q(`r$c=k_f-QDmK)-+Rx^Z8_N>&t)KxDK*WWM`q_sea+HfIvBK6(DI&`NVEll zQg6TUQJVHWdWz{%yF0*|sQa;9Ge=pc-aDJgnC8@}9;CPFbW0p%ZZK?i2^OAYkVJ5p zy^aMg=`G!@%Aw)|#Hg;+o{2{hd%%8=2F(eed)K9+-Vkb6u?(E)_Q~r!Uvfa&yb-Gk zdd$OpL`)vwHG1bMDFjTM_!DiH>*CKa4b}BqN_o4B#s(bj=H?k`Zl5q8Zfv|w zJ0>!;!Q*mqJ223*k0j!=jnMWSeM2*wYxy3}tq_)xIZ*E-nwgwpyt>)EIE?W+IJo>s zN!qCNTk_ReLTrKZw9NjKdkKX30XK2Cqk~4#CBwskA;wDk{hd-1%blRNZxh?yeTH|( zc?}*`UR}Z^S$LOy4mctRd#;3;4=O)=H)I}Pv1&%3FzVzxABn+WM0MH+cOUEP)6`Lx zgaZW4qFv@El2~Cvkdqq}I#C(U-nD8V$~}J*S*H6`Qot^g<<>^c7n?E*W##F$ZY~9& zLg3zB2UgC6wWpSBl6nq)zn)fpyWD7VL^GEKWPT+^)f$@!NylVx*n(Oc=S8BG~tn9|;b@zdSm=$4+SF!`V+NK?45 zPbBuu>dNijdF!Q9Ynh)|Y0prv!g|fd1#CL=G%^mkVW2z9B}~h9^Cw{zSHnoXu)d~3 zXv-6Fotph+AY;2!LyK6OQ*WXgJWEj#!s*h=A?4kvr833M`PeJJWRYDu8`ZARJTnc*47uCt1JRvENV74rEVQJP7J zZU+v=R^3=D_wiW$9YICSv(4)3x_uReEF>)@HzKpLAIS{#W~cFph{&SGZfezgZ|1J877*UGG$r@<$DAI5 z6uD{Xlxb3w>IW8)U^+cRiXF5d0LrI{gQs|9WRgYu?pNZ>XF1mMkfQ6rf<>gusR9fPKI#-=fK-T4Gp z9Zx<><8@}eoPt$+#$2+hL{^vQ0EZ4MM2*zX7F6|wF~=D)Rq27% z9MVd?K2B%s=QeGj@Md83Fxa((Cn0ETod}c`*QPLguOcJ=fT&M@5{)aWtLuY}z1P?S zj|9vjv=(b3&Uok=58U6z$&9}WC(;MeF*e#0T;jAJXahP-vN!x4f=|`?2}?>27t1gn>Q`^A=H6v^^}ttAn~C zy30b-PAT+e+39tUnUwb0T+OOyF|C@+c0!936u+H3Y!JMiqU-JM?*6`6hnluV9rnm{ zISqZC_*uUXcArQyQrS4pCTZpl-3pm)!*Py92<}(tAkMrFrRd+~vo(G%!iE?EMisP39 z@JvcDcf7oEreQ!2>wuW0aGr#7bQ}t-ZL!_zz*rA|j@Xo;A7hF6x&x`Xq_r}Sc^i`| zkqWa_KXgu-U-~EmR~Hw5~0Sbyq;F)e`-GRV+97&6Aw4?lR@4@7O;cezBk1)1*8d zbhPuacPl$-YkK-1qlF_4(J1JT#}rYiFXk6#{=C3{uiE+IQH$Ls-j zo4E6nLEN=9IcIgME!umdrzN%$xkEWogc{2dcPm6)?Xh*Aqqu;HD7Td26)`a4xg;#XV+da9Z9Vnl(e7J$nWn@Qz{5pe9=2 z){t)n-gv{1v2BcQu|d)9`jNqcfkV?<i0H=XWt;z z9fv<_3#Fx;{h|}kFGwthSnWjp_Hqz;vICX5J8>kl>u9%Ii};oL=vTCCFBE|Fb_sx0 za&{^a7vg=CDUVNI)^*o}m|@s{c2;mC>)pz8S(Cr=!t=Nh!d!=uFxLaBh!@@FM-}Wx zuH=6DbO$)IIr^?}D3B(b=-VyI61IlzZyk88WisDzPA(-5RMTu=297jx!ty1^R&m|VJZk8nl4kKsb}mU`nJ?wng~9pOYu0gnNfQ9cu9!C?=dYKJ25$-e&)tOx84P7U{Z0L^Oo5GR1`+nIuL~)vL0Q!EOIc< z%`ECrZha&2QW1l|K(wk7aNCV$PEAd`OrSXTM;&nAx?+gLAEEQAiD!NI@+>jDn+pYk zHV_M$5RBfvd)FIqKYwAl7EK5f1##lS%|XMrHr>fg1Lf8XX(q9Ib9}6aK$n@RyQfQ(e$`+W&2n;X^9_>p%&#Cfmqca;BnR!T1KLt` zCRkXZ+AgyVT1(Uv0|T`l5> zv+D7dLYH`KQDNt6MJ&pcuGV;Ig!9Ber`V?&f%hB6U#e>Mhl%uT(uJL&uf?4v0$J&T zz!}z5Qqp`As5f~cjg0!-Qy94(Qa_TgT3C^)YOZh2scxD15ohh@EwZZK8@T?-7`=gW#Tj3inWkjPrkw} zR$X*enpC2ad^=spa!vaTkC@Y1*4DCe-!}f}6QuQ6@N6Q^AJGwwIO@hivKTJ3=9sWa zCl}RhUOI#SoDE1n->)rg9ec9Y)_XB2Dg2;b_m-CBnskGC*jQ#2&VfuM;j@KMn8lxk z78xCPpOvmZrDEfAq-y(sL;Y_=(Z!3>kIS|xk8_xY%`z$9VXyk>^e274$_AA43nNw~ zcdCp8aISq1VM2d^>F&YQsQpCEdo3-8nVY-TgNU%SketsMJVx>AB@*pot90WZ=*P+j zdzr&$Q(D}+m*y!bPP!{RxT^(-XU%Mjck-fkR(A4(lSR6wSd^eKz2r)Z4cq0bl?gL- zYp*aP$0;CNw>4Y?#UiGw?D5h^}%+hacQ%(C)H3)Myfer5idCcdkJ?H#;I+kHT^ zZ5-ey(}>!A%EUBeL@(uPS`pq;a+fNtf||nSwAd77$OYmRHHjVobd*S=*MEV;x~(_y zfD_>tZ#pJhS_=9(qoYPkJJ#_x%RYxs=2R=y*{`ya|MLVcFmxE*$!Z*z{NJb_ z@{5_ibOC@se;+S6{`gf`^Qgy82DDhQDNJ$7DVj;-BF>e3B5_S~do zyWp`S$7AiUW2jg4Fh`ERe#{~G`k$W?&j}W=MWV1YEX{g!J0|LlJPVdzU-{Gb^O3-0 z47QNbEt>=1JxvZ?)D>A>w-l!KEy^_=eq9*QLwuPiiwEEd=^O^$cSGVi44la$@V0N? zUlb-M3}t94wY@)CK{^_!5M3VIBR;HOB+}ED32C}VC1~nDGdi72B zX)sB*BppxHAphUzV#WB~TpO{!_2vzuH23rB(Wj;EpBC&YzxiKLc=F_;v~lxr{M_~n z$A4!Xc_5L97MC4~mh%8xsuIYt{kUp(dj4x`60 z32>M}-M-0cRe9XI74gr*c6D1PwbXkE*KZ{$|2gh<44143H#P8|o01L)ot0*DT5?ai z^o=Hs`3&t`9qFW2r-jfgBKSA@Se!#x_x(od{O|_A!))FFJj`RiLspEv0}8uDc%3St z`>C%nITpJV2Q>RQ2OeXxpQ{&m9) zENOVmjIouxAgBOtttIB{+~Tl{bM?6@$`s7dIjHabPzzdQdXgbWl2W_%pJ{snX2hdA zW14qgKAJsk#B*EsLOizXSGYlw1FFrTtnJSUwRcSpb!RnMeh}4@dba0bG+-`S!fetr zEeW0Vr~k)3!IDyA8Axp*{g1_U{0~Y%+Wd3uzxdbk!LR;z?!ZL<=a9MZ|9tII{{Qy$ bhaBE&Zva>fQE7{{9M9RqCtB#euH`$58A$w-;z1Qz~ zj$ZHY=XU#@Kiyo{^}HVQe!o8+SCF#e3&Itv?Q*%vPn{aWAT8zKEtQ3%bK>P6q-gPW?X+z3!`wpKF`%iMl9g}?Z?vd6{ zMEw-Mx#kz@Qlu7CocSXw;$_hy*I=Q!nVK8V!( zxj>-FMCC&D0{78csJ(+whp zsaS0T$YRBgUv&ti)5GXJcD64VZ+>Tp&EyJ!E)9POM_mQuTv@)6z~MyHbV5UO;nb%2 zeJwZbFV{UCcQ!KFh}%_*J(2qFWiI;ew~pW{6_r^B z0t2o45qm9dm76!vm>b|T>Elqm>g-{cLG~>z2XOcJz>}M5c3i7Y8`QEQaHyD!(Z_~Z z7M|axZe;K(+97P5~ z0mW8nv`asyoF2BOcCz_>O!WKOm6;kDtQJ8}m{)FflE{gnsb@fI zb)>p55JCy3l-GWesS|RLu57OM1p4d?-*H5+6 zE-nRktd!2I;{Hko)5Gw-<-VWKJsr~2zeUD~;0lK>ja(ftgzfRna6LN@ae?o8q4?Ox z*4J~Xm~FdhrtG&h#bp@g6a=PEGq!d_sZT~=yx+~HPP#<==w)(t5>ID`&-Q++F-8Pj{}&4_FLpDrHk@;O*EyOR zc!K^g3lYgKh;5G>A~UM4N!_7OnfI6TH2@wZ4?CmvsyCJS79lP1G2|ZFalHd5$KlG? z=J`-5F=)^UUeB-)>y%Lz!j)xBdm~jsDi;AQn?RX%SE^-RGYh1v>A!R(Bve@S&6KOc zsVMZ_`qW#_QVby@m*`|E&XilTjqQHfDa$5<+7LYjPonMb+;a67g;;m0TsRZ2cQO z>(LB>r|Ub=y$$(1sm}SPGP%zC%-?Q?Qpm|EY5X-Y9y`FJqDq!e)RvW!k`nK+wX-xm z+|ZDfV^mS*JSZx(g27C>206)?f=&LmiX)N8P_4?!%Gk+C=J@!a`EYtdqx{Ej4u#KG z`xjeY5vYJ@Rrwc7q4|TH(1GiX&FV$x>Xtg;S|MlLX{I8#dk7izK0czgGcF;eQ56;z z7TQW=dR6L-rKdLmKV_m8^;<1?wG%@aKNK^xQ!6T@0398x)Kr1KOSG-?ePZ(V^dP1% zdp5iyHKVjN*N#I_@Y8DHtLD7;d3zo~!CI|aTe`x>g+?vy`55CD@56|GEJo84VtOCJ z&)hrFNhkyBvxFwB5Ti51=v&sKI7iqAg<5Ey?~j(CKhiWr83gHjh-ur{zp`<<;1pvo`%uD37D?<{1l@9jA{93u(%oxIwNQuA~4I)o2JWwo^h(w9}4neWM| z|5|l)a#96{4)a{n!w@?A!CiOu`c7*C)XlBBZhJxFOP@Y4jPo|)f&P6VEg4NYuJ3BiW>F6SLY#V%Y%rY<>T@JSjr;ZH-1O(g`Muw`+ zN(fF@ey8##evz47USFRN>#2(<>*0`r_-o{^*)?OZ>AWVBlKuQIUB>!HAmi&uWiqQe zRKhT7&{^D#O2TJg!gi^>t@&(b51e%`1#ZNTL7w8`Za{`(*d3?)o8MznM`Wt3l){HB zG_|iXHZoG7uXS58!I=~`-nqBK!dAB>|6BC1tja^5i<>(RF8(1pcNIU-rS(tpllowa z{w|aer5!g5XNNw;B3OfsjEfjY6QhQ#CAWZJ z?MZ{?ipbFuY7ZBl@Jog3l^Kc;F(q?!p^-{D#e7Vz) z28KyY9MTZi-ZIfY+WycxIZ1NnFeUQ#ZAtZVKo`s7_|{#Hyy^F`nx$|Ly21vH7Fd=B zbwzu^(9~Shl*n^;cO0hsn!gjuDM3&gZBE|O(5(Nxob`);&Oy-Ssl$|?5=N2b%qvq1 z3x;J?*@%GP`NF1vU+1f4vEsJ+C}+i7*$DF6z2<=~rJ#A<4Ty%=97)>Zk`-cJe1wZx zgrCniFPoH+DzViu7kA3$#dT&p z$3SFd9r73*2yXimg9DilZN#*oqoclFI7%s5-u@+La#VD5r(j`iRO-c<*i-BApN|SP z+IWoJJv@$6Q^#M7O*)s9mR2pdcSRl?)5}Qw2&LK@3p^cKOd;G=k^{g{y7a2QNT^%r zfc}Eh{HM3*nzIl{s5}_wyeG2=i*vUY_U#y^N5!EJ9zBX&TVn$S1ZF$gQe}S(pBkI& z#ry8xeAWW8+=LvXtqqrkgx6-&6r4nMHgut(-V*xr@_Y{y0)ce?_%TOEQKuN<(Rs5aPr71AB>TGvRpY}NHF0|eQ*&?xQs^US z$mq#Fs?oS@dK3g`wIB*ajt+&a%)KMmiaSy*Rx9x2IE2H1pVYcjIzMp~8qcq|-_>_z zI9!#0#Uj%uh+J^#yVJvsUeRQk=v`Pc1XG%hzqbLe5pr)|mS2INYR%L^EAQvfQI)VD6BrQ*87~(YRJy~Jm44n^!V7M zYCiMPX{P>DJDa6beLFj%$B%7r%|^x=hlM2rF+<` z&oZj;a@YIiQ#aDh+W7>(hN<3O=@%B{s*L2}@yk6%vJs);^w$W;4+=Si$Qv&GP{0!V zQb`^KCL(ew1ubhNUmJPc0bruNgLX9jS@=!n+Pkt!8=}V6&3W_zgNC2u<2!?w%>vIa zHmNVjKt?7(4_hU?8WE-{r2R4x0U1-{G>!Um&|oO>6^ueY8?C2+yr$Ki5M4*=M{9*_ zlZ$oM0n;-?bqB1Wa-Mc6ed;`X{*e{`Ox1F?;CwHtl#w2k8ti!sXWWj_3;zj47X8bW z$%5Zc@rK$%DRKh>?i9^-w!Suv5g##Tj;FM$>khD=2e&gaR-9-1_=)jN`?7;7)$<6b zuFxUVaB-=3y(5dMSq4D+)>nuY@wMDR7&_3Fz@AHccEd4GE>EgYg;}H;Bus?t55A)d zqGf4wc@Dw9#by!o3~L*;vqR(@vH;@`|D%3ef& zGb*|i@^yf}5jqo%lQ-(u(E|c(#PS#tBBe?tj`CAz&LEz@)!!_=gE~Md&id2#@}@0| z|MEArb}enAP(LzdY==f+p<9jHgBjOmp;sky|Am+`wFn`Wh0h{A+s2 zC|5q~gn`=d({Pl*LmvIU8NVl7+6u|p(RR8*@vxTV=Z12eZv##OL6 z*9|94>&M#nqaO*|8m^&8ffsw%&lzWD%Yz!N_21RQ$2iad@hTJvfH>9xLw(5rnVPpv zn&{g3Lsr8W5XGj|6c5mD&|oPq-}@bQ`iAV=t%h+nM_Lr8JZR21WLm}MW}*Q6b7`%rde4T z^Dr7v$=618zxyJ~oRXYZ2A&}?_LT_Gbqw};`e+!!=oeb5K$sNrT${nl($i>8i6zYq z=Oz028$W^lbkmtaFAha^WLmoPP=NXA24O&l5i1XF??Z7gEA<+)4L$+ho>Gkz~%kz<-t-6k-N`OPt zjW?KEW)p5W!?JyUunr$*fs&vUxWk|<#OvIu#SqMbju86#=vps|M*T8AkWC~lQr#G@ zCnuWWhobEce`+#`-H=Hb!=nfsKL zug{Bmyp{(mi@~G=%Ky>!cDrV*u=QZZRy~bH#MUz}rtj2XfTeLp=?z)a^gK zpyaigEikyGHV#P`AMeY1RU2+$prz=ncy}?`;q^x^3s7{R%;h}PZz|=~-&A&Okb{W! zAeFD3_JPITxF^j1Du_WQVlVb&e`0&JO|eC~!#5-j^X;XdQt!lsgrefp)bl-$&hDFD zR=l>OQaU;w24sw+BTeAk$bRmlqZhWOx=mUr?6%>3mL%+#y2oRlL zu;{y+IeF79lpD%#HSa2J3IF4a0t5&D!kqUY#ZNLB=wqFVF_abt39T75 zQHUMAsMUke$P`K6^0)Q>8)^I^SY5m+iXI!35f$09o!E1OtgNQCPbxrydx<*6dC2D+ zRXNz}(YD+!tKk|Rc_SmtT}A$IKx$9)$0{nRe0+VM*z*8%E-fd=1X<4j@FgWhA0;RB zbN^+u#%K>ANU^aZNO)T7QhAM&@o+j-ne7LWeyxYnDu(xKP^B_je-k#5m=FTl$rXDo zS1zyJ|2gM=_wi{rL=^aycQPYA6nQ;34o zb;>7L>80G3AJ&HZ{F?GCCoRn$KhAZu!Wocxl2ucrJQDNBay%W})mg@AbWCKD5sxUC?+yda*nv`UQ~w^3|vI0B})6M?N2$wAI!byQfJ!6|}mHKgtNw_mt7N zrpEo2d*eA7-&d*my?SaxyS3p`taZ@_>$TxJ%;P-2bPr}wW!TDi#3dBgykc1`^L$ZS z!;>1)pBw=Z)lJ*R40lXJSs#$`xoT*GbJ8ITH$Li^v+1l%ZOe4Tq0VRa_6f%9PRAbL zj*n6W;>%ZgOH3@dTSM-(DPtUR6!orz)uBEHvBt(;CZb#a%X=18VS^ghM|qo za@<>0WagJ}Zh+EhSo+4tNj*X??W5^sWaBNB@|IO_XI8Xk=lPdlvU!Rlf<`-@%s~_D zT$2vdTqptLFb?v@Z6*B+A$qUXS8`N7X7oTZv(#GETp= z7k=|MY&zxU>f;)YS6u{jR-FksuiK`7bAOzbtUXd`pjUd=2V@c^^W%6)z}9SR)`6cQ~3lb&S#v7`eIqW-X#i?RT9=M@KuUgg^FpuH5C~;=)+k z*jSiqI8_}zT&Lr(gU+kUxxpu-fTF=?YtFyfVD!r1m(3X6+2Q5n3+SkvOecN&T&VGR zK)&SHdN#ec)D}r1ybO&TAQoGG7ZCO;69C+lsrhz!8g zK$DUB0zaKD>pvOFO1UX|{2a(ObTJ@q)ifL51=7G10GC9;5PsVQHCxZ2lB%KSUtfA{ zcTF8oQQ~|~6%8P2yma5$`CF)NV>8j^eSS0q=XvJ`5j?eVbs%t=JQoB9KbgGzLZG-< z=b%Wn)x!$%>EYTK;mj!D80wtsC1|>ITVjbynXA%!lmR7RgBX%|XK;4=V%gwi=<f_LMb-NCo;W*6|tFAzvLf3q7^>h5*-=-^XFGgKB0U9UkXsM!?a2P*$+_=WJy#J z*BA^}in6jEmw92q1ej)IOw2)rXdr;!AIM%|g(n#PWzfYA*G%6H`Rclb@jSL^pwurw z?j^%*D(~AwDBy`@J(5vmFSq5SPc8IDG|K%fKkCLLN#u1PHqXsdRTp3T4CEfbI$x+e zbjOWXS`V)A!jMN_-ND=l`C)L-@%se3deak=Up)*OCIj}Ivs}z6lu-ST7A@Jn=yn+q z`rE2-4`B2HEtI-Ti=KHwF22+^Madx;tHu~~2}3xZx$prZG}IK;pYD1KaMNZAG?${U zhd0(d0?~(pt!PSuFoN||fCdd_7j+PA1BJZyxMNN@2_OZ3KY~|vP`ffK*4OeoUWd|` zC&8*SdDvhU65@)Ug+R-kHdpOxplZ0P93f#6W!%LA{dNJJSzR@bV#Qv*1kri*wqhCE`zXZ$7Td0wZlVV$klwZV#b?_|Y0F&8N2VE|LPp@k)^2_@ zFhGrN46lZFIemkB77DSf!~62aJS7suHTPweZX~cX-1!`titfD^DM-#5L8a;^Rz zk|$wirN)0#i@0@lbtm~c;f7Ptz%vOR$=9?y*BtGX@)Vc3PliAcqVn_qrnd+qbKs*gxyPWclt9o#xh|5{QRYqSFhDe14rtCP1JSl^ zB;0^c|0p~T>}mQ{c|TV&;#LdV+!tIEROcO^c+@0J2qjZ|mnK_Rq8cePlSrU?vaCkd zuqz)+jIHj!`Q`NDaCBwGG%I}Xu22F>N(ma-H|Ja?Ma?4hF+DVyG)%^?#o60)t^;m= zpF1Wd=E?$FCB|)ijTNtXx`kQdVf6LbsJQ%#D;@jaE`W?cU8pbsItqE)Dp(3(U|#o_ zQk=IXs^7RHSpC_C|5b1YG{y+OxEvjfF=Y&Qyp2{}atvlyaiMxE56-R_bI>QF6Kv2z zpGR{pS6#VGDl;fD^rD>XjFy?@5Bj1IH5-Lu553pEbnOq-i&mEbky7yMjplUIN3+Ky zpw$3vfgXRln>%d7iK*LvR!QC&B4gKtG8j^tI7Ix&hpxT3y!Ys#qm4qIVrMkx`S=B0 zdwcX%`9rk4s$W^sKMW6J5DY?m{Xfpatg!GnE0ddL9X?kv=8@7Aw`7s$N9olFQm&G# zz()3IU9i?I*4v%eaCl)(I?WbK(=R_p3ptRL8bRLxIlhBODg+_kzLVG4x2`kSV*cz2 z_^Z|bpAIAAF8I{WzpKzcCFP>Ne^J5`Cr^(uZ}FGkqXOSo9a(Id2v_uQ8#=J&&dsNt zmkxR8MEnZV=7k%y$qU;;+!8;kJ4B%tr3`|2N4zU4q!QVg@k`{q+!W;W@??z!I}qsf z#wiUI?k&2sjn%fy3uu2qR!HW~5tWS3;E+~z?d`c=0=OEA@I3okwCp2DZ4uRCV^D@q zM~HcKZDe~n!1+9b;zvBgm5gA(%I@3y1(=YrTUJ9GRVh1~MZIq6d*yp4^FnMJsm!Tk zf2u`u%~LlIE(`-89!7_HG@nf?YYD<&W34>QQQH z>*mCPn^wsl2kY3_n4F#6-sdSfB_%qb0jFzIPL9jkZZL!CE7kt#T5Np0{g3mM6lx@r zwCf?E2Kpr<(#-@>V^zrNgA#~y<7cgqmrTzrDdgj)yt}oeG_Dd0B2Rseb=Ke5x(WaL z^-W9ddZnGCLWKwT$)&q+@*!AV?TKOIQcnwa;IiLD-IDHXpAJAV{R;H#iVl|6P-H3obdz9Gd)iZYFkDNwJa?2cl zCP?I~p#!a<_j2F2FQ#X)6bq;p_$@Euq6S^}bgtB*2%OS8T{GONAyc(8?nlGf=In&5ws>>GrNX8KFuEnv@gB=>ts>qEEYQSum0UX^(p{}qd+#QbO;5=<5e*Yd@ zp1Iq?z+zFX$HmbRIWqDbz`XafJEsSis~wG*DMQwA2`ttw(7BQeFaatNBs7 z&2wX(_9UEiIm4eUV*MGe&Lv1$5E>BK3hMwOFkG!&!TLu(66UJ(^4IFrK&L zpW1lY!~_rzEfZ5>kJm}PsX31KuR8nwrvMZIE`b3A_xkP6J(oR~lU*@3Mkoa6>st&z zB{y)r3xsQHJ4=M^uXHA7MEX^H0J1dJtPGy>t(2h z-kNRv_UsM-FcH0__4W1Rmx&#iRWr%43}=%_WiyoM8KUdq@~j0U;7WS&4QC;sUIzye z*WhHPP}PoQTr1hVY9q)u`|leA$@Q0kH@~kN1k8rgY*nsoY+z3Ziq<492-}#NBA&pI zuIzk=j*5IVsvk;6d(>F9Mu|5 z?Y5>j{~k6R^YG2)A?oz(?0P@ItfpQVHQfV~5~I@tsg0BU&1%o(_elbce~Wr5DmJhU zyDB(c&W0ucx`nTPOBmvJY4T)mSn93TEV!Xa$+rnO>sAh;sS8yrASv1qUogYxATs34~R1F-wZ5WQy@s6LQ!N>tf;FpHm34mYnR;-S zlCp}8`AHzv{8AV@YBA}#Urf?JLUG@!!sG>aMn+`S+mYv{h^*a?EQ9n4FX0}M#-!t{ z?#!a1>7JBTV~?t!y}g@&M6ttC6(GT=kU*pt)^b}`fzonV1058fd%lFvq+zkcLtMig z=g5-d^Q7(SOfp+Xbf7x4ZNk@D-pm>>?Av6?vzY&kx)9wArikn^i%sdFx^yB3glBDh zr2A-l_SW&5KmM%Ygw)56j9cl4dM~uKAs9mj9{87g>ok2MBWp_>UkmGz$+M|)H4 zoAs;ON3Be%q!viTHW>Ay?mcJ|-tyEqQ~nl&@Bc?_cg1@0bs9+kPdt0a8zO|NuE^=Z zKGN|;WYOh^V#o&}d`EhXtTV4w-HSR~Qu*@a93FRzW@l81tVAZ8rj%(SQ-5AP;snzd}F!DoF)0@~(H z8K^XTpi6!q1$EYe@_WzamjB(BC^uyl{8&F*!&G}AP;m< z0s%4@OnA$5@#-mm_5d{=fYJHOVH5>|AVjvGXkD6Ly04g4tvfTb%AV-BxVZF}IgkV9 z?Rh$N85T545C??`C00J|CAEqI{152Fv4z=YzeXQe85xO?8#vy@*o61|8Q*THb^Dt` z{;JW7Y_@$~Rh4`kllu|0yfEA$?Es#P5lrQ=BwCoS^{T_62ls_y<%SJzT80_NRO2i; z#(nRzn=x$KoN0|TD8&jCASyw^a*nnKNL8?k)iiJg|TDF;}BP)BlyXUx5>)ubD9k8qn)``uA>OnV#v7Z4scC+g3%W-Iz zeMj_eD~&;BR#xI;9mvT6lOprni52_I%!|}pq0D1tAz&^WnyUUgWK-L@_w?Z4068{> z_3Xk3@&sseMJ7WVRrQu-m3ADYBTt?+5+IBJ-|l2UdQ1Qq?YyzE9yPmzo*Az#6W3e= zi0liBAx)&xp%y@)*WsIjTP>HMI2?$!AiGqfKh%4SjZF4R%4|#S>`2k<@QhWwIO+@A zG@5yD@ph6wi+b<8qM{zUZfXAd75?(_a>s*m=-v@8sCvsBL;Vy`y@#M61->DKHe8<= zBg<8FzvF*bd|8*d!2d}n&97MSOa!8K^*0K&9aTp?HNtr7@PeD$@kkX1wH`E`K7F+0 z#?&J+XSK8QwzI2CL0byOSA-gHkPg*70UH4}y{<4_d=GtvJgJTQc#q%1EN~QJznE}; zk4VR*cXxD6m+ZBgbD03 zATUjMBrov#>Vy8oJ7aZM5z8*%xYPziV$lR40?L=CkGCjnjLVbF&&kO@O(Q~|grSx^>BnlxnsF6t)x`SA@b7l|^m-V!V*_k~AT7BC z@B9i3M<0SWF|oBB*2Zh3n1W?c^@}kWdJQ#pg3&Kq)eMN4Q2ED7bgY8-EFu>Fl(sgI zwyna+@CpXRF?yI~&HmRSXZU4J**)7+(%T-bu~1U4krjj8N%D^`RCqCCxX5a#f6jjydmt zboJIV6bpbo;_mmH$5B8Lu9Nv!@>_bmJ;^r!Hrmvg@-OX?QGY1o=YtwE2YgvjC^l&a zy7qI8rj%HqH@ZLdXZKqsNXsVus3?5}x*-={`*KG;G;N#b=N3jI(K)j8u+ONtKpGCA z)+xniGm5Dn{^>osxwj#c_j_Q~sQo!eT6^qZR%x_=1x9Wu?)Mu`LXPT*>S-=k>QQ$f zmigb31+JyG%XbnJCt`hU4^1$3yS$0_H~?k=CYRgX5z!i`rY=>n@(1AH*~5Z*NW;#) z8bbv12a5f$M31_FcDiC50why+wrv4!+jkV?S3#fmvQrubgJcAhT?OlQ-G}?e8(sRU z!9Lu(`v=Xg)Nmu(LFKkI(hxP1MpS-9FB4|UA#Oo(4G#&xe^9Q&Lpwj!N}DJ=0W9C$YR*kVgYs^_UIxzdI#BwWm#+}0 zjKJNCBYMwyt8EV&SDNvi3zXv|5&uo7qqx>*UqX46IddPbv__c=$=N{1=!1vd`I;h6WBZ*P%2`aSbD0bw7S_+03y#LJ$ zMr$y9e|A*py422r^tp1r;bEuJvp<+w%<}h$49;%Iv&fmT%18tQ7a)5F>KF#CbBH!v zg4CKeP`iaxzP5)+Ce<|eYgUqEZBhF00WGJO&`p=6JBSZ~EjX*otq%`o`?MuNx5S zoJX=-p@f%5>Ggp3V-pxIJdE?Aj#6tLMqV{}pn^MPX`osSJb>B{xM0 z<&#)ugY~ft$Q8b^#+$gw&F;^#>u1s~E%(m{(gOq6Q%_9KCaW!)78|2L1q~np8H{+> zH+BwmItX4PPO=@nhQ;4DV(shwt4@++Mc#J|9&7hBo=hP(j#Rr!|@C{vock^Aj*=sotEQ_VDM zZB|toh!hGxPHLkam?>wf022NRvS-c7;^|$Vw~(;Mo(opEI)23D=upGWu*)Z2o2v$X zk&22S30hD~y2vKyg21hdOgTb-O^MZ906+Ec(=$N*nycn45c>Cg*>R#xpW4l!snb!- zY2WuK=f$Du6rJ8_0T$r;fzpIfkz0Mgfy7tO9p@bH%owF6F6DfsM1Z+=^(OVvBo*-e zE{khNIqLFz|MNYS#LRFqdx#Z)W2Ovi`63^2i9UPd0Ei#Fo9@US{8;9yg@6w=xV?SD z;19hv>v>x&E@@FQ>;PgI#%C3>H?>~kJ!;l58#ENZtsiHPDKAJ@HTE1;_7UmX4YzFT zNNQ?M4-2$z0LGz~83FBfgT{mX{`!etWioj{b|gUlf$m$=x)R)oXU={j6YQryZ~?pD zC~Oxqv4-8ls1sZRVT(vl%O!-Yq~%aQw0ZzaXl$B?hq2Q~ z4!>S;bm%@ii2E1#u&tx+tm$&~^VPE_Tl0nA%a(KG8DRdXD(ctwo7cFX9o&Ebbda7d z;6bNr{xM=pLUCL)?Fw@LnXk~!i;x;+KFTQ{)df@;BSDq13~di?|J7SSD#UrHB<5&m z?DVt1kAl2u4_jFm|34NA%FNbRn(xUj9h>lZ=c=W@sKi5o7lg^<60q&X)2l@=a|NB+ zy90X}h-)x1ldhS3jJ#^Si1g)mr_;4lVN<7j1})`7S*VAz{m71L!ASPjB}qN~xe)UqnFIV(NP$Z>ZO%@Mb{csV6FNn?Ud*pDnKk1PEA;imt0* z&CK_1yBj-!FOvXrvuBqVp^zeHL=TMs$mtp7+1f}<|{f`}f}pVkI*ydm6v_K8scN$!2C1$X1W|$^(av_l7IQH0V3#*H))!C z!1mAKJV$feC)L~15r2X6p>^dvIN-^rO#0u_xBI|oss||i=DOGWd*1&giwp;!N&){d ztTdg@2u^*PI-O&hu5kf@M^8onpYL%b?TGwW({e=O+`+5@jAe8pwUX@@pIMT0^U-8> z2lmZg2}EEGP>(|hm*zzJZ1NZHchT_}P=6hzLHZNw)mFA=>JC0bBbdchkb^M<=`H~I zQ7nExQvUB1-j!@PRUHfWj*d1~Iz9n>R5e~U;#TL^PI^K@y>(OsfU(eHe}jMm1>nG{ zd*b9BnR<%v^7{Va--w9xY6_9l%581TTXu-l=YU=R|n|RtixyyomF^JIL0)@-5>%Gbf63`4}Yq=rLceA&pvat z?o#dWpbTnRBQH$QF8@#Vj)2}fTNQ1wqXAqo)~-1x3+;zHAb=ur^PgB&Ne(_Up^^k{ z3~ybPuf;Uiy*IwDA|yy5gQ6b*rtiEsE-P;vMQOJ%d~`{ z1@ELEOU6ie=z$dc?cW;#;Nw2@9?#Wc6Y2(sW4JR@4U~;8s~ywHya*5l4F5#|lDOPL z?oIdbyMA`TRM{eF9RCZ}IA{E1+*t{k8)*0)5s(zC??dFt?6ukM(oEif>b_B&;P9b} zLn0prup>eE69Yw0Gi~=*>x#SSqzC5K<#~BwY zBPVCZ1wIu_6HUm=$_fk)4p!^FtEDc1!oJupg9<^odT)(TG32Ycge8mr@4#h(2X-p|lIeRr zc3l&t@o$rpP!Ip`gMt7?tg5Yj>-(4E(wdHMxvSX)`M5ymAL8_$yoo{WxkMr$sCWBr zvf)B-H?cY!S9`ojw$OP0x(b|__3)X;M(+k$`MpeiOwcBy%%(q3<$QcDdG$M^z9bVm zaorj|7xn2XyrKU5;lq}P2r+ppD?yB>IsNa zGGkdXt$!X*jmG!e{cO11(qZS3kg4v?_*`0}6=^lrhgUyj_l3*Wu&_&ujf?-c zha}-fWXK1MA|f$6KOqNeqTMAK4mmkFi_G4oa=*m=xs*D*PiBH0H+=;I zYE3yRoHt9mEA^x99v8d+@Id%CjeoW>T?(b~+w^2|T|<7FX`p$Jei?G)1M~L1n-;C?;q zc&qaX_MA^mMZs!q|4jDKA~OTK;ZQF-f0ZJUm(uG9Ln|*mPhQeg>pk=XX4Pt_)kx_f zvD@)`NlCs}WpD0AGBIHP3i_~Rm%KN!Iv9{G8*ZebL%ih3e}CB4nghCie^hu)m39~M zLl=u8m-g+MwEClePvSb;IDa}vz2PPVyC$@ydatmDPjYVDI4%=3~7Z+|^c}ivMuBO+L8$;omR#rDy|DiZ{ROno`(z#`?jGCpl5Q$C)@3KkTyEiH3;q=Hkl*d$?xF${ z_w1U0={QuOPM;EpW4Vw`R17-BECaOw>#_W z$%rh4jl*iQIH>0Jin|2tFJ*a)4-S6dF&UiyRA_C#j>|L!o zO}NE0osYmq=~V0Du5(zvcJXJqp_Q@gEzSKio6b3%;M{>}Z4Cl?kMK(hyLedVAwuZsjYh(?8^56(1#0)zc%t-K@o?5z*f9#O$Z3 z5H*MTpYjQ#HjhM+%$(}?Rt4H*%^z_sx(4+I{_yO1oELW(AoA)yZf$Xzn_8%-z4gt^ zyz6zk#eLRQ*&yj!b!i}Hk@aaxoOmti8jJ343 zHb0zf(S^@HzZdZ2XY?(7Jw2>VyUBM(TAT_7g{z=EL`7>F6>C*5Zj1$e3Ta>5-4?w( zBu_}t9xi?q7~QpqRRr(A9JeJZF7ET(O%W{CVuM1V+pAN;3idE;p7Q-*rF*#FYd7K7 zzmJdfS6^_)Ui<7(lo#)xYF$xR~+W0SAdr;m7d zdkd*M7I#(Nl*PeRJ8U(ZRD0?IU?*BNuD8ZqsF$4TA9*H-JkCGWz4xiDIYC`c>+$ zFDFcdZ7xms&dhcvg+IBm`TfOh=eZRqb%ixxWMHvzc<)5qDY3d{yLe$Ft zdl!qW3XP-5dgWIW+)phS&$ck}D9JiBKYa+U`g~FG>BF0ApFb1xP77uAi|Oh@oty}x zqSS&%9+>yXBo7Sol2R+l=iS#W45Jr|M!%F9jnep~`ODkdhiQA}RvvD#wt?V369KN_5rfzGKi zsqebWFM1i=n*Lufz1=vPbeZ|yl+qzHJHvE&L*_(0_d}AXIHXlRF`>P0mf`l@nB=zw zUGvu-@HqWtR(tAL?nG!kT0-ViYkTVGg4@MZUQv!kOi6hMYFql=#m-`3+jYww9+ye< zsZDm6sey`^e9>TRbcM(ULaBrC;UnHHlpoI=C=;~g)P~!JlGKIf$v?! z<7Nk}6jHrE6(_U^H_6RYTaJmAK^`uMFGRm9)zf%KcTi>!`mgH#a9d^)^*wxw9d>@U z_*s}v?M~1Ke@W2&svSz)PV2*%3-f3j)3T=&W50IDTEbv-LPEj8dr>b79XM0mPt@i6 zJHFi^i069}esB;fvglMJgdzD{YOm(*UGt{(gTCJ3;Gxx3>QlszXVb`$WumMa@t6-H zzHtS&;ujVMxm}%vf956z&WpgGjN7!@o!zR4VDjU*Ep$!COL}5nsh{88s5hgiLbiGJ z23~F#QyqJ1(TShd>J-(ELoL&Uyv5FIfk;iTk9MnJ_$D-LeD&_Xg2g`+e7d`7pXWhq znYSNE)EmWvE48)DKoYK$5Ot4gg;bofRV!a(q34(RP>P?>=@})P0asy7mbWZXSbR-Qw%*#)Wr9_SPHrwMXH>@)+A(!%2Jt0nhv7BB+qT=IFY zIk3G;P4zTTOtC5sK9`uC<((W8^k4qV!gP^82CF+F(h`8+<@V6!ZEy~b3Tx?NOSn@T z?5PN-4*r#G_H6lMqGlK?xyfq77 z8=ak^L(XoO?6*u{{Vx3WE#up_ilL(_`(j!GkpGdE23u-a3$>zEVqi>+#$RKT=aPc;t}kEd0_XQbwcK5oc$nQhr}i6APObJPyPiwVoe=}<#?>5Tvy!3Q_PNuzeO)OSdJJ*fKEtVhJ1Qg_AC+}Mvl>Rd> zf4QFEj6nmXkaLU0G|eSHD&~XnX$g(*a@6>=ePYKm!?$4b(1xTo?$+qffZ>e9Et6C#9I6Wafn^W};16^uvdX*w|0Lf=^krt{|;{AdOqhS`O@XbX5Q7 zsG@&=lkYKi%_6%t-zNnfXRy57Ug))Z9hMXM;}LT+6fWN-OE79{z0T0Zr=iuJPS@dj z)aNAmoR80aoMt#ruau3x&`DY)R^oz2QJ7;@?uOAnY%SRyLE}k&aDT}yiI|A!>?Wh- zAt?z7MhjHIfp}L+(!t@8`X46?&(x?nd@+mcU~B*>QM9MIrRAu}rSY!gFQ~%B$zHCQ z#n0AO$?bBEDS1=Vtzt8P^Ur3KRxNF9*QpZ|;(sPr9*UG*xEir2ov-r%u!9j`G!wxi zrv;yRMXc+px7FC+q|Uchr&B3*^0V)o?0<*D*(pu^&b{i}adE_%*E>@G@Sz^)4WI!; zh9p*|Zo>=ePe|{w$MLiAgfLc)Nx)uV2hGj~ef)Uujjfy<(Lv&PQ3dx?YbuxT4(S33jX)VlbzbB@0aHyu4_%+eoEeq=opuw zok(C~6d-LWGE8Vtjv?Winfd?N`s%1EyY1_Ph#*LbbcraSbT^2CC@Iq2Al+RmT?Y~A zQjn1D?(QQT>CQuUAL85izTSI(V|)((a152_d3LNd=Uj8`t?!k>3Xjds4pI!@*1k`$ z2rDYVd!v3^_Y5L(&b)c14s#ue)!$S7eV6txvQ`m7I_}`D2wvymy%PU{OS2WkE*qtL zb$&M?QBxhj2L}`R4eqQ`ct0Fw-L!F3bi7~;*3rTN8anSHLC3u9m}Id%DPf8n2E0-A zFTiA9PObzpC56y25ZpwsrL&*i$0vmYfLUs2iqqepFD@x*KTMd51cGe7>fVy##(0@A zF>!&~hjs(fVSy*&4_f{%%P?jS=Vq53Y${|Q_s#0KgEu6;2qp}|S;BcIDKu{Xj*oJ)dYKDA&LpMZTUhM4=qM^;9fe>0 zdS%+&^*krXoZs=3WO&#qtGuD625Q(T%{M&wMkF;NZGm8k_`%@VZ=pFYuncs(=fB;O z{Lc`rHS9DG)C)&+wztUp+=-$KEO~|>S^DBNEGVAW<_>w~CK&A9+*zUStx7G~SME~3 zk{10lgicyC0gU*-Pb`C_P|l$epV=M2TUlfQqwy56IjMrr&jrA|Lw>3&%DF{ce0nre zze{Ibi4Fm<ReaGu^+_7py$iWgdw1sg)}TYH6DEc>BeS_I9&YL zI!sG@RCOK_8plt25T;xC7h7bz?&{8myL{^OzH+G2bH15X{#=rkpAF}tEjm7ioPy5F zj-nN6d2Q|D25QV)yp;e@-bK;R*^co6k{r<) zs;_psBJ6@kObp*SU-)8wb+ObNml=^4rUcM!H-B72@3P5An2dG9o)pGyMqH6Y4DeBj z%t>KoMiUbP48jWjX6Hd6A#;qA7@zV&*`&$52z!2I!XcH3Fp_f;eMx8 z3T{4s_p#h0wKw`T(x=(*`nYg6e&dh0qTYGlgo-PK*K;dVR<_j|3jsnYh9hs8i{sGD z%tru!Qg~?veQyG5E87dT{4j~&#ZCnsoqp8@N7UZf0mEZ7xLSiF)uhN2NP|JP0)wbYQ^EuV`=oTf(XHrw}F@>HONF`qrMKScKu zhb`1T{=2KiVZvCAt_-f5le3>Dx9{CB2h)tt*0LkU?fYi{y1(%jWA~THP*NJ6cdjKR zCYOED!l#n2^}MbI{Ree-H^J$|?DTyu6EKGCPg8(2laQc!M@v^l2d(K#yYN;SPGGV$ zi$)K*jm;B!-c@@vhLt3gAJRPjB5)MU_?4MSU zhx%HUg=asz34k8sH&pU0(=-crFZwU62`lpQn6Cv)xc2zjY3$BX3m>Ty*gbkt{|G<@jTqvdpF9>Wq3zvKhZ$EKk){1}QeB7A+vLWT7^ zeA(bKm;7hG&Fya$1^Qz;wSRXwsYCT#;7Mz7T1xTH_}#_SPl}Vdp+QM38(>H5KFR(`PD-k$J0*tbL7L|~lUJC6E{FyD>Zg1feSy?F%VT@*GJnYY zDDL)Y=$ix-hOkS2Lp5NLqK@cIkPx~1$}*x?X6&?c%NR@BDFmp z(nrLOG)BM>%OUdl24^MJ)1x6F@e!6{dKpA%sT%AwV+{X_a* z(|3XEiHLkW$t#r z)O5UQAcKRbl`{kJ$+BQWqT3&grFuMO2zryq5nuK+%6BC;Mg_uV_HO92PSoEi3UWQf z1xlFDW$+6Upq?7sbwYV$+Ouqr>w4*!*uY~wfzxytRe`&)Q+=-$(zJ1m;|c?b=r=Eq zAqi2YY_h$E)8EB^FOfJkQ1=SR3!+{G1&!=Cm?FvHL0gx4ekr`&=Mt8B)~mR&=j#3f zFBLXQ`9nq~WQVFSJ9`DFK697hlfK(~+aA>O2WK^%6_%3{ZpPRF`ufWuXQ0VZd4KL@ zM2T7Nz&A3GZo2r@AWesxVcUCN$cVREL-6oL`)ap!n?e*ei&u0+p;zbszS3(9D|c$| z)5W=B*er_fnZ}l@XaF^wBiMD;Xr4%?XFkzrWs1%FfQr$iV&(s>({oLr3-@Vd4&8 zyFkxAn@bm#Nr`){DM=5+AH1;{%lQ^{h?4T~{?NG#%Ih~NAKKYKjtPe$+D)(!6BaB~ zM<;)A*$>SD6mU$)r_O@vTMbCxE0v(2F0&GmU64WiYA_<rU#$iqXUeXI~OYwPb71^BVOvNAFc zEJm3OmqXB|E$HP)T_*;HD;m^sb}vk z%`=OCqWzQg8*E2SgM)1S=B5FH{__i^N4}}@nzV<}qa%hTKv(ECk0t{n0X{!H>B<__ zHyVyR++17=s+S#io-b19HUlJ-M%?WFB%1x%D6f z6m{m*#!y zJBLR%boct)<^~tO`YeYPd+_Q;Z4@oRNriRh^J+yBkLCuvz|deT^PBIvAS>Ha>`zF?xF+6BX5)CzpkfC9FObR(#<1ZTx$P0mms z)eN{`{tbfS;sX+a_VdVDT&mi2j?vx-Q=WV@V zY8sdqho*Rhls!G-Mz`bdUGbg1Ds>XKh}8Sv*Li|Fg{vvD>1J8l3iHNM9D1qtH7JY= z)$o@CEp}(n$fM>5ZGQ(SD2dVgOm}zTe6IM~6OV}L2Ayz(tOVe)*vjH2vMrRq#zkZQ zJ0_11pXjN5C8>c}y?1H5W_(jecr|X{*{3*QN5&H~VG<@KypZ*uj4Hllc{0Jc8+wgN zg$!&AOtirQKm7i!rz7IlVOrxmPFea02ZdMz?!0u3(X#H;^+rAk7o@ZTGOE88F+v3lUc52=pZSpWzGQUW&X_9> z^hOCK`mqig=3hy*N_Ab*seVQ^Bw+b(4-8eEx&@0zqyO%!eMioIE z+TLV9TIJIAr`)2iKLRg!`rv=wB}{_3f49wWciZEw);e_}JN7XJMvBWd?pz7u%Xeu( zVX~ABbI`cgy+14IENph>;!SzqYH3h$rNoiIC@XbdYHj|y6%WWv%K4-PM9aO~9^h0G z$NG|-m?>X0pka%$#3s5qWFCxfuxvR|r@|!l6-be|OLh+{A*rxlW?r54K2p+`IrLxu z_m#NMBVz`cW~x;u|Fz5=j?GKsl1lX z%+L27B2ZRR9o<_^O-0;r4KdNi-sP2Q;@S0+n$eBr6Q7I8Z`)3*yTvFwPCRe_yZ1Oo z4hdU4hq72cdNiXw$iXw9 zRe&7y4@$NL4kcXR#FERbLu;4J9gOUKAfH)ybOyCCDAq0lVDE?{uR!qiQ3_tJyYFHi+ z867F3Bc@QuAE004KKuxhW3hhcp25O`K@DPiEZITj4m)b9=YJUUqjU?zadW@NWq)J= zh)D19O(cIAA|8HeSx}mB7Ef{@*}DiV7ua zYN!XjH3BbuuWGB?q(1>OFk)!xx0Ht@x=meuc+L5% zf;vy-Mhh(#URdKH5ro(K*q#CkpTykK%J^^OGOCKtY+N+n(HW~nuh{YH$W$zpmj2it zN`rt<@VCc^=$g|q`jx%`y2fvDsMT(pFySE^JUC(1 zsi%(fR`-@ueO*Za;F%>HS^txTBE_DG;(HEA56m~`YgRuXV)AnX&lU;_TNkFMK}m7N z)N#e(vDr_3N%WAd!`|eTY}LIe|*O zLo!vvU#d6%DP_*MGIj32=ZZVoOF5-OwO4rQbeUdttl^FT_!1R3~>Nz*O@svF@Ao00Q3yZ4JS_n zARf2OBkV_h9+CZT+VBg4k&l@(*$(suPJS;x`n^#z;1^llGz1h%`^Op5zA`kRl~+<~`)olXd;V3k$L+`uxoFJ#qpFW8x(oYt7UY2c4DH zsWZIcPJUKknlaiqw@stK@hU@AX0BkTfA!=%c>ftt0syAfFV{dO9R!Nz zrvudxxn1w^Uf!GPon%|( zbrfU`4Y8u=p!7UE=Vj~n`BNzbfBm+>Kz)g;4_VRveG<>2O(!o!w(BluEwa2kY$PvB zejb$?c(W#pdYz1#@G^T>22f=xpq=_-uQFKveMSCWH(_DT@U3aANs$jYqd!Mw-vm4f zy=(@@>wV_7;>r;=%rq;R?+541I550tq|Rz>#!kI_07RpteJYT}L6Udg8d!FnyBA13 z>mpz1_K&O|>AiDsRBK~;acVam;j$&tWnlwIlt%@ALfqG;9KO=#h$7N*Q3$UE6fP{E~R!u>~w3~Xs;gaIhm|EIL;NuqmkHiVw4vT*zPCp>M0Q9nPV;2Yt4JF;By4hU= z%@-&La3oa)iXI73T3S+lL7@MM=V2T$3#js1wy`l88UMIXPTv1+t*458wq0?Wq zK!_X~8Icng-(iy!4}eMVX4=9GMZ!Cm4!5>Y0aup|IU zIOvQ3N1Sxp{CWU*K%6D1q@cAxpWFoaEz4`wAXj%PSA8XO#%9JbX|`X1>z3bIbXIG_ zg!3r;`H8F^vJ4{>m7XVw=sK>|pfgxooA2Mq39;tg6kSV-mDOyXyr$Wk{zBhelELQF zAnM64J>W)a4UWTC8XZApv)Mk>ueH(qMoTBK{#&=GUq;^8X8_-Mr~3&bXSqk+_n)L6 z-VmUR@M$f19t{Ra^@**p9k7zaW-2Y*BXo&jvDa&#=2_+L$cllWRBE}_w@-oimiuFt zABQ$cFn#2{srXUN3Xia39E*V6pmojKq0^9c&1#!$?f9E4*mY^E<6uCjTxuu}T*6}x z_I{O38~jUiudJ(;?(RIbppvm*499VCyw!%{q{FDFTH8&&VgG-zrD<(z;Kh9J%=GSj z=UZ)$b#6KOJPnis;?vgH>H}DhgHFIpNlE%A8H7qoBSprOETDD=D&#*h@97f19VKN) z>kK(N^SMBY=#*6Y8g=+Is(Kq9i$^qaFu0f#jRoYWrsmd8*UdfRDsbx8=4@&pAUBM( zgtgN=mQKM*j5^+@d^;kFLfO?(rmx;!9#M2$deN`Uy!I^DC&3p9TcdnH_n^3yu;cz@ zAAX_wIgo2R!pM_zKQem-u!Et2N$568D}?lJ{G!kF)-4jECgv*Dc-9oeJp zq9vFI8ysA_w$JTcG}N+u(4TJwDl_&Ec~gN>c6F6o>5R@49K5IH(%)WJcBLI6&3M|s zlX444JKg=fI_!C1K(@@~$_@~zRB5NL9e!1D_cd*-O;bhHzo+P*?Md(7yp5{%Yhr_O zVq?OqMM_2$go+GA`3d|NRdCXIUsL``*WYzoxRj-LoK z#}5r*u9GLex3an^=%ie1FittPF7qWr_1&QI`Py`!YR#3ONW&61BbF8ixb!dCfp_v&)H){*x0K}_q5b;2D!b8pcKEGxeS5^0ARZ2jQ zvt^{d(LflDl?qoi_3**&$;nAV(f_6iMj$}SKD_c5~karmFyPkLU}(8y)-AE zrXHjZy!6F1@D!m2s~UvVFJBgd!%5wnrhkpe7#lO2DbfMK6Pb(v>N?Z28`sVo0s{1; z$@ae%$BkVd6QTPz`I)mfW`If4J-e4L!?snb3ZH$6h2qxa*YIT)3SKHQFDf!Ka(iOA zANuqhc)!GN*VsP7Mu$ExT4b}la=b9`&JevXX;*YeGnbeLv}Nu6pT#E)x!)*`=xeV=J+lu zih(76ZnfAzMZ4$B>m<7^NTwLUag`m?o#d1_yNoT<_L7Q~_;wJ%}A_qrg@LRcspA-u^#z6*{J z1^55deq0#ZpR2jQSiLx65fm!y^br1QtdG{m9Xe8xGRpqM_>u1ElxeY=`?+HeV~(52 z^qqu+PZ4%;{K8R*)>T69x9KlePUIC;M{m$E0A|Rq+~H{Fhl-DHzpBLQq+SS9A(qP> zf0JA3(6wP15RK6~SQ<^{OhJSm8GtQ0N0AiN#4Y)-$&PHjuN z#h;^mmPPK-qK@&EP1H?ES%|uOvB3l#4*kj&Y9}p;4~SxM4cg=5y~KEcP4Xs++n#pW zd5W+CPeuIauaM`it|XOHj!)z-mW`pr(Zz-c6f|~wov~e4eDW);F+{n10Nc;;!Kq`` z@gH&<$f^Jq%6nI(RdVp~X^b82?Sr$^<-*jatz*4X|ES0q8BI-a3Rq5dLBZmns3@%R zdM^}pwie!QL+wolO4dMq29M=WatQ;mg34$ANyDK(`9r*xi}^t(2q@xnJTbmcNH`1( zdiJYb?0}b3TG!0)i==}4R&ETU_s)-?gn&&BJZc^HU%W;X!~GXI>Z|%x$lg=FBZi{+ zVzkQrXcrTo%S*)A6j$5i!$!WD=Do`Qo#f~=pG0XD`cJwujsB7b;I+sEPG{(yO6xNN zwW_Yh;jub20mSk>uvIK8W~fV?+52Y@(n&0EF7|Vb?sf76NF0~G5QO{VbCVW7KOKiG znfSiFG>#k#SOcz#L_p5t#*A6Km8cjU_DI2nX|%Lew(Ylo8H4?v?NZlp1*fn zVBk41wAFz2QBg($yeM_VgW=C`*$?F89@ubK*idhd;UMk7fOROT)c|j4Uov{s@4M_2 zpf>=VXec6QzAtWFyKLUla-sX;@XQma{FEO6`k6jpe3mo5yFxSse^<4%knz&JfM8kq zVr?(BYNcnGDNeN1qpqMKGQx!dqUY{R2Pri%*&96e+MM=@D<@`cwca_q9;rQ{y8V!r z`7BJQwrgAu){!6vtKaZ|qYJxesBzS4Ny_uv1Y4P>E*93Eiy$5L_uPdc7w45ePeeqZ zwbr!(z`szWmIsspvP2DF`2+=#o1IZLeRU)&n}1W_oU)tb21NrlX2xjRGk?qJio^`* zmEBWaosMOD)FM%OT)-@>Dz(}#+&IUo1&Iqsea#X!XB>h?XU3)&9o5-v=sb=o$qtd6 zL;zDdNEs_LA05ep%R#Qna3XUG3eu}HqJb$=q;U{sZtrlmVDL!P_FU7nm;`m(hL0#* zju_fYy%c6o488WG;&p0>5$oI5Z+Laz0KvD6t^6Jkzt?pnKfn&v;8Syx;fdIIZ@Y8)$n#gLB=KseP_tl5wB;Z~|N9 z6yBhTq+TC{DHF4-r(BN+QUhsJ4ET)4;X|+GQl86N3BxY7oKD40`fR+Bsu-5EhyF0H zm@i*2>vV7n>yAZk3cp{wuMQZ2O+Y_E5aoTEQC%6lSxF<82^-U2m&)JT z|A=3xBkF-xyj2|(35xb+4!2G-K<-!zjsue#r*J3V(z7iWXn)xkmB=mysW_ph_*rmZ zp;*QRu^7~Pn1~qT*Pa4t0^7l~7@m&<)|k?laXlS|58i0sq>3diFoPd-OSz8Gk}1>8$+`#)9H zxUO$OzI?gQ=eo*JXgwCBuh!M`Is+UbPCz((Q>VOl?nS|P&w&q#FY#h2>L>HBzfoP#BZWOuXlXvBDmD zy*%&9+G59BTJ`$P)r!7lZ7ee(4Hm|8z)L&98sFHN8UI2&24yQvaGO+L*zfJ)X%C$J z^*h&<{j9Z)PZ(oixB|7`S$Q=1X0HcSQ5c?yG&63@rZ*I4AYZDr7CHK48k8S<(hLH( zPo)!im#D`5!;e-V(Cbzm2rMiD1+X5EG!Jm;y$c)6^ST%j#(-KMk;E?V7m+zbI8p}* zPrGftrxk_ow~WxBv0R?DGZHU^QF&7P_a5$gPMY9U`?iv#6SR!D=&782y4vnfO6fjI zt^P`XwT~Iz&pdS*tr-;NcfkpWSas9o! zCVgECyYnAw?JX;8fpT|u(V!Y}JuL^ceQKLoc^xqrPYf{YWp1z82(vv>dQD zv;(9&E?ik&j(w{U?}mzR6B5lFsP=#^4W#Imm74C(S&O+^Fvp{yN4Ek_8{xGzwxE*Y za?-E1-eSh9U6#mV9ysQ2*#1IE)fMi91HI7{4gv*<0A}!9` z7U7O@S5}RivMQ&VSX8mZfE)K6a`p7&0{7%Hz-nr`<@?L5Gat+n#B^WH?smj%`-t#g z#POO60EfubOUd!`W^14*Ghkgzr?NBG3kDTyv|?bD{u*C<#MBouZJl0TPFXoQ8y_3Z z?GH5ky{S-}BX>&O%l&Yfgt)GN$_sA5v%#4Uy6s$Ed7#M!fbtyk^?*}oK#DiKTfN_) zSPu(}RzQG*p*%m|Vve`bBV=Vo1>}l29`5{EkGjS^Lm)|Ps>7@H&U+zMn?r;b>%8*z zySwbbQN?ar(HLD(-hbWe1Srw$>A7}Rp4C=@F@&P4J6*t*EjcU?E{uivt<84@$;idZ znnt7=JAHpxKKPpauu)Ja^%1J(4+&Ce%64K@#bOl-*Ux6L zz~PiF5SikLtiJd378=hgXFQc%UY`$kB3%!Xt(GYI5a%$hi46vS_0r6DwN~^xHu4zY zRIQ9_84`B?K1=WFz_`dr=ywlmLdsKpszge?Zeb4TZV zuBQyr#4r;uR-3T?^vq0z1H3cEHBw&Ji9LSp=O&E51(?Dz3cd$}o;|G^XZ}eSO1}IP zy5E4AsQ0sli6~C`o$>)q&dkzsMcnP^gP4q}xyQcFkN@K+YZMGJGuxxapvE7sCC@CJ zP$}HssnyU{%z9`NMt;STlrl#LrY7HnK7U5e{5{oBjcrTNJ;eup#Vaj#J_yTP*W0pa~2DE+%iB}w6)lrFD>&ynYhI>`M{oKb<=SVDLJBF zPBw+Vs)Wx!TgFwxpEAj3c@mnsYGnWFaTLd+HwpKf{gmhlGR5b@*2gnhjWxJ@yo#=y zJeu3zXxS$qhPdLI1&bcHg)JW2c;$>%k6;n^E%duopuE*qg!%pO`?e8E)Mv1`vL$9H z89{UaCP=-(k_GHpPsseajOhHjOGxq+0rxG8++gWfI~-#<$!(o zCP|L!o;(F+bY~mi7_fWNB}3izqFt)CR&u-f)qYV90c5_vo4x@mhT;+uzlMeaMlJT& zHxge5MclamvW`;1i2!F9;k=g)iy37ZRKQ0q*byLJS$dfXF&XGu0!H+A>eKM`5u(+g zU}%U?YtFKkeS!0Q)!whUgI~5}+I$Kht>Az*%7rGszCe*v{k+I^Axe3pEK)B3kwW#$ zO}U~gsJnj;uyTNQChlzcj>&ZzG_?~qr%4EFvjX6rq+dCyRDmFtNCn>1t_-yjgLo4oR*KQ7Q6C{kh4va&DdyydLx&z^Z7CDNM#v4B;+5Z8T1;(+xV zsLp@@1F(hx5HBrf*LFStsK1&rO1KHUs@fj*!NI}r@M!O2g2F8q&w1mesKcw34-T!Z zt-!3eM8%}nd=WprNS{@qT{95$luxA(cD;z}DkK8Z%@6tbKBv?Mab!Zz=cG5pa#?Gc zw0KjFPD^1KEJ;MAzCS*d^%!vcmr{$7oQhgpF53e^}%5c(9wE`5E z@PE2=EEk^Ef$K{wafuW z-$ExKRH?BAv2BWLT!_i~wa@Rg+!bs+t7Gc|{n<=1GIIerq=Yom!8?iCApd z@w!eK51DRHbNO()c}8&T4L4&+h;io^re5c-Z6V2gM=MEF2al@}DTwNm$Mi>24Tx{BnWd$fCVTvC*E^lr^nFfuKtyv5q@>;b`YaU;6Pnv*Ft!f;0sYZxrmF zI%~C|_S~~B!&t+=<%eOeScob4lKP#06ltvicaDux;-Hi)`^th*J8Cjh-k36@dIbxbsG3qD?}~_`SVW8fHJ{7ZkT_M{>kkbL-gzomCsXyLA9U-_ z*gNHZKY2;=LE8gryMI{yRVr4|be-QRlL{iCFz6KimO+M#Yx#j=gpH*7qzkGRwiuM zvNlhjzc9XQ&{k%w$TU`D^k4=eK8Rti|6=c1(p=HAqz^=NT*1nG&90w+=O1MipFAXZ zhBF+%NHxJ1=*Js;Y^FX|#P|wX#uSFvnWm*1UySzxjAoU8vXFkC8^0}{$TZ5DPEd^( zXP!D%uWC;cF-=?9(fCIlsMXHW7?kC_-cVP*$TVKPA^-ETW~f>TO{%s{#omHwgw7j2 z7YUZx(~UC%$C1Tc)2{OHJOvXnY7u(@(jT6exwzkQFlUkTp`arzf=`8uETeWZZtF{I zl&q7}6JcQ;OiX)_rBQZ#iV{0a7U_<6yszCZt{`%fM8E*P04|mjFK^~C3H}vuszsbT z{KC5+5n7;@C7 z8yg5oQ8zU0(QF3Bm5O`|xQ+!+;#iHbY)?gvD@S6o*U?8OX(HI7y&`$?wD73y$_p7A z6D6(n@Z6TW=J*%qZL0Hazjn<~JdH*aV@?xkVhw05TnV9M*L-=H^m___mgJNHOG;q6 z=6NsBZ&19`WCu}U(QQIKnZDCB?P9gGbG0T3$&BZa7u6dl8dZt8F5oqV)u+P+!jw+#LY z>q&U3;ZG10_~srXX#*LS2}KT}>7%yh)77_@Mo)Pta+uZKqB1m!4gL7&ezxHMS-Sj= zjwD+hP>O(hk2%Yi?i=PNEgjfJ2GRHpUwdzs@1c{YDsN|ni@kv)Z}W452M{Y*djT{h{~8ro{uoWljqd zH0+nR5_R$xwm2){JLA5}Z>}-DQT4dO1?#H`F%IDl3Ae&$pVh=627_--z*=*nsLZ*``#vue-FK_o`CqHb)RtAvc9A|p)r+JwWB-!2DAkhT6w z`05wr3C;cB{Sm>`7oJe(N8f_m!-`gTwBknoTN)eox^v zwdytm(n`X1*9KlXI=ioK9{%vmH6NPgWMV~`yJ$CY)J}iM8fYWDhoO>-_JMci z+)sM1aCi~{1aO1mIJ?>;$dp^>9U_*`bFYHd7NnH<#2($wdQLb{Uh<5NBE+jGnv7>N zhce5cy@1T>cgX`rXKt!FJx{KuLoQl3Vah;>9!l<@bG#KOU*Q|GVjORk_^pB{CZ8)u zN9NrTlV4nefkL?9WNrAau?z9w>KLyG(O08lBL@@^jyOEV4>5Hi2E`lV-*)C51Uw=1oXRIlVdM5C?+(>A4}U)?zL{|BpwnrhYqW%LmYD z(UhYFwfn|D2}RiN*)!G@*!?Vy4hgwWb&ZP_}Cx3noH2m66JK`Y<; zTlS;SCyAKq)#abp+u=x$A2~T{{U#-)$)3o>h&e|SlwVtG%eAu}RH-m|uS;VRjsZG( zyTQTnsQW1-^HS>@Ic+vSrt0jTP(|E;nkbsEoCMcR#*_U})fpNJ=#MOEDu6#h!|}dI%-`NV7WMy56v?6d}}H zLgJtB!j4u}*}P|Hd-`y51{Iqau6DaG&o7)@|Hxde;>FDXTHh7(2}_@>vT>2gaT?iZ@?*+l|1w%jS(tK!or?q%oJ?6Rs${A$j*@h zFZ%I$(0csx1@a`NbK^*%{QIZS+&eay*NP{;@2A;4FTG8aRiW;e`A*p^0yFA()+xy?4 z%KwbHo$!UX;XrTX%OV$63Xzl(aWoXFb38lN*uV(Ag$ z5>HJOWe08EB?|cP&_pv9Ykl(2*((Jl>)DDJ=0uf<>WcjQsI04l4E7U%->#8`-lA6? z{xuG3zXIPtmmgsT!toQ+NZ`j-8Ki252?yL zOT0uOu!~Jquzzr~gbJ-RM~aw5a2DFS1$2BC4UZ}LO)WhzlqXV<&c-0pXId=D%*y6hMLL^opO&fD9i zqgv2wEznT-9P%2(?u8h~3kC|oTR>c+uN~5@bVJS4#AQ?=QD)^pNMP87t{(XD*G#|X zcRrpQ*5p#@?={&we-UHD{LP@^ARx&wQ!dqPZh3S@a+G_@nwZeIolEQeV+!c*AU(1R z)~mml#;>Yn!x7&#Ky_Jtb}7ci96)PP^kr#i8QgNdHVWy>W@?}BTG*_X_IuF*nN%yznI8o-wp+6t`Kp@&*d*X#b)M1`z+XCsfKayzNW*$BrhN0vOl zf;IJ|QuL%$*Y3)IfZD6hrSIVlN*BG3>XfvM5Kp^UyguPdxbx?mkb%@&XnNyc&OFz| z3|B8BQr)6^Wvv)OLOKN?0t_B4s{%~wT(PV^YYq_4?E&nIUYFFxLsJC5T?rJFZS4-n zNFf!LI=-`_O|{gUw;zg*_U~K{#O?vU5W6E1I7>a3CF+CvrBt6vCO(Q1hg_N0AI%K{ z@M)3-n~6nk&RyA1kV0gYjY;QiZ|6E0sfLJ8mZ`O6AFi$QT!Bu62Vr9_9bxtW?qrtM zo`e)`DLqz2X=7eKXe>4^krxADSY4$(jF8h>(Zq#wZS{6J8_L4RaH8v=J>&-wH)TF{ zuDS_`&YijSd@n@+^pkQ3V-2YD8iHZX-R~BZLDUa2R1J&gd*@FJFy$1)Djy_GRAe|m zYb+UDQ&riz&rQ8nvQGozYGBNn!ACGkLY!lsKK8avLd ze6g%6#BM+(vo&Fy)Hvoh%4;jz=>>?A_#*|tZo0y9I3Tw^J`C@-f=qeED!AEio~pC( zl1@5m(~jWzkr6nOh$i7Ae>bR{tn#v8-l=Phipm;b_nBeRn5=!POyFRmx;MQeEYI+u zW36jJQ(ma0>G=nGAH6zzjcA&EX*i-A}v%YuG|ka0*G3fEsw_u_2TLttZK2s z>FF>R=+n5PH`155ajkaO;0AuPXFO%tJJy5eW73O1Buc~5CG)^7(d+oBVaMQ{loQRg z$#;KVw$3YIP^v$`t80fjq)g<7RuyE=n%-s&#A!(pw!Kht-$;@htGPRp@4L+SPmupT z_+aVuSWa$xZRO(4j`B%r?ITvIh6nG|oE`jGTd{HV14=YF8V_dx6zMAF1hU31! zho%QH1Q3{{0j7Ty5EKTMzO3|K-ytGjG6Huc^lxQ9*p|nzh=1=6AO+2K4sG|p;(IQw zm>ewO>gYYhyZwCB2>{Ae_$>AFQEnjEfEl`I-tqE}%Gv*03 zpIqUtJ6x~zJnR;!*b#coMfOrbsiro(5lfn=AM_&B)OY|bN4{dm^kjziYb+5v-o3%W z!DLTn)qIWH&y|gzo>f~5)q7oBii*n0I-&hTydHx@oRj(W+nUKkv9KF@lbL__XG{E& zIv;-8p}B&4;djhVz=(1xTp76q)z$2t9UCq8?>+W4sH+Lt zjWB*zM%C^cp30niv;d)Z1{}zRml&p1uf2G|ea55%R&Kuk65(V)&2|6g>pFvU6O|Q0 za+}6d)DOdq#OXQa_O+uHWrNQACI3g_Fj)z+7pM`MHom|jJ16+4ytf)zAPWI~FR5v1LOO1jsDL_GxmG_pa{{W@X?~$- z(BzhsHL_LIHH=u-J<$O>oJ)%&PsXWMTj;8s^>b=!;@*Z}JP6b`Jf6fvDA)FdJ3~hJ z#II*bDAO4B)=Gp>UEKKye-0f}Wu~eJ>6rBVSd!tYa||P2d&muOp!f@Mf5gBxrFT9! zP^yoY9{NbRRrylmzAtEd{H(e9XHxM8kNm2P%YP&J8**>pU$v+6W{+;-8_iyQ9I zoG!UL2g}BklxY4AR-!XS2w|XkCqAy(KZE6GVa1-KbG*a3QpZ&R7)LgPuPnLVmVjCV zMGqtWxr{%aL65o6BrJG(I%#ZL!oll04w=yZ49fybt%@aLSrfB+fSmuUmjIlLnV&kt z{T2Vgii&Aj!}in@5V7p+P~=QidJN4y<1Uvoow=A}l9mm45`#KfSN|Vhe;Jh5+C&Y) z5Fn7??iMt-yCe|Yg1fuBTX48>cL?t89tiFdEV#SexHFq`&NK6V^-WFHCdCha>|DFA z?$xVTcZ2P7|7NO`PaFga>A#W2wq2%5ivXgK8BYNB*^1h=iM=fIbC1DpsHFo6YCV9_4pho?YOs&m80v`6QTai4h?M?ryz`5CXW1j<`* zfMbS}I=>HCCnL!o&xtQN@~p zGFvO~M}oWxjU<-(j{|O_`Nif0+5j<8|7RYSDyqj1s@fGf0u!5!Yh=Hz<)w z?w@F5jV*U=?+_h=-qvc?u}|!%n?Bat;|xPrJZ9LJBt&qkdFyb=sN(~hJrUh!unXYR zF*onr0`=>N3)JYHZ6o3Kv>?(tEjAf4f+!r`hJo+ict+^}K`Gn#lEz-=KU;>y3Ivsx z-Lq^VWf^Gy{xPq`MX~@Kz*LJ2^0Zf{;1JbAc6xNVdfXUlvE4g zr1*utM^9Ende5)eQ7RFXBC7JW+tYB6{gP-@kg8~}DO0Jf)>pKtXuAgdnZ_hr-sa9a zB;eGto)4;4s^?NJYb_$>uMOfx$b>>Qm+sNM0V5h4TG6i8C2DgvOd9FwX}G5=A2YpA zdUq^qF7X6RjNtM-{R9#1y(3`NI9m3EzcouU%PKaWgsaoS;*|vdgJ|ZfBK3D|IspAi zMOyE-_GfUjs<)dVIfE}p#$DF0@hWzf^MENPw`-pG+^1FmlMp&Ci=3HB1OX%oP`5UQ zeHSngTJ3)_YAdU3w#4>o6?a z6aNmH{rko7F*At9bT?p_efm(0g z2IX4pk*lUCJie`U--_t*xsG;VxX0O@PAh$4dJJaa7_F1stMwV8?S=?)eMI+Zx%jIj zk3VT%Z9!?RQ?DUl&;X{Zw0yeH=F3tTy6AcVhlnbVLLt=?1o6d(14GNE=I1AhfTEUZ zi`PYo{qLTqmRYYC&PJ_kF96?-0j>qebex=T6?9a-sU-0mXTuUV3zuq7lm8p70tXk| zlsBNS?IbZ}zq1O+ieg5gk^~*h^%L>&WWWG^f)l8`;LD;tIqx?3D_`kVS474y?J|Iy-A3DNby?#z92!--y}KwMO7&+SA7u*fw*U zgZ0Z>tDO91lARhmt~e{+T?cN_@&*n1KT4{Tb~&+4Yb)Jk$SY94V)INRQ2I+z()FM2 zo5^HX-7XDfvq}a`>J*d`CZe=M%&gCZ)T7>n z&fT$=$5A=fLLmqp=BNxInZfX5`>pRat$UO3 zI6W}iO>&$1;J+A)eCW_2ets5hYguZVt92epv-hF+#se^8Zx>%^8G{b=A$*erUrbb) z>(@F$hP=s!R?FK_L`(mff&5h!ay}sBWZXPnnqo)oq7`<}Np7;CRt2DM+t(lzMQJ({ zpjK2(d~+J~95Stjq1uKwR~5o;slWJrN^F2n zR$m>fJniwJFT?(7nD;k;MHRxK^$x9nDXmJ>6x~Mhi1TQ@E zy>4pOH#oloDi+~Qqg!{=N!vc@r@DP&6Q`so2-AP8l;T=f7&t0)B~Jbp%s+YcUB`lO z{{6m{B$+fBF(y`R>ZNvCAi@v}_TBYjj@oLZ5B3ogCxL(4h$+U-Y%urPYYVPpN8x1z;6}rHAgAE0+@lmN zI^ub~JoEWt?2g)Xd1tFLUjsnfz#e^?VW|a7fo@N~yrBUf!AF5%85kh&X=w0LGtm4x z7{v=QC8-QqW&j)pV2iz&(gQRZKK$aobZ}S3_2Z>#cAr%;N;B#DKx9Od2-XEjL6~i5 zYecUd|Il7zGIrZ2$7`^>u3yDmJ&r)|`atJX6Ou}K98mj&L!Dp3-HcC8XVFnrIWf|= zhJ?4gOzG0-ImSP!Xc82eF#sADo=;fALGNjkJq$|{x!ghU5b0yXE-2=gj>`RW5eEkII)OnCDFMQFb;;xTeo^ zbHT2+%~No{hA@lFIkmy z^cEbu_DMT@1lPl`@bnK`_gUQ=z&NTjQyug5Ev&j7p(7FVj{$zJT#LE$wd;C6;A@0- zO-#Yx{_8sfFY$k)K4~uo%Sx#oo0FFY1aYc16A6(CHzqC8tW6+SFRd+xAy$-jgFbP? z$Gzc>h9DGl5xf^uFq` zxk*q#;yU-(o?Fb`8c^*DOW1b@j+%iHNXb19L$L#2@r;Vz)ZIIowE>cJp;h7Q2U!jT(0Jcl*4vKIWjtO+%v!yg!`%KoC-c1xNu?qfR2n9r2mnn74MwC% zG-PeyW0?!g%XM*PfbBDEXff+D6K1(SSJ~SesA5yOx<plqhem;rCq+v2Cp)jyBmw>x5%3BxKoYoWAJtU|C5H_N3yJUEP>mzai_Zjf5N z_o>(U;K1)U?EAg;jIM-TA(+#JcSiu7`A;5-HM6`f7tAnr{meFd&ALAE^d7@CE3$S` zmc~LWVV>)Anf-V9t#9m~hrBtKHG33&ok5D8qR57)caoS(om%$V6JpZVt7?Z&{d~X- zsd14A8UO#BM=zo)!0G3>i5rf#eZcpV_0s&(23Gzr$HV=hW@4EMLNu*?XQ~Ep(S1b7 z&d1>I15xskc_t4%ADENGaiR#KCfpa{#boolT&^9|x*+G~dO(SKNk8P_n4b@L*k*5M zrctUKO3sSFR)RTtY+HsqSbslfTPw8y?;S@x13u}}9*xp%blCsxqH!1*e!)KCxZq}s zJJN_@6+?wI|56aZwP*?iUWlc#=fgp)ksYfS!#H4)=ZGaeAC6FaJOXu|aFsWT zsktx!IG$#L8D%Yw85P&BilR8TBPCi^15AY&VAz+cgVFUtvRZ|1IMN@}C{B1&c^uH9 z&6x>s;{#qGz4RCwxJN5|LS%-uk~cqvXS;Zu}%k=D9gF-9uai+Bp_Ot3qjKB92nGNS@O)w8ETt2 zZlZx`36(~N3Um9vE?VU^sC_N*UY<}7%?Wt~<{R;@oo-3~c98G+07yVTA|Av?`j$wm2SX2tdwWor7Upc6!;&d&cvffI}_yN5s!@ zAuQK{cX!`%6nM)D?;dkS_Z8LvuNvK7_gt@iqdQYGY}!_wv6XAk7+M!$f|e~*=O*y-7;@(VIybb$WwdQ0hAA-c-dnPt{?8E zW*E1Kai08)MU6j4`)-97TUZz#)j4AWlK-<@^MJK#4qLUpOxJSa`8@z;zMWl4>%Y}_ z=zqOD6uK4=k3}~Un8*mRfOL&s0CoV*iU3SQKsH!|pA*SbTkzM-%~}sutOr}vY7M_M zPvGw`XNGHRQIxmvYB56=G0-gIO?5Ye(B!aOZJfuvSI~$>V`}sQL`j2>7uVm+N~0$n zx!=r~ICCr&_7x)0v92WTDA%en9HcHgY?;C3Sh%+{BVB2%Gx5-M|43Y@RYowjrRy7W zn2HG?o|n^XAE&=e1Im=V{cEt;t2NCPkg8Dp)F#{_6Vra$Aw{T=(_ZwOTDgFzGw> zXuRBWhB!HB>BPW9nLB$HXyH`Ep8!agH%LYq?S9|@4+vQ-Lt)vG#IOLJmcd@uZ#8SI zMk@{P-@4fU;>F&voqpgx{FM7ZC024CB@GTg6H(3L_#LR35xXw*Ni2wrUTk2AESv`ChJJi%aPO+;CR}pGTrU z1oa#*xW5T<5Pw8P*`#Um0-W^P{f+9-*pTQ9+sn0}kEE`yAS_a*(bfYd1i)zdi<3(N zD~o+EGOKb*X*vMX^YPYa91yGk94R#&lfpW(;9vM(wG@~4(*dsU><*py8M|(XcwGqN zNdCx}5e?}iL= z-Xu6|8j*w%6Yku~$uA%1G!gJQaJolI+5ahHDIOibO(~rfwXBL6W@Z=i5c6ktM?@QP zlzq;T>Z5}-uFG6FZW7&RNqe)pvxl>%7gaI-e&3fn%ruR^i$HCg8yt%m7WKuQo8jdS zxl7=5!m|}@QF3zQU@SVCK>2l)>y0kAT4|>&kjq)LGT3;+iz0fBxVbsqz5y6`C??<2 z(QPdpW#z?2cY52Whqof208TFfOWe*ByKftE&qpU`a-MmA~Hv3utiA=2kXP z>!)oTlwEV1B93lHCnwPUU&p;%{DR<3R$}Ib|AlF!)9f9kcDZg1W`7@Fd8)Hvc4;Iv zIxsSesm^AE7gm=NU$8UeA@MnTTAiy+GT(6Wa(%020NebdIY_mYZT9}$LkO36z=Bnt zoiTmboq@)rjXHZL->VU-PDIA=x`Hg;G8Mh%1*k*=XE8J~v>UZFN6~b9i~lwKH+Lu~JUWh#w2C~z$Km6~u^y{IIfHjILG zs?71Ko)4iQh`hH~69(RIVlpo>e88BV!jGRb;za*oKsjk@8kzMg91H3F=P_Z1R+RN@ROM1h&&xR6o6|UBoMYj`k$72Kl5+CXDk=N|Zy+$^vcjs~ zGIgutWj#j`a0j4|7t{<4;_h_z6?HF9#xFO-K3~{MHyn)R`SUUumwg{YE%TdNiM-D} zZ2jsf(l54c&!D-TTeH3}h2|Cmb#JvWw2%14hYWjzuXLj(iQ?b#=8wp*X5yBD9s8k@ zzYjKRlAzXdiGDCZ`%pizM+{zQFgqD77pwg@sS%RKrqpFLslryw#%`LI($Qrsoe%#z zAAWcu0``z!TAF$_;4i{X(!O@FiO z!@wcZB7>bxOIl}Sl%9I@?pnt6`9-)0Lao&c9O~Wb8EKg@RV?b#7~tneB9NVpUNmmM zp#YD}e+8jZ_SBU$X^!jlti>Gil#U~;KDDh1FM+}e$(N<0^Sl!RvFtqXAm8;hVlGx~ zKC+@T_b06?L!NZGOu}OcI6CY)?}XMk^S3=RS1v8F8+1sLjAEYB@d~YXH9{KKOI?7*OTnt!84q81~3Cpl1dD;-7we!S) zMTl4EzPS%>q&8E4K`O+=TljOj6*@vIs=c zPDmcm!4|M1NtT{qOpJbN>g6EaqpKV}x(0&>cp-}`wEg_5!m)_~(?Bs3 zz<6|ck>D%VsvL7>nqkmlvyaH|mPz9NVAHf#x5(zCjllGDcfK^lJ{q~9Mj9~wa zbNC9_H|5|Pd?imnBR`#-$uD4ATLjc|{zWY)WLLMsw#ZjE{+us0rh@~$jM1<_2c_-i zDIWcrd)C@>dXw8j)fxUZ7}lpVdv?cHQ4aBOa;AhWoAkMjc``m zoZQ00Z?SQRnDlhVKFbJ9ijt!d5SGL4A6y2`hN;C>S@o&lh;iH2!|8UW?UOI5tkEp# zT1}b5HV5sGPF87 z@gqWZ^Z2<0m8p#~?zQ_XusY@IOo`7U#_Rm*`*DHlj3oDhUMrK2jP5Cy2d3NEX6!c| zCsEAEJ4z>9%Dl=E8Q5D>5BpK>kv}Ns#FFR4isnM(@ZAGe&DfkY3_I%kJ@gl(_7sjW zQkOAGY(8$XntznGxlrPynYR8KdO(9W6*0N995Y#P`d8@aX`0TYS2!8O%`j}rJ4l}p z<60`Qr?Kk>5bn)DDOE&5RM&K9(_Iv0W+i^A^`M+H>A{v&E*6*hA8Y$I?+mD6j{u&} zfFC?`%iHka4KiO7Q=q3*n15)P9A8485bM0rZg9{qndsEUbz04!46`YQQJ3*76Q zU955d3ZYdD;>G_6Pa;Ts;Yi9jahI45-{Kw1_-Yg z$7+Ke_H310598D7ldt>)$mS7t{dfBX{JRvQ9@YU5*CfwVB-r~nV6~jj=?3h{J#Jiz zKU;@X>+DOW(L_SiCAe_%4zzNo`^2^ueKZ)ZifSD4b3t^ayDe{^+93;Y|5S_aECYt6 zQnzmhcy_{W!$9oe_;7AWlNVvRA67+6`&s(&sjy9l-ZCqbBHp)*z$H48cUYp1mOPzZ}S|omRmOQJD z&T{LMbmdb6KhALNBPwXY!NA01n?4PXu)a%JCe@d`Fho+T+CVxT@PSWDTVLE0^YyxR zxI22aUy(CrbLo?8DT#5I84OqM@DR*WBVI{MK23>c7&CL6D=T>2o58%C!o+tEY1gh% zuyRl=Zv7mgIz`^|Dcqp-bdRF9f@VJ&{Bjvdu(l{6=Lz>?XX)eOO~FSKI`OJU2|W@~ zM*v59Rbr)ybp75W+yrWS@k4Bz%!31LJDMbVm~I#~}BUWMVyz^p>+r&3JvO z)}xTaOI+Wx<ImXHAYmMtQbk?NS)A77fI>I!?hy5}@&o(u(`=W0f(mW36d$zLC=z1;JC z;G&ENQ?lblUUa=)tH0|e0i^e=Wy8f~UEkR73{Q_!0mYkM*9IWj_PN?p1>_6U)1S1E zcoDJFfZr$9I)eT-zM~!t zY4;*i8yX8}*BZp7S8I6?$+f?p!#^J$seX+C-yd5vp9$Q}5ic)XHiJTmWK^bb>deqC z%^u(m=hrUib)6N=h9jGJ92JXA;w>HGX|0t4aTFZ`vWcH|Bs_Qm@&sNMUR!=#XUkcG zw{JzvOOJ{Uv;-rapV{WFrxr7GMtU}eOEZv1~9Cgh^fMnb)Jd&-s-U;9Isk9W6=~gm8ZR|JNv7 zmm6h;{-Y3WC|+`ek2=6fdcq;|w+Q1ZV7H$Gyo;uM)Pk@dKjx6wR>LezHtL3{=(T`d z^wbsG3Fo|#ak|(LbLFcwX4IUv)G=5o3cVzy@5zWR&7s`bh=B+=zAcT%n5psLNXC~& z9KBlOv-s=YIyJyr~eSV_WyM3Fs z#>oPMsPIzv#nF^>nH?=E086W#ptVY?rfQ93@VZhG`_yKu(Mc*QYW5$&-p)+2ZHXee z-VB0LfOW6&&%@qy*>M^*sl4qa-sl zDX7IoXH{%Sq>9XliH(NqA{%>E{@y88$LRl^n)@xi&P|-Gw?c1#QPcfE_mFaZ@FVMP zTX7nGl~Y&pgn9vMP`2>+u|ihetp=50vZk{!bxi zs$@bbIVLIa?mf@BEH~7JIn_H4UCScKpA9+u$SI`w*qF z2(k#(Gj0U?m!9Gme?y+rvKK^7P zz6c4VY5A?`w}bSauF(=lgOSDIK^h7mFa%mFfyFF<+@f$ph-||yFj`jBK%|4AAp(FSw#x)Akj9dvBk`~H7%QwltsMkO3HpL3^`pnvm3T$>31 z$C)>)Za^e6nttr)%Z|r$QlsmG4IpeQ4L&gDJB*9cAfmTLewA`=Zh~;6FW$9;rxKuKytG<_{cg zIl5YjKT?9zW99JIE7o;nL()b0Zhji_msMO+?zE%1gpB7mtSy?wLo+KT{I7rj2Z<7i zGy8c+ufSEDx7uw-GTPM6&0Cc-ucunYNOuHI#X3o&IDv8U@651*WNzrWR?ifAD4WX1>qj59DE+H6Tk#Hul}r~QvS z!eb9TIKDUs-$4DNc_1S?+utv2X@F8@2WQ&)CepP<24BjwKdvG@JUkO{ImJaDspel3 zT>i1T_!B#;K+5kgc7#8DI4T8TSI_cjxT*T6llcIx(DyCB1||K46gfmJkVFn+Xz*r5 z8fmag`>roN$%cOR&0yuW@r%o8!pWth)y{0RkD6 z(u`0(Dp5*{N1rRch5%?-)cYwJ$}=ptX1sMhI9s#Oqrq=T0LC8XfAu%g!Xn65y(7l6 z^A;NZ06)oE%!R}?Hb)4v8&UG-E$Raa`WmYv%q)SNsNX?2F;Z`npyA|esFzw|u5Rqg`$jnGo zA$YRo+UMoe=aoTvEo<6l0K|8Ulw}O+fI72s#h;jwDR>Wj`h3=;9)AOde)Sbwe3+iW zBF6v~*e8rG98(H)ml3<3tWco!gnnr7S-BVz2>Nhqe?EvQoD$gd{)Tl;L85ZARe#T! zo&ZD<+0knhQ;7PGbP+P@h4ptyy~hjnAP9qT~#+U;zHT2U}!K(RlQZ zRWz}(<>yj}%X!I4xq8`1v9Z)D9DB4lly4{h4SA}K)ivPsN&H*QEB&;TO_=wxu)z6Y z`FcNsnHXD0Yvpv()cnMi(km9RkpkUEAWz*Sb!cj^uk2LSnxTVqkBo)t>@?`#V8E$E z&Bmo@T(3u=UNdbONUKd08u>~%Fi%bkFA1OXlf>D~JNH_3HRcNi74q`wkbt2*y1a5C zZ({}2I?%$v)8@H<3V!#Uwaj3&fxaShXzg{8*gK${&d}|7Ts!3c96irCwaK;`v^}~Q zynWVne`gy9^n?mNofScHIc+zE}pTPt&TA=zB=4VeF5gff?Kf;IY=b7uBg4%ADmen82vNc}U_dAj9LdtqOu}?S&&9H6EruC4S==BW# zzFRjlzsBb3-eB!vsUTr2&H{(DQaO2UBKY} za@wki0Ky!Gv zIgi9Kl~RbVl*+X3s(zqMV>6L{B+$dqbL% znw}$gj%u%LTZdFFHs{*FV#b;OMLzEz!6=gLc;YZI9Hbs@0Sg7yK-j<*1qO3)@Y!ES z@_RM?jX{XM4IBgl^t~E8A4C;Pi?jP)!$X;iGjYg|e+YnQ7s->ZI0W3Qnc-~t4?yYP zo9Y#>Vo#KVU!Qr$7USV3Cz3OmAe=KKBj4^%DaTxDrd9e38bC++U>;!on@31kBLwEW zK4||WE#=ks0uXeKfLPeFVAiZFqp_gf2&bq?uYQQ`cc!j!$% zXjKa$?tx=bhq7_`rg@FM+M&*G8m6-L{|f`mD~JdPjW2jS^_@koOiJM1CQpx_b)K~o zZZRV;6hYs=!@&pxhms#mJCt*s_xgjTcWhrFZwgIHS%&XnutRmFi%BqKZVu`Frk}JB z9TpiAQ*%Yp05HEjBrdz_r9b4Y+i^O#8(tb{yE|U=Oh7zR{YlN!iqzM4&z51ky#Ss7 zwjcx3k5IyKze>xFkNJWJD%C!7Vk+3s5P(xuFQ*EwHvrlv;(mXw-KoF*$^y!qP*b>w ze@@ENYP%h|$RKZUyHI>gqLFKa(>T7UxrSQRR@b=>9;x@-DKLUVAG}B>dXDz9h+bS* z&g0UDB~dj_1NKBFt#ZoO!4XuUthgHuMH*d(YQLq% zQOl!=*rSWYVqnLRz@b^5nO*5P`zl=rzyI|f?gz;g|K8QXX=znw>HF?e(T?j*TeAdB ziZh*|@XtADdy*swQ#ExvjBq}TZXK=wbu=Vy$1F`Z#|Y?Ll&=v z(c&}*5&>UuVF};;6)^F3i#>nZT<2+w15M&L`%jCjYuL$phme8E@1Z$4-#6UlY_+iQAmkJtwR*u)jyERif+?H`^=1AURJcRPZb^9zlug$C~yA}b@!9-lQJ0!9P~TC zFC-{h;?pH@xesdT9wW_&)aI#v#&@* zr}>5bppMS07c|ag!t8qt`f$9+6X$s;CW4*PhU8kX;{4>i;GiR8=T|H{M#mDLio�+9AS?Ou%6kDR1 z6HZ%)Prt7KHaemMJ@8bpCDN9P{kBBhnR8B>eO4a%jRj4{O6 zSqH9d9)WSEOO1=)Jt;+Et9=D*ljWl}x}&fMTgDTr>B-}VFED%m=NAJH?tp4^a}=5+ zixsp!h>)NJx;+UIH~2x1`DiPF-NI4zDw#LC2>KmjXFj25eZTQ_3#_O*qvj(<*q7Ib zMJC?63$uYpa;|qcTSKz%oCc0lk?>C}-$1VygGI&--#6N|R z`xSPte;0>iR{lh`m6&O$6Px*65upUtKaJrJ0)X2ESJ~;y=EoJOfeUN zUw>sdTP@*COk^zNm|Z>j2V7t`cMU~)#mly7gBl&^RLh(tFiA)?{AJ-(z!f7~&tUoh zVIV5oU}7~OgRh_q_GzrCOaA(1$Ots>a+2Efh$5;xYiiSyWm$zQn0BxLR^)prONLnR zx^%W`ei`m*eI_Q_Xu;OScjS$i*jVztzAlZHV|wLEJlXqpL#MsyBWnUn*78KZU0@odu4p1ecmFJ@=_YA zwsD-?v!@6xbfE~q_r1Bm^?2b=^1!vEK~)i=K-J5sK2^c}cxW)H^|SK*lh3)wl~1CV zd50xJ6KDp;8C%Zx#}|^x10Llpne3tcSlLCZC@T)gFVy+2_08HyczB1QeVYg{;Zz6M zY>)TPVrQFR-HvbB8-=mtEp}a22L=#5`uwM*B{gc&*@C3;(ac9@#Qw7*>dil6qmc;n z8vFajKdFcE)h!)`^@&1Yc2%*YNin%w7}3(#W@9}u(dd^ih@ex%gfr&{;SP#~m4w;v z#j{PmA7e73!!Lv?umTpRG!;rU}4$8*(h3s&4sNpmMPdAkzVaT~9{RI%8) zfBVHkqGa43iJ$gA7}Rh%H?Ws?T5%W2X67{ch9>K-$QtH_%=N+3G)fe)@?9U7ZH=c! zvnXMpH#F=}4If!r7J8NYheu~5(K@wd4ByPe!^>JcV#SBJh(zOBva^phlU#gX5%n;f zU`dz}{x*-}kxnN@0kmXj|sHSF2T{8{deQtG;}GhW;&oeB<#l!r7&E zePikgd$-snDA!8)iUWA0`Az{O|RVO$FD3+;UowHkJfeN z?Hzx~>5`2zvT1QAoUhX5J(%ZCM*V+2@FoRcX}y510Wm0{P4%Kjl3JrLzb@?F z6%o_E7v$)5hsFCFUGL29ZW6GY|Et?YQ~nH1a=DfQie?hMwXq)cpyrb6R82k%MA>YQ z@8f0FMdsNvK7TNF*A^P+DeZe-`&eZ|Xbo1C%Z0$BK&8vLI0Wt`1M z9hjkqm^Wn1{_qgYy@J0ft((xjiYf$U9{k8k{z%HobzhcQ)~#a6l0nDboL258nfn_E zXNLrvPBTHf!wqHUkITiS8lk-9NqW=(AQg)Sfy5%(RP}WiN4_9@>NS_Dv^I=$ck9I#pk_4{G)7V?;r8Dk}wWM z^oqiMT;B1pJIR#_FRBNpje9?7&eLrojwPDDpaT=vL$k$8Df*N6tSzx<{-N{pKOQT9 zr3td1UFZ_OPS7e(Am}s#a^z6Ywk#DNei~$tcz=9jPxI7hP_f z13swAb}4LWXMI`n=@{vTGqwQxHEZ?;0@M1bh{zWW16;N|9N=^;?6tK?Xi1X&--S{p!ZM5b6qS zIB3JmO{pa=V=luU`Dcr1VPee>znA` z{W_?3yd~rW1LW_yZI@e?GVR|O@-QH5LoPVYw5mVjddGrrYBR>S;ow;MmsX>HwRs-b=HG?j6wGlgf zn-0)3VxCK%{w`t3NQ82Gsze7lv>d~rzSld!EL-qhhwIkMH%K=ekOPCK;;BJlQ8z&Z z{63`y(q7-N4UQ2%_1kr#D;PQ%KSrd~`1Dv@Pjz4oNr#lga>Qq=D5Aq-*kWpq@gX`8 z>Q;0m4g`l+R&mC|+K;%*X(Wv;_;@-9LreMi%*)#ArgQ3u(P0%e*~a`9lU1q_iG_lV z&M~JUN)8Zw37CMiJEG@2@9=%$1^_lJGU48ELo%>I5jYY}PTTcXXUmUEubat?cDZFC z^J{KZ(xL7*_w$1$j%pQHFBcP?TKhkFX1$Z#_?WA{;RK@m580+L2Zu;OC&-^B#K;?D zyH-LAGpVU_d39HF?&pH$p@`(BDir?KDBwYvpi^Z6yxrC0lAt(=l206K{18^sPM*N3 zyQ{9gw+_(i&TrLfPPo)~R=5|DMNvgfnN&}|S$DhYKdW)PV#kAbw@ds=B>1=k4;`S7 zL^LP$gF>Rv8vG7WG*#9bs(JR&2yIaFx_YaHQYvDgG^!bT3ZmHnejUYB-dd>zd`ooXZ7F6hcBMl~F&Jgwe^Xmr@@y=~>azfDz zZkCq`JcR3-Jvwo;V;twJ4#h+MF7a~Ob`#wg-s;sI4g^u1`<Bw-TqW=c#FLpLc4LFaOL}m2?3abrh za9^G7govF{yHs7v-IiOcH=R^I(iSl5DND)#9l7h_yYd!8o73k=A7stOPANx; ziIGVB%FTX6Rc0m>lK7!oyNx?k1{Y8MwWlqyp&f^e6T&a@7+7qN1B6}XTRa@rS_A8P zF=sHkilDGa|&#jF&^TPEE0`mS^oK|wi!QsN`Ed5 zEVu_GY5o=wKkz8@oTd?qH-tEa*rEU6KsGqEi?GZTFMuUo_&|iwRs#iW>q#dI@QU;ZS$yqqg?DB~C~?9{lE~+8r;8KNsKP4S7~+@hp)gp9Uz~@` z9ob)(5;ynwcu&7)5A<=q>|dIt^^a&a)&VQYl^Ur3{#mfna^5MJ?fmnuOod@1Pk4aq z%AP8F)yw;<87>QS%Jdmho=A!U9R*U`?|u6e76!l++eMcTAo#0fO z@R4RizA5w#iv5C4U8@6)y^FjDvQZ)^mA_@1!)?k0qUO(i*8`aVh*J^<92^o6PgLoy zTAhb+-@}w@xm;i_bv>O$r9pnw#9pPxv6m@V9H|kPhQGJFSU5v{phrBpXsIAKH!gt< z?dh%;qZDE#OHQAEAJ0ozwWB=ayaMBVZHfb`oG_VLC@Unp5DB*NKgkbExiU6p+8j&9 z_*y8b5N!5~lpQ83TpCCHtW4zCx2jqoWAIBP)B;6OV^kChf(q-$`%o zP|ldy4%{(BaGMP2iR5xixyjQRGNc;N<~(RkN-msyR_Tu}TOjW40i30JyC`%98Z*fO zf-Tw*`PuP;YRz+tT1#$b>tDR!5V5;~v)7D?RY!QxC&%(Se84&>%95m;pF5F}AglAe z_4UAsC!R4V7VO&iIYdV|I(^pZOZQf1B?FqvMI>lK8Z>^n>9)5m@yO3 zcE6><2)g@Z44V%7QJB%d1;%(#MzHUrkUW9`2o5&J&icp>VgiG{T8=p#`Om$;LTNK5 z?NFqdOuEnSyn`2}lgTrATm$Eo>^u--3ra*?>)0?wuw3&H6kzsh1M6VhHrbY4!X4;ZJZl+ifQKWh>Gl7h{3h!MNr3Xwzqseh2-CP23Uk z3>#H_nl=keW2{}Iv+d{zIXOGtm>kGNKnyzf(5}0{gV~DzGY!O2rwxkviG$6p15Fcro_`Jyd$0RkU3>gi3DhuUp z{gV8a{2lI{eyFPM5AM7{G&c4_8_X&(#!5!#vIr~{g3GcW@_;`#3HM@K1U!G|WQc=2 zI*~MHVA4<0`oM=c7$7UneELvkVfQHY4U3FnbXEyN17l~?;l~0wd^iR%@I`e)<&rSFBgNz%vcuhf!J3<|}yWE{9W zL0=d^{@?O8O&W|1l7+I0vG$Y-vrmsn066{NIICM-KvLJMg`uq>vo)YU&S^nGt6TZX z_KNmMpIR2l*{SzYXixAK@UKU&O)BbxYzIA$2JuZoKzHr~t~Nr)>#7`f>pB*L)i!e1 zjol(wc%O9qnvCu1b|x_mTR>#9Ci2w|pR(LVq_ULy0qB*cEtE^1)xX4;; z5!KiSuhv;!QO%G!$2d|BHV!Q=CFNIyE8=-v|9rQ{rh;y7Z~Xh zv#54F-rC>VZg5<=LW~OPRckK}OQiivmBE=R&5V_qlZW9AW|q6)+}EjHoluDsx~faI5J2k^iY%m59!ZLE!3_ z>D|)V6D>f27!I`U&B9RqKaLooRO3|TLvpjqCw=};bOfJos-OcyfwU^ zo>MRO+tcSSJQc3^?)*Ea7)?uIxhaD?wFn6B-L5ux#mN;hm%1+6Tyb#V?E2CY0XnSx z`*wx--;_^AT&1sV3^pa5Ws*PZ7`?5eQ7I=Tf_1n2((uJ4aa&oRGe&PzR}P@p6D-YD z@zkws@@KaPXrmd6y%p50)q8&8>1$qm(O*1Rh9G4Rma<8?@?z|&C4?9$V9%F&!O*m> zbmFr92XaM|`s;-<7I{AG9}yco@_GV!$DjFmWO`f<5s zd)YRYUCXv@yH(3AyOx)2doA0xZN0ztJkNVhXMgR~`F(Zo{ovlz1RzhX+I*r>Fius| z^GXI9g{wd_Y3VI@#~SH?@IE#3z}aTH+s^)O?`A10N%jKClo?mm#IFMxXJ|NRx+2B$?qbSxI7?x}%3DySl!D#0+ZKpS9N- z5dwiQ9f&P!fsEi6lifz}h@U{4jAzZ)6ZA=@G?UE4T{E0kro-GyvZgB+ER=wavNWz< zj8Q`I6Hx9YU)KCI!#FsxYRo0OTO8Z?@coI0f-R=dB7@fj!#|dW?JFkbT#8mK)%&~a zp0xmQ)>~R|mNYS=xz^K{-LBW3pW()Nmb5-9uq@BJ*^(Fw%3l%0u3o#(`d+PX?_Q(| zZ*R~4p)-6ycFC^HrRgBYFScO)t%#!=4X(LFT+ z!9AT9>?!%aJhN$`eIY6PBp+WtF}Y=1w`ExaSjfibo5NwSJ5Txq22>s$*d)KP8IH|R z3Wr3P(yEQPL(CZTx;PbDH;XcFCr9Y8KBllVS{JFNdG)M(BkDu#e4@jmaJ}|^hw6m* zm6H?A@cQNh($;dT(*hjcvqK|*k@<)gg$QcbJ1ko6KFe}z>puSI!%Ns$&=XFoF(j=g zanTNQ@y`B7bXaHbAt2rUTiAQ&%@&Se2-qugwv7|&8*WqE;7!SowIvUs0 zVgvu+VbkSRCOaC}l{pNf5A4ks6>M!+vEM$F^1{gyEG39(%JOR_kRRd-Uwj>n zdLcYSqskjOa%S&hxom9eJX2B`2io0HDYJ=}HzujePLeqmxM=h*@Oga(weLSl+#Z{s zYx$X##V-^~2>@v@c*oeP4*g)#HkYS_)kxM)v-aijD`4!aR~EzFkeY=zMzGHRvCIMr!4E1{jcWi zULi}T8t_g`oZUC-%I)7L?kz_t=dUpwu($~r*P<hdmTON@QB+$u&#{ zb{gtbZDfU^wU~dJrDUpad1$Fi%AvsgMSnq*a^1b613`24@rECn8yoEKT9fYz_)xu4_^&hUETn{#e(63qchQIWx32Vlg%D6 zdX3p`OnyHOdH&kq&COV$_Xt`VZMW0i70+P=RD#Of1Cl63k~!Y13;YxBcj1Q4+8-ZC z?H7v3YVtX~){xQkiu9Xtla>_Q0Me*8&hkU2{2}F4FV<;Vrbi1lwwi?I~@r#Gq+8?-A}?Ii|?Dr4IR497VQ8F)6m&kbdvI4W(MTp3iEs$ zO5g>(vi*`EGjo+Q(-nYjbVQ|->+~DP8N`zw*2(`Rw)%HiVVJE-Qbt`QlZFO9Zy4~s z-$15VoZ?KUW)`~6XM@oj#obDqR{P^Ti{S9aa=R42{HtKTGInAknnB+c+#FN|kOp*k zd#lHNvy1DKa230=DrGuEOd`iUGu4`$HH4;jK_sq&{yQWdNtgE$WFjSfjF_ffhvdSZ zFLp34`X@(~%}pJjxGZ`?Qg6`_61#J&i-u!E^`zzN7mkf2+~3KiZoX}q++C$%KIp@h zb+nsfX}2k41%^6K@iCo%Eu$pu9RSpjq7bk5mRmLEs++3Xj3nXS$Mt-pjG6lv*Fd%9 zc<9w!FcJA~x@?T-_1sQz=5$&^tl5WCYr$t`XJxSXSXaR$3q{a|%r&M47UaR)=Cy~} z<}Nc)0}pddBeROjhhE)HppwOO{#I#+Hfw8AFq9jbY zdGzWtto20s@bG9#BuNZ5+dxhO?1hB~kFfH}A* z+5kS`QnDh;>6s0Z6ZfWEIoL0J?-v57^rd4ckNG1=hiLVd{_YEKG^;m8fD-rp2T4f5 zNmB(%Oy;vZgVlZ%nBzcc&fH44Pc9B76&tP?tJIJRWasMGc}AkeFJII?(51jf8P=b3 zJQ_fk;oq$MWek$sY%COvR>vbv>E%s0R*_j4e9W}#vx@UXawgD)P^HEO0b3?OzNd~k z7XpK!&B_#_%(q8drmE(cDO|vTnahD$Vm~<}&F&4Prq+XbtQ*WhB#qV#sH$rcXWLP6 zU>Ayah4M&~Vh@U|{F-2nozZ#A(fMCN!7!S}=seglwk&P2l9iGg@FR{#G!$wZvs4osOat#BpQ4T3TrUKPrm3nyY#6Wt5$+j#oP0OAF zcpD%k;~ok9Ud4?FdcWVVOI=h$fB!DHLbi4MU-S$)tPcw|NMqszU@}!3PTSq=sGD?3 zOlJSOs+L<*5D*$rA{Ck_Oy}o)X29rc#QcyNKtpgBD@zthg;k*pwW>v)CH}0Y2k2E7 z2Sk?WTr{IDUV`?jwbMya^m$6b{=>khM-vB(kImGl4*H}drmVg;lnX~kENnhxTIvKL zlf>P+cwo$w+j~VKop)wmJsTE52!TmZq#QG?JdX1BO8~QG^eUk7lQqUv z;e>gRamLI;cZBC6ipEK6jX`Ex%Tc%eOhQE1xRBTp##45bDN%#r`TE^K*r=wGOb}=+ z;g_86D?yidsN_S&p)}+plux5b+G3b>F?;JZMoUH8Lq!_T$M;k$EyJDJik`y7mD{dr8c-%8EfG$ZK@_xht(TsC{nFx?#_%eCw0~ z@8VKFGYd>Eqx|oEVblvt8t_R>nA`VKfen-vaVv#~B<2t=d%UfmIG7hjuiR91XzkX( zfm9`_P!}n#__v(}=4q-VRrxTtWU%Ac6-iSl=ZE>-7ED1H>1W1N=$dI|3zoqXBu(y2 z(y&qC>0$*}B70TbLO^A}8kFjfV?y~MDNx~i|3*LXt>WoY3zhIfY^y5)5@p&FuDNs+ zui~)O{!BYEg~?zDjvx*Oj$9^=J|Tq+6{GxNAyN@Z&{?6+I+UnoyDzA@YK%0M(Mq)2 zxaG-jEV5mo|5NqJqo{w1#1qWz=9g5Xn#X@J&8b?Is$)hW=2_rmQpjI7u0JwG z@nf>5Cn^;^!wQXu;8x%`t$h)1b$@DJ-?V47pU1~7x|l|i0lMw|Q-d^N{V^Rj>gvfc zd{HBXkCmvN;YXVbPK-O5g^L~s_x;=yMah^9#Y?Xvfi1264>Qvzs5*}lTzi#e5Wf!w z3ELy9*KRM4)grJdHard+GEos$mrR7F_u!BY-1us#soSg0+EgyaARRS18+UGAy=7Zt zSPhdSdNGu*TTuY~Bo5T;xx5Bo-_#F#nhQ5s`s-M6ypF$%zPd4rY=dGMecb-*NME|4O0+e3g*p{=nhFRXdt$UbC zIUbs>X>1@6a>&+)QBLO1jW%y0@w2DxroA+zbm&6T`&Z3HnSjGX)HKnSwKCkWBfgKP z#<1KpP}BBp&o(Li^-Cd2cRo=ZmK>@mvjMmGd+1l96av@6&zDnwY)9bOq z^>5#HuiXXRv(Ex4AA!Buk;EdqP*AELwD`ZnwI|S9ohwa-Hl;%iuu7X#SXF>LWN0L| z@NXK9`4vvVt*(yDC6+CEa_IsfA_$lU#V}y|Psl37LNp77m=gJpo!iY?{WaX2$c9!L z6(&CvzTi*(EH=+4$vAtCU3@wZ3h5}~mV>|uU;znF7oaI;U!ao-H{>j+kH| z0*f7LN`I)0wTYh>dpFdXROTVg8f#I0OO>C{JLENPZ`*d;kf+UsYgLnGR0;nkMlLQ#n_d zD!04sAdZAPKZ3S|``VA6COq;#B19Q;2R2Q6jz#;&eo;)YZi=w)hw*JmPyRXDW!8_T z%Q&_*Iu6~@(nLryA2{b=wHkEkCCuQesuS6~7DIadHp&R%e;}ytV6s>&6-`3=7*IHI zmq{|?CJivbPzrj0m!~8*XBI)PL=7Xe&XdMdawQANM1i0WmdNU_sky&Kx zxZvRM&$XnUxRBF73N3aVsmZ5j{3@@UBv2^xJVjMh@b3*jiN?m}8ycIkr{lU2?mKxq z1y@)T#vpWhjyd?P*i(##@3-BdOObi(?GU?uzD1ELvM4Fu9T>nMP?=R9e_4X+>scw zFE+k?)^F)+jw42yNRH|=6)>GSt1-OS#5$7vDPug&^;YTUi6CpUC4}&iPL1%FluWrC z!4nd2{tfs(s4$E7S-_qLNSEa;IzE6-RQM34RJ4v)H9ZEc`+ro_p9M^1+ckH5T-zoT ze`V^?b)Wme?K5Dg@S8`Q?tDVDUB!jN9$L)cI`4kd>|Q^wa-+lF=S4RAkjm2+{6!|Y zSG~!Nt@pCl;KHRRs&R%C-YxjJk3?XAA`{rnVK{1&f?Kjx>VRJzz7-qyM{?-d;xTWI ztzEm5Iy7T(zUV*TPIBvX|0UaSlDu(r&gBPkNocWOLsE!qL?ZcWG zv1BJxPVNe5@C!lOKE0EbzYKOAc2iDpg=78i%QPQ_!BN=fW4Q$TrUPY|m30Ce5co2 zH#VpDY}(WiHR6B2bVj|YYrHUh4z%iZmCu-Y-_yhgs`|$3;1ub zg5RzvukTi zjh;nZ^D2lLdwo*rk+rEWTRqsXxjn;PZTy#9?)LyHneC8LkcImDJSsS*7=cTav_i+z z*-{U``+)GBzs zA2rwrdR#%lZLEV~SL9kEKu{5fBI+IF&BcBOO+yFFyq*zNkGU8d zUJTme-8Vbvec)AZs@jQ|DpGXj2h;{gksG(1mygt+?hGYGuVKIe^i#= znmyLkzWQqKZWTfZ`RK^0L@TL%-NyelIz~$0J#@=`kNlM0lS_#2S+uhha z@aQPz)K2I04!Hu9z)!r?o0q`iFF{xUW|asQ50jAEt|&{C!bB2lXyKDj3QLPpe4+{y zd^7bla+3TvT2cq$;dSHopeKA;2Tg4yq)G8el_@L+^*^7@{AF_LEirPkQ}M`95*4mc zGFs-;a-o=nc=JXB0lD9jQx*8|mUvM;^5Gu*k83gu(4w5KFKAwO1Uoy$EFk~nqO%P^ zLk$2uK3R0bQlW#TV`Im?Uw4C(=D`4;nCJNO+3#v6$VoD8PBZ9KGVo8wZf>;t!ti#343bf zvE*AdGW>Tkj4t$V>$=@9BpBa&f8?~bHjNyA-<3A#?t$|#4wUCkm8k|4CrjlhR35rG z;21NINzH+Pg7C9@n^bG?sLzq{GvvcSgM`CuWT;q^bHJU~5u@C5AX!2r@J zNg5AmYuB&J?RdkdEyNWPn1<&;w=PLb(()R_+*)t$5U#!m*IAhVHoD9;eR}!?6GVc? z?-*#7wfq-YLr;QAb;f1hH^NTAX=V1Z&bUHy!Y~oX5r?6Kv)ox$^mvUN)oPR&3QmtB zcz`yne3aLFqaBs)r~!JLgbypo)A9rI!F@mr>3n~4Ip}x6RW?`L?=nhuR=L{MIjCnu ziHYm<%2$7^Zm7w)`$(w0Fs5WCm*up=_O?fLF3jiwjfh53zRHy%&5fYWL4pWGs_gh?BgT~O+!!u!u1a;P#&o7b4>&mf zy?>%o^rhIZG-PD=WIa->KS?lVvP+Ln?kX27WjNa~@N}&ti`y5QIJ9XA;Mi_^WMikq%cqW`vWTasu;I&VBaZ5 zqvQYS_RUQwJMdr4LxbTaHh|pTsrRWqxSFB;g}n8iU}rl|Xa6+~vUaec(G?k9+<^;c zq3y5*W{}?2Vz2goC2)GNYYXGG&rk0i>HX~?qE8&9t;U2$?{VrCWTobi`{>vbn7(eZ zaC!@Z$2ZrHD&HYQiefZ>7PDaAn^=1RYlhKiy)S=szIi$WifGaT^Sgz}DXciy%9_SP z;boug{d~5`BRniPMKrBXxS}|MFL3DJ^(RWl*9xNA?Gk@Vd>Y-76=Dqcw(ucUK2fHM z0)K8+|1U~MWGXY8HL2dXSE)1F z^abxet4;L@Sr~3Ep6eC6CEQh}MDWoPNT)6Um1clR!;q=dpBmP~KUqW0j6058EpnL! zg#&*B_nfj{xUxH7fb1KR85!#hi<$VR)h}C^nL3#!$^RE8YBW{F3cM`!nhXO5ds$zO z_E%vMBUHDK+mS$dd??7pDa4fN_u&JmO+6QelfS~ z1!CWt`Lfq&7f9R%y5oq``FNV{zJfTX=LQL4rOLpNo>x3M++sC<@zjucbBFn*#aLtS z2_**+Fsra@Vmuw59p+t&+_xbaf}lwP76SAY<5t6^ecd&VK7nF3Z)t+(z%5{*@15%Z z6t&zs$p>l(LNf(6cZZFIiKg%p;)6*}dKq1DO#j{|QM%cpNSuL;H!qepnc4n{jF9A3 zDw2!&^gDvpvwNRK1l^A5X?T0`{Q8v|h&5yDVBG&+U4AN3Lya%6BSxHl{-eyuMA|x3Iu8NGFwL zxrXnXv*qViM*D9)P8zE2J2}hh4R!U_D~~s_Ktgk^zkz<>Rqfb(jiY`8{ik;N@~2E8 z7JLv$llN%2!ysg!0WfMfa?yILg2$G7zwHO&VZp9%r+6qMCXCN3D61^wuj`$|j|g02 zwk~*b_Y{c4f?>s=*MGe18aHGNsK(W4w+9n-T@M`r>It^@m<;lHp!S-%#n}8c?2N@Y&I$)$Yp<|`6p>RZyV7>$^fSA0Gf+T;&N1D z`9rR~ZSLfiZK8y;GQ7HlgAX(qsKnS?;x8k^TA)IlKTvaMMG!2QDc_nBMgQM((ME#t zHvw&>C=8(>>MhZ*`_-OOmYS=TUQ0CA;UWB$2C=gLKxari`0!F6fqt^nkqJ;^KP3CT z&L%WuT0`{$m%7?$-%7cCD3Qe!WCbBnCYnGXm~?2RZu1D+3qSDC^cu1%3H>`U2 zzb!nQJx!A;xRrMla%Itb#+ya5FB{q?jBCZ<)!hLbT^Stx=EZNUDn@u!s=Conwm&vi zFC$S@;060nfjsqptLiN>nt1mwt#y`s57)INgew>_oz>uDk5K78ti^ey5#vp1UFkQx zSO0B=P$^`40#@M_uk{gX6R;Nb{7xW=`6RC5%28kii6AD!uJz3jGy)67 z0I$*BDm9R(8@p8L2lvQn3Wa?-tH*bk(llCE-`Z!UGbWN2u zjK>w|@?jDEN}7(947GEip*o-dLk1xBVEXoyoQGIRmONO90Ars%s-R^=6UqqExaJ5B z1t7$w&$PwLIHj{-`->bvp#ee%*G&3ba+@EtIl&u+krj@j%gOj+ixH!rC~1k z=H>=DTqJA|)j5lIl)?8A?DLu*4I5jcpB^EfE3C@s=DpYN(o$gVB>J!>HWn<C}CHG8d(6K{13ytrLguc#^d*q*QUFw0qElVEXZ>;!mvv&)U!M`d_pS&BKyrw@q z=cZow3)vr4$83TXJ!Y8LS6qzA)=bNaM)AQ#X3oLzC3BT@hPG(NgjL*#^8^9)Y};9$ z@X+~F=Q`ing0-vlk0#OpW=wk8Y;b@FX5>(xfCua`UMsnxRo1DxHCakg&$PdRH9@Rt z-g=taeRL_QmR-baZdO{+MbOx;A3=))lZ)|HQdxzJA3e&I9dOSWimav06By@-t09YF z(fE@uea6y)OKnqDSsl0zNN%zKzgcn97T6ChhDyw8`+uOoh*#WY#$==6O(lfXRu35P z^wTDg6J)-=p|=i)_nJ{LG~BB@V@50seJ5^6krC2I)>a=l(&t>AhWZL3_Y|Wm%-RRpv81VF zHWBVOu{`}Sua-Q^Cz(G}nxvZte3F0zn5bt{+8~3yxK`N z``67LH%@L_&S;RS|L;r_Adl~9d_Ou?@+iJW1d89iPjHysxxo`orY}tFG};cqYl}>% z39!tTx)a!BGrt!Vdd9E}$5Wh)X_BpKy~y?wVoj>lZ*c2bIy=J**HEY=KKFdH_w8$g z`KSRdU`#nPFqI?Yo^zC&Bgj%nFh&3r$LzL=RI+EcR{wqpNO0#BBcYtM?3+dx+FK?_ zkVZ_#X1kHtw9g7C6t4j470Ak=fBdj0mdQqjrY9Ic{RT@Sl{*!di_;^y$?uY**WZlo zsnZEMQz!W9_*)Ow1;6XYTbGT^v-$ci-Tf++4j3pC%|*^|(NIrcW+^8b*K|6E01zMX z0HZGs72|Y0U{$p14!OVg{(3%}iFfa*?R_hKe_0vB*B?hlw;c_1lS>TrpVYWZWLW+D zl@W^^1>bbChkRu3@AOo81m?x4z#|y&_%^tKz%!YZt)<9QtGF}-yQ#6wC*oT~-^?S! zhddrKWNia@1R0}9)A{T5I6w&iQ{|R07>HseCY@aU0sEA|V`I@{#QV!`4=-Tv0SPzE zZD?2!h#%gq(kv%M8Di3LE&xRNAOC-@MPI0!4kKb3&=|M2M~ihfP9d=V0E&D4v#K-h zpRPTJS$|!Ced{#X6TwQ!jK7}55{XKwH1AL|QI7$~jqa7eLxFisk`d&F#L;IzCc7X< zO&QT=CrFq5l~Bfldz=`@L_Xb5#y#RWvTQ!N$7{Goj^aB&MfJ~3q|cvIl187K;93cx zDDpj32*@QEg;^8G<=p;wQ&z^YEyeMJ|1l@+kKq!suVwAE-ebWpNvInT6DT zslG3&Y^DGSRq)kK^~6AP6>I90jsRUmS|O_)InVQ%YIie00P^1F(%+P~d(UzW$_Fi< z_MP&(F@%vy(z50bLF{I<*2;dLc^E5DOk$|HEb@DyTz?0!!H9k?pjIDFY5 z`Wujm`hbIe*zo>bj@kM?_Ss<(5E$qa`LCaY5)sVnD65rejP0dYZeLR9ZpoYzr4=$W z$L*ZIZ!)O@#ro_0D!?Yc={y7KcHh^VBxbj{kXyAH4*?_JV@c~@v;|%4i^wujbb;eD?o&C5muiE_C zA*W(?@8w@U7c_0we^`H{pda$fkb~fuBm!IORMgoZ{3|0uM(KBAKbK4OeWmMRDb!Z`i*Yg~j zy7p@iwAlag{zM}biOTp`CGZWI!6bQ*U!_ z&^WJ$_A2;M0MNOf`4hguwRILXm6?EgKhj(@kB^^ppnhrbM~OPP_YgFx`;CGqNC_73 zi@s|F9mfZ#TidA#Vsw?y9k5ra3G7>LqS1@2OfyA~4mg1bwpO!!bN@RL+0LY$Uz2H1 zmJZpU9)q=fA(;hNIh0!kT0qI=IaFZkx1+%R|Fi&r^#hSwYq^{VK zPL=|^LadMOu5nEtT!CpP|xt}`8(gXV<1B|3BCWC}8+yxu%b z4Cn;#a_Z(QlQ$kbYPvgf*bz8U)6L1HDMf_!X&d&o8Vd(DcG%=`7qv5|q9Ftbw@;r! zU`ZmEEie&7x&8w3dxK4(&I9MK`5b7GyS(G~8zw4nui!Rjff5>42 zB`>d}fhhqcj*o}67Xt5?H_p9!k=4GuQvC^mR%D&D3?(8r&-(w$zgUJSHum0(rg%yJKvNapDd7jk`=dtny8jU%-z01>q>A&hdqayA6d0u|hB6%1^#hY z&>gt44zpJby+>uPgR+3zBNpC&_sfI&8 zX?M`h*`4*7v+Y?$Q`-Ttr4+99IO&`R5CN75EsTq&^mAkYd{! zH2)9FnbWf^b1EJnX1;#)d;Cn_Y0Z@lH0-a?_JRStcHmtb-$KPYe~6F_9kO@^EtNNF z>!ln|Nhe(a{)<=%a;}$4NkA(!`1B5VDK~90pjyuDK7sp_f`FA-CPH_#4?x^9st zH!VA$EIYKxWt=6*jwPL}j*cd2-DSy+nYzil)(#bxDP=VEku{hsR+lf-gs0PL>EudC zCP;*#!F_cGOL2Tp>HdX8R4f)db?=qs0W#vvg7u9AdgQtN{Mhw!BAK6S>n52#bv%Tl z@n@)fRHp&(5Xm5VdPzkvKE7-rD}}0ekc^ib_6HmqrvL3>Ac7dq|AH z6YA#WxPB*TjQV}2$#635IzyO}5{J5?&&f&#p1N*{Ux_$H9~c5ddMDm3ADLvcKe>Td zwi!nkUq1gXqs{ykFM)VHws#7$;<&9+*5<6yeU~u194BQ44dERmgdEsStgH7i z`FQLY=(|{&UNMj2;ln>3+R@TnE_<>xE$HUg>~W8%gOS0B4`3Mr`wqo#*n%X%}7@HLzu#Uv4mKOyg8|q={0sF3Y-JcgKXfi_l#!&9&laN3gW;J~wCaFO&@RlvAa@piE#f<1wKr$+*7_ zAbqp$-nGW~CdZa2yXV;U2Yo}!kbXKmIQ122ouh;?FCP(QgUHz~$hNB%qz!IlxkUA6 z%=>e7E@ujhR-ZNuL{TEPTlLumsH(~tgSH+QX5l(~$Mr%(dlY`_5@^%@S^6-6R1$uZ zj$B-C@5|<_EQGLuZO5gQ*R$DKXS@y1(g#>g84k>~En9C2Mo4d{@;dj$!5z`1l2;)k zh{5*5Kuovgvr~}^QcywANV-rY(E|I_W?HoCZwnBaZD8?%j+%1`!V94zzwRmUQKUc*ox!oCC%kzX;qGvNyg4j zAdb{&wLW=q`_m_3-nSr)LaQC0=;zHCU${BzIVb_2d59NhIZ()0oySQXvOcusU_%%R z*UwT%GM(Y5MP>eqc$+FEU-|7`C1EK8StPO;<6-k3yFdHNIlVG+=rN&oPC$>CnTUE3qV0)a^nX`^% zwL__Q9`ad5jlEwcXPXU<`S7+rxvVX=W%_rf4KuWYfK)by)9}~ztO=ybY~CspOYZGv zbFioBY(!w}t${g1iXdtNyMWzAfItoHNs*LEmeQ zEwp&KiZwgrcZ7G=pXiF0xhP1t?bp*uiFjE$KCe1FEq9g79NsnFB^z#fTN;KNuvuOk zs4ab0zw$=!?{<58%8B8ADO*{FMs!8O>RNog=f$2rH((tZ??k|2K) zPLrlp`XoBNfmtv2WG%@k~br5)E@V)hlzPZ_Srl!^9OcvG=4-)|zN>apJmkT`O$rU}8))-7< zu(ve>+GC9lYvi=TTB~ti_Krt>mwM3UvCl}29g&=PuqrBNf&0Uat1uAzc=4mBESh~< z#VH&v3A#i!uq{4?G9y~WCTNr5016}LZ#^$Xno|9wrkE5#1{CWxB1!PQLog^(bfl8Z z{fQy4FSn*5>J@(d7Ppm67oYlq5d-w6;*KDN!PSHpeS>NB-d^~&`A@R_wpm`b-TeMW z@=or!^I%mpAE5wpM8jx2zL4eJ-0rmCwDHJ81S400g#L-Yn1h&=YS;Kwi_7`5-3XYio1sma|BNJ(sAghH02s6Xi$*e#VDm%(9pu%(}JxF{Z4~Y zp%Y3$h9-wzWxV(+DkL)tzjX;A@N~GHKaFh(KI?tQLWo;^ep^a!W=<&tBmj6^OKMfJ zH63PFsVn(vy^JTfi?4Z;oW~sPW`&TTpS_O|pP#7x*X`uI?>$PIZsJT%7Mf$;x{-Et zlx`n+?;iT94#nEfv$QlAJ5!AvQ^X1NgzqPfP6QT$D~|ufdtVhjNYkg_Y2PEe?^IV0 z`s__#Bv!_#w+j{L8qC2v>zJ*Ej1Jz;`1pLsJ3JIl!uLMSkw?CGo^Z(|eA%}P6*074 ztBL=t=S!?E1m86%;K{e2-3a3US*If-o9_i}yi!BlYMAypij;t3{e-?vpSW?o<)~>) zk_ah7Xsm9zs7%W&gUQaaV?hb+x=^+o4n>K4uRQYNOANCZBh{qxDw^u2GR0^8iVc2d zb8a=mZunUJG#OJvd}p~LaKF5Ww6ZS6;~u@^x+gVSOzF%Bsmh0UG9(#|=<)y~28@bQ z4VdXOka>yjZFi09DO_MoHRLcx%z`T~K71VMB&(I@q-HPmPX=Hoc@-HIlsq7>GzO&% zkM4d+gp&m-qhtIrgN#?KKqdYKO-|KPk$3zl-0>xwDr6evhbMIk`>7?^wCymeDdN{M zaFo+L)o5zSo=Au_(frlxLp=D6n^EI|$U|l2reQ)r^OAvtzbdZ5m=Sce(vqK-Mfkeo z0F?h(B~l$P{a(@?46HNr7eL%#9i`3q6DxyQ>*NNt(;sQ$c?>>h=N$t@_&~{G#tLS? zdDUO$b4sTbXJrjL&o1yc^X<#e)j4Xl|zk!lU5 zpFsd`m`v9bWU@)BRYhCEEJIrJ55kTYA)rloYGgRz%g|vSU-R3G6x*vGGkx!-U*bV@ z8tUH%MUqbulBpl}(%uK6P$47g=^gguFpF59Rx=sywQ5_ZE-8kr{g3`378S#_zh%kd zi+*wAH>uEyy=l9eL$I`k5oFvcBq+=gr@%z`*l##G*$eNji7K#fpGr46_>Z+6c5f>= ziz}Qq9pM5*i8xES9#BJV37h$E=tY+G&$5O3M^O1+j@XXf{f=!h=*$iJgMm9H7^K!r z$uJz2n;3SHR>DWw;IEhqRgmu=cP=p$bAgg94WX3o%q{FSNZioY8S!29eCCpK8J1*) zj#nko`w-4;yHXdvqn{B$aR_VubzU!-Z){%ev}nyt4E8pN#>22&kkeyg=?N4>TC;7% zXMgw4TJx5hJP}s#_Mx@b{nGNvn&v~}i+HCepv+iUdl&G#!fMgw?7oi3euV)4E{qi- zmwfwNdHhM>Z4ni4WCpS=Zs$fMflbWL3IkqYv(Ey-}S7u+ZPcR?2V`T`FO^>cB6<_YgheaRLiXQ?iRqzH6(LzN-uxaHVz?UjG?+G)V8 z2=07^y3U=W+FPgw1zFyV+kLs-t9|O94-=E+S?T(f+N!OwiDTfR)$}X)6{?kUX&MUH z%_&c#(h7_ZVKS$zOg^d+ZKFKZrKvWP9T~(ImGrdBFG^f{Hms9ofkta_3T&94F=UXj ztF±j4eDcnTyCvnOO#IsM(bZG~oSxtLc&s?w$@>@pnq{&*yt)4z#S?GDz>C4YWh zAeusJiPb{*hM^`E35cTR%^j9xl7jAqP>$eNt-D2bHQ z9@q@QGWVOej9tNDOiq03wIlEe$e%sVL^n9HeDU53h*8uz1xyC6hSd<5x>YH6(ZV8m z4tI6Y5#D5| zh;yz%)SsVHq-DtXf06M2{*07;ktzJ@0~zp1N&j-b^>>BK5|Bi{hgXX2q{2Z2GEGb; z^^zj`FvRfjR}b;^Zl^*=Yg@<3rNJB+tkiy$K5spuWLA-w1ug|qVjno zIqUFKCQRCZxmsv%xDj7$;t}XEhv=eOn>lTUMDXTubjfl{?PuRAm(QI(&TNN~f93m(fYf4g3IsnuV(=oiac-1$@*vK^PTXAGg?o;52ZIv9lNn2N-WRe?XvY$33lQX~SKPy&`!`US10=l@`tS*^9O0ZloF*muF-t z-gRSa$uJ~|?CS|_9z9YRqOn89&LKStfJO+IX_BvK;rp7t-@N(sSwRhR?VsI-8>+s$Z(P8Et}0yHaRlc^&{J*J$fre zPchlGAZ`YImskp+`(Vd zA_2e~`c)U260AGHl!Rmj)R5^~$SSd@=B9fG_JaP?x#8KfwOk6ZFVYsiHtRE`7WeD# zmLLN(>}mvb!`6}DGR9g?`4kJ}p2)A?Lfri@cV77mZZ660f_fE%HV^*XCV|Ac41myP ztoKy#;RE9U8PUkG!WT$nn9>SSV<>FQhR^t(JzQ_COvZbS+KV{x5HV=YH&^dn6Y#zA z_u+lBxDoqW6Tj*!9M!1LaZ_kzL!7-Wh`Zl}B_H5IUmc)Yoj5pYrEyLyM)6d_zp&h| z4>jQ1XuBRVqlpTtsLa`|^n}Q2EwsA{fP)hs(IrQHs8+{_;=50c;JEUD^0Tk4xZrMd z*%vHdB@9?{SwL@hu){;`^ z7ZvVVg6p|mS|>xvU(1>`8P+4|+J>EhOpdl02lvQ8<)*>$SKTl?jTS`>rSQ86^=(c9 z>bHdOTsbtiHP`z;=KmhNK;}q}9(*E$QHgsGzHJ`&EJpCE!y-Cq`o`4Z}86rG$BW_+RBS%pH%zq=7-J!QvS#Qq82DsZdJ+53x%8|dfkRVa=j%MzSojXd+>G5QiC z@QS+T>4iHOjqptkQ~wb%0w^*Hk(5%?RgP~}9r}Mbp1T8MFIG#2-H$WNf0qn&+{yh% z_T2nOq{XKuFlzTNHeyj2d?6N&3^Kk1DxyhfkTOXQe07x=YZ*$|k;oTR!gPdKmqx;-|58W_*H|3Z zW;6G?9eR_dtjsF7JHiB!Kw=h-^bHVpSeOD)DLfack(-Je5pU1$tui~egj?5+jLyl_ zasPBaHzDO@W_ej&{v5<>8L_Hz8wr37Zc9`c-E?D(Oa=OQx+tf+&5em#dbL4}lV_P~ zK}yofDQjV#R_KsqNLFkyM{D6ap(LU&^}e^+H&Z2$qew>M(Vep|HkW2*W%*)^*)`0! zxAwpIXuHf7{HgIktf8&dU$b6X;#q7u^f_y}_5?!ED8QCSP=@#NoB2OI;pARIZ1Q&B zx2PmL+$JV37}c&a0Ruyfj_=w*fTKbYgcYD)vt3+5nk9h(D{7v`wyG<@o6*DQY`hro zLFidYc#NlD}fbKe(}j=n6h5^%Yfde{KX;ICbR^?HRZnmy;>*_dFu;MqkY zycrb%Y5ONs(H`mMG-FhG&Lhhf!f_E0Sq@D;HOo7S&qH~U$e7$?8aFbr`f-$Wg};Y- zggkhbXgmt_x*vY7Zv5d^DAOWaj!;Z(EC+ki4c_cEB6K>-4rK1YE&*76Mg=_iAg=~t zAr%+P7hPaQfU5$4XH=l2fJ<)tULmrdAe?r$E~ z7R$-U|3}m}2ItvDYsZZnt7&Z8YV0(&ZQE|#*l29qcJm}nW7~G4bLaienQx{und$UT zbML*^T9=j|a9tpPPjkebaG;FTxpX9f%OSx99FtpZ`3&aqz-|PUZmTkRqELf|5v~=RM)T278 z9Vm>U%u)yZ5)ludgo<#W#Jf?WJ6sHkQ}T=c;mIq8lR~A`ACuskxr;!vjNw|a=%|;X zk$|b36xo^Z#P@CJT)J$~L4{~6d7pVWssfqk&GYslHvIt>6$lQLwHij>@*4m`o9HZQuV^(`>W^2@ zkWZ6x5w0{wG^Bu@IolUjOVdPL_bjGTo}&Xk+>)3DtIKQsE=5lxWGo+egMg=WPPyk4 z;(jw*ymZmg@ww?yf+tYDy_LM{zA&ILZ4lNov^yb&^@B~?W}DVL&yj-SWs4q9Aqk7> z>-+V3e9b>;P>U`-uHWSBYh+1GGD%8@!|;-nAZe{*Pc-gD6mv9&w>iP;u!H<9Kcq*#Kuf-2wrAR6_3hyth0Hu#`z}aBnhmfk{lhd_ftWZgusCeZ; zlfGi5wZqYM2Y$lYJrexcCn=p3@2rm3y=ld0<`4Oe{H(%%moG70R*RA)jU&|{qjt|I zB1}n?KAs$H90j3x%!Go2Wb()fr4}khC>X&%`-^`Uf__!hu@?56t6ljC1`Q2mpM<0) z2lfG!*ljfVF+j(7wJ|^S$7i5>M-Kfe8#jsbOq6kXNJ5K|rcY$wmjmr`%SuO3r+6rh z$K-3DCzlD$$B-OwDCJUTrib~Bo8SA3mYHl# z$**5-w}}(jW_VGd>5xAc&EIV4LUV!+Ih?_7E<)UiJwljBxe!J}#v_?Hf=_T+oc3Rd2jWr*>!FW0F(R?&mS3zg6rD`Yt42O0+B5$jh+?3 zml^vAP3G_!i=dKRT?8SP^yXaa|BoC$>7ZIec&n>_2CNeXeJ|XIP$GjnQg2Tiep0?w zxnW^i$OOmYvy&&ioc@JOZ5H(`aNS}EP?RIF*=HYB7OWK`v=Q^J?%e=;<2m)+Z2uO9 z;N?~qDp=&@-^{^`>y3!WL{>U*;6w}d8QTpHUv>|AjJICQkX`Rk;xg^AzJ79whxq!b zeQCk*N6W^Nv~5-H)PTg8i3*w2eoowmN!-ctF>&_PWkYyw|G~{P8^A7!psx6}_$-lh zuE%5Pe<+t_b;EWt$jddVlLOg4uyf#zMoh0xv7oYjr_Z3Gsla$avV(}Rv%*_V$unxx zu*z0h$P|v+#y}3P&x)|iRmzVOkZ%F~5M#uMy6t=|rc318r$5Kl9%kGJT zh_o8BsgEo_gGr47OTjquWlJye2BbZ@b*}gClZfdEpp66y(Aas`bAFQix(~mxW zt^d;kRHzYbjKxw&|8PN0I|%qY8K- zf~8J`((z8aI!^{c)AM;eg?7G9ap>uFz*Roht$BY2`>uPo7j%QY?ZmTG=S=t-Oq`Kv z?yx(`3?pI){2`MnuB(gHwr?*din{Ffw%^NkZvns@to#`+@8wBj9(;I0El=z z^X=N-r#VksI>q#(cRJtp0z@|dVYo_`X1^_V3*2=$yBGN305iA2c%tez!H4uX%-ibD zn{&6f%VTk%dVp`pmA9Xv zjtCJI&+abMd;X!tOGfu?qBLZhCDEs}+w4K>G+#rgsP2u<2=L`dAD;|Pasu}?4G{-M z?L86`$t;4d&W287R0sS24eB;GLS2|hDKW4Wu8O+$ZZ&DIwDNp!aFM%cS!;k%HnL3? z3K(q~6``_bNY)!!nC*8WB%Ljaaf}eimY%iF9pCO>9P8ngwwz_Pek;2NRtK2TdqoU6 zfHVADo!^Ei>yLzT-rHWi?(iBH?=@W$4oCnG7hpG9m;bcV{LR~Yt3937aOr8YLO|J^ z95{L$U0tu8x7RRQ8JRYm}2eb3tNsWA@k)8l(!K##byrO!WM%c(yUiIBQD0=#el z&GZQ@3Tq15U?=mU!S2<2)gAPEvZZ{M{tlrlN1#pSKcKai4qa*+7CyNw7a-()4L z?>kD-B!l>$f$}#yI}7T6`V9uaFl180OfE`gE`e!i@afx?E#*#wj`MF<^uX5zZ+%Z} z!8}ckt3Kho-<1K1yAA`T|8?1l!asIdvJVA;XMGpb?2VZw7@a0xs$ge(ys;W3f)Eqe zp!S6wpio_ED|lk%gz#e|04>HF&dsemrV3FvU~3&D(uGYC0OGa~RPZsIbEmWiHP+R(dY#`7TN76ngmv0&v5% zp2TL2BzI#$TcZ)=(87@o-z60>ryGoK9=K;7lA@bL6|NBQ6oA1J=YqouDvRYljddvO z&Y{wN_g{g8VIE<1C%o0jp)l|rg3t{NmYQH!m;-i_{;u-+AWUoqn7oP#Y&m65zuzaFoo?uItR>(Mc{i|O zzm(N?4uVXJ3(O3NLW;{??ys;fRa9A=#Yww-A)8^9zm%G#G`(ByCBd`m0q^bDNg4nI zY)(#_KU(0Aw5IJ2d84g)YA)Zq0=2eg8yGrY=XEZ0^Oi-Z@BxtRJ}CAqyJIqfy5WXm z2C}oxk2Q1s4J<%m^$t}l!#vqXV1VgyuO_#srG+qDWa}5>s)*^{Ung7yfX3PbE-wpi zvTg@2yqUr6ZH6!1OsS?&hIG+WO|D?X0I}mcrG3N5*`j4Psz*7^2zmd1Qo?Oqc?;1X z+zNkk%ErMkx&vt{IXaW!a7%O@m-FiHPS6iA3@>*8e!^Y#hMBt7v+WYWXbSNSTyv%{ zPeVX~Y0R!cW*yODU5s<K1V!+1Kr|E{sje{k zi-7w$HRg>Q5A_nr7*0TQt61Zf8fdbj1jt7;F78&Eld%lz3t2DUJ~<5Bq2I863{n{B z(L7coB5XOe3)%kb5ImnB3~+eD+r84*Con|bc9mOhRz~y;-ur;q?FUBxH-Zkr{rY5V zCSNapP>o;Acdp^LU|oYDwBNRt+i%Zy4I}HW2X~X{{D}+e9+@6|QqON9_lwXsW;bKr zcHC!zsR_|EllYAx=_*IQO*o>>X$S$^!R4UD(`O<&`k50^F znyx7>^p9v&(6a`U2${o3(=iWaw-$E253<-PER-hv zD%F@UXQZ#7VI&o4v*tObG5im>gxOe(2Ud~EU_mp3-If?fCLq)3O^xtW61NNoO& zDi6Um9YM@2kpkxR(stz_lStNMu7Cbv!k6pb-_@<#2Qjd@Fu!MQItArVvvP?7nY6>V zUN3(7OCpwlBJfv%iduU*xAE;$?Z5yv;P?1dULJ%yd!iW2no$cxdiNQm{c8`%W&x@@ z1vbDFTz=koT5cWIYc!&cm#icL?7aND03$?{sCXpshnJ+EU*;w&-Dl_IvjGl_gtsiX z*M9VHLKibOp!yY~P01_AlkooMz>Qg~V9!;y;5Cjbmli=7@<`PmjrWy$(gRzI*-R~K zX%~qFQ`V7VnAY)_^-;rBVTt+EAH8l~wiw>Oh^o?-;r~8qKhz${&E8?kko9Hk=D7Z) z*I68o;q*z1gy+_$`%96YBClAdd$ucFm5s%Mq49uo0rt&-KZf0~>3eK@(WiA4>wI+2TL1p7nluup<9N6B^s zT>i)abtu!S*n6~_%pdMqL8?AYJAszlzGt-$$~BJUk*DOs2c~_G$5hbb*UWv5Pn>I) zhH854Hn-}gI3H!V*{AlCE+|4P857gzrTPo}KLViH`kP;4?;slDEg)YaTrtc8$h!6d>Co#jvFbo~VEbPDA{t5KbANzr^%cCaOPi`PjZkyQ?CSQ-jQDufhzUaTw z7b2y@OFV&oZaKJw6ihL>cT2Cu-07&rB(Tr1$(fh?b z$2S>3aXwTC?rYTBkidVuEu#Eb^b@NZU7Ds7A(EN5`Xo6T8)<)7;)o;aF$R$g+S@R^ z72c!;Q$z5OoNIRCu~rLxt%?Wbk(g@WrpKsz!Q!@5#0Yeq-jgm(1ogJKBAw%)F@sai zPnPlkB}+Xv$7nhTOj+<@r)QI)Or!XctQE?R?j#w>?8WNZt9n|VNo5JrQK5+do#{BCPPS)4FCqrJlR&~S1G4lFT1XlRY`( zUZJC8VNyi~SJbo@8mx7@Ih&<}IREgh)V-rHb~X|KqNGj}l@B-vcFIjjDqE5%nX=a8 zhh`UT{P$so_Bd!|6BcyutNA0R-GAzpE5}~yitmx=soJn~LDZskUy8EQlu%BRDT@!s zSYP3X!_RoGl49@hWMtz9)svkQSoiL!k9Kj*2erGY3bocaw|=1u6g>#YI32em;^jm9*edMaCTn;0!hc4N#grX&$>mUdk#YeT^rpxf}6xh)vHWkgX79 z!DG4nJ+GCO03|>Oy2G5Na5y8e<=jXNQ zrMK{fHZRA9qi#o<6yp-fzm7c4@;PYnVt`d)n;LVAflFR~oS7ABi+&$svof&UGxRI# z2ul@O79(AFaZCe*6^=w$dt(SfE$?pIQkXUnx0`(-+|w`JmXcV zw8`|b7OpHhT#@XjiSNya>t6V>1q!v_n(L*Rjr;JN7_j`+49c7{zS1=@n1r`}CrvZK z(TZnEphHuWZc~iR?}?6Zjm}0%$U^_V+v$fT9iLA9V^Nl{^z{pcF^&qiiFOEh zYu(I5tYpHJNk)<(AO(y?&Q#26k{`k_q+_^MuV?3zW#Vt=EiDtS%i@ATED6An&zm_b z&$C%=@L<#xp z3yYO*`5eD{)fYd)4`oYXtZOxqKpQaEm#JjLWUe19tJ21s-o#A+Y|d007R;Y(p7Kdfc1h*eMv-aX zs8O=;59kLDFC_5j-%j%}nY>L}##diQ7P}i_9+Y6JTJ&ve2$!!|!OT`~n17PD&v(N~ zkWN}>BdJ$1h2pCYWsYqgElctx7lZUcHZ_@{Wf=G|D3hfjrqDfw!Ns$wETEdY?MMHE zaM$dreg`UOJIWh@{j^!V{u=JpghkVYN8F~FTl9mU(@9@L_sgH52M(SEc?RqZ!9Ma$ermK%E2qQ6r`wIX?N9P>o&AR?;;TrWG=TJRb zK>2UZw+y4MMBI0;Df|)2A_|C%{-Wt7L`3Au*<-C_LO;C0V6)%Z(PGi9^lTUYp%hSq zm1ouDln#6*6I`Z^KMTyw%3f_r8Eg4A9#Q>sm$< z#24baqFYpkeZfW+!cx(`fux6(DzCq~pZU-9PpYx9Pw0zy!kJIVazu9w8bkLLUCj_8!uedHl2!)IkF-3R;GFp=!1a%9vi*Ur6Q{4Re-6?!6+S0& z?mD3Q60<4@kMzR+@5@uk?hy^m>Z;oWd&#cG0sMAk#D5)G%|X0`Iyxe99YA-1&CTgm zxB5WSGU}Seot%9Bb<&1wxpg_Z1W~49ckl7PBFnQ6k!6(lPRZUKFt|2E-PVeer2*to z-5Tvv7wr>6ii?$zOfaSxhuDYwPwi7wPoFIy7oz_)6xty?nS+)^0)UC)wCNz{%0so8 z7ZoA5J@|dw11ZNbAIC=ZP)tZQxcbK*3_!8^yU-_L7eXpQi~*(lDHq#}J#a$Rtp$K< z&pT}~LDg>`=rgdlkr29fr))PzVbvsV!jJIELp-uZ71K;blPlqQ>SI>gdVar0*p47$ z?C#9L*5}}5weNVs!`n}pa$_`G1K{e8vmdV>dM7pntryK%8i~zzBPcFTaaH7r;I>BI zhc;|q8!-x%dZm0ul<_^!znK3zq3}MTaXZ+pX&DB>G!YX4et_LwBUXq?ED7`>7zh7eGIr_`&M=VmblJ*wcvtiaOh=CgRUMucJy8Y}vD$8Rd`vNn z^qS)kJte!30UINn-Clygn^(Z^%f|6H`Guw4Ly`9VGP@?`nePKUlRtL-_+ptRvaCCO zK>kHAwUJ1QGkFpT9!n_Z?C%8e%uNJ@bNC?)RlMJC$(2z1H$=+M8Kns`; z{kyS=Poy7cwE>0 z%xMpDIddc7A4VZDXa4X-ouBIoZ^2%9BmiO$KqHCW3Lu6`M^{2eH8wjU3nk5Y0N^Q{ zj!?Waq&b|eEZssLU>$MC8`qYneEpruLJuS}=|&iX0SwXOKt(1jO!%oQLu%sl zd0Xgw3WfI?V8I$FZfRLYU)N=zq-E3`zFW|iJW3{(gYp&X<_hkls$P{7ges#Vje-G1O zk|Rr^;7OvunnW2}4^u+-v&*rf4D{lBgW+UHvU@>dfItEekM>K=D}%YFJ@ZBQDaY$I zD&q+1=ATAzR7+<`t;?KCKXpB!Zm_%9tX}@ zyc1w@_j_N}0#NeJq11)hMU9V{R7Kr;6LxTilyQ_iu^4Bw#jLK{^9ac+7OB=i572A4 ze*t^`>~b}z%*c!be^ap`020y1@g`Z$52dHmT7(d~r~@Z>I=jI?Jl!ud*V~%)cRCpB z^3a57S<#s?{YGib|2~FsDEUY7A;ud2JZn2A+u0(eUWmyO7;xdb5Xy{qc&E1X10H-X z=^;Q64s%)avslqn6>HB=4G#;ctM{+XRW3s$FRw#8(XUFzC`*QdrwglmSf4MyuD$3v z=Q+W2@HsE4v41fVLfrq{Ke8XltI%S9!8KT*Pzf%hVVTaF*w;r40QNZ^$De^Pt!`K~ z#s|Qr1gQEZQt5?fUEI!=K*O9HCDk0gp%cH~EA$jF1@w;u4>i^8MF!gL-oJl*(t+Hr zyJ1*Virjbe2f}o1U+h$l9POS$z~;C+b7%7|J$$MUS0oo`3IO|QF>o@d0C6)y;Wymf zGmoJTu~f7k+ViVF`cnh^c}4k9?kPb1FuG(#qbOfDA5nit4~2-zVY?|Pff$0miKN6n zBVPr)nVWRx_AB39bPGK1WsBJ}W>vH0L%lD$)Air?xq+;j{{08HX+Cobfn4l>>FH7A zj5P8olO0jF^K-Dx;W(c~0#^HT8(=S{_fs69pfq5P00fII-Y@7p#R?S%&DzylW9E9r zHx3|Rcc$_W6c^c!O5Q#2o61sK{M`54+3`QBy4_YBNA0o(;8V)*fiF=zNlrS<0i zWcW#WB0h#!IjQX64MW`(DlzDNd2OoCvGj3c>vSjp z5YBJhfI9_DHj{&e)nqCw9oXy_^7ie5`)%JC=5L3`Gg1mzSq>=X|bT9v`(>UGW<&=n1)?{Xm0o~j59CNi}&xy^K zTom*r)yvBw4qZoIyM&K#F4>H((w zKaKKJ`t}2;hMp@2aAIXoO|LsHVi^EZ3C_6c{moxG{xNi!3l@TJGjaF}P$&V|@3lFA zcM0P1uExh|@E~I>_ECl+4Wo0--2}hDz`Qe4(6iw=BzGh*B$|41bQDfZ*h(Y&-x$VN8zHlWCv)-Z2*=8q#8%nD-kk{$Uc6Xo^dIkrrtZ zDTOR!UrKnbDY0L$0c^@x&98X{(v0rO4KwCSbR9!I|Ci2;@88Hz%B<+mMSu64H1Z>r8!fz@7+z?F;_zm;fMg z08o@-iui}^vE^3(&8fwl^YV`%t8pTzuP1ze6Q#995+eUON47f;?@T$;5Gi!w{w<|x zZ0egK{aKrDWX|5#`Scx~Yu22SD{)jl?G^X%-yKyjS!oBnk}t}NwwL5_X&Yi^>~1Lp z$mo(D3glBtML(T;7BWlyCqr6fuMD*aCLI0r3!Z_!$4^S0mEk5(wMLo-Ma%+sq((d! z?2HcG(!HTzrjqJQh28nAu3|Cba6Z2YJTpKhLFQS~{QBN%!=~x)Kf1oK=k#ZpFG|!X z?Ysc!(aqXNXL1C3F^v4~Mx6nSv2V}mgZbrl$cqB6@Za|^9s}GZW>HMam>-5HXpy7D zCs7VRHbgc=Yo$64atiEROZ?GhB68Cqyuz$J<20k#B+SUswx9UTT*)0xV!!yVq^Hg^ zF>7|Sh>~>eqtxr8Ndid~(v*IF$4`+ACHKROP<#TV^9zbi2`=>COu#E6OP`u&OI&pa z)kTD}0`5jpm|3&3E>#D69(CX;$$yq);@@JWrR|dEMXUVqI&}vL72qPI(6>FzwTssG ztB{?D&rGz+i{IvB+!Y_fk?kPv4HAA%P{KG#J`*dW!BB`?qgq;I+!6;kt{~Q_)8eDQ zFy(N_Vvpw!{;_0ZIG+B&fZ$*oQ|4@>0x+Pdzdcfbx-Rhmdx&{lGFkdbb=M@p0;ho< zmuAbG?tGQM```^;*wj`v0l??f{Tt$(W=lR;sN7-fegdLG5C@nR{cEG%pL+blW{+wa zn@!rTBcl{1Kq5a!+87RTU%^gQcLo>EVo0QJ+hcsLL^1CrYNH6!rsm_Cr zDd+h}|4$2W%9bj_7-)t{>ngA8oc{DY^Ex_edw#@pBEag0NW>3Y=l0`_RScJ%&xw#8 zJpSP~bebm3C0#dX>dbZy6 zOE6oJZvsD{q%?}|N21D8J!Fy45;F7ksMxS8Rm`osd2Dg~Z^tieU7Z7u8dSX3V^iSImc!L0wh)s)-==+{F zux93Rl>2&A@?rYOeS0#$f9rX?yLg(<^|jLx{G_|!c^ZK|!tTc6?Dl>STZ|}B@+Ay( zs|hRp>sNtkO*q1qm27=A^^lXMX|`+v*)J0@+9fBYG45X_8in<1ik?BL@>wjQ6cX{0 zd!sAGyHzbC6P|YT^&?-NI_u9iX)e;n<6fPK;t?&$!q^f-XvS(iGjWup0K^=rY%qLs zwARC8`&r!w9H7UPW;ciR?J-xiQ4QaCunBq|ow+QTZ|k5jY!`J(u*CeAS}1W#m=(Eb zgzhoKy{R$hcOrodcxjZN+K&bm)91C~;ik*~$sP#E2!su$PrLmyZJ-A`rUy1=2kF3N zBH;U;Et{)mZ(mV!hps>-g>zNMEZUDu%)X6je*+cBcXFHys+<_P>ERK6pCNnzvHTxM$>X7BCo=Il6H&Wbs#grhlA#Jg|Q(K z_zDs5*`x<~PN-MtUP{srE*#V@PLBz%+_osB>@XgO>4L$Vdp%TI$t%m?=9BCoD`jc~ zwztYbe$l!|3W+cxKQ`Ly*rrH_I&>~J7TYw4VQlp5D-SD0eJoZ7zn}b~mn7Yw(~y)# zLa5L2-_DFFz4f)u;QzQKN&z=g0ej2_GY%HMEVGr^<{*o@U%y~^-I|SYV7{!c29;Z_ zK1XP4)VDUBUWJD%1A>PFitoQ!LDfaihs@RO+eX#eraTrdx+Tw}%=?oJ(FNR#OLFcH z%xDp0_V(+@y@Eam)+I+JL|o=+oT7dNw>JWX}^ct2|w z#HMQ>ZN(P1H6lSTp~5+lxf&e_xXNP(r(cPw`}>F^=xfynaj%1!^y56g)fEC973mrI zVpC9ZhVb;!js&Y6-reH@M;|8Z*86KoZeHB>GC=cgJe`D39xVF*06^%n-9hAlWg+&B z$A6HOdJVA;E$8Bj7%EMd?};PIQowxJ?QLf|(|`T$-fU~Xvoz;V*YYr7@fo z?ixlo+ZvHE&SCpEaS(b&{Rz@nRZyf%7&%}1iB0?%TH!4Ts&rEb#f@K%hG~@aKF$DE zsgqH1{DTIiLa*=yoTkaheuOg4ValP{49|d8i4;$mU7i$-#)+#;YXYKkB}GU@VlfN3 zV)FM=X{y#8pH&KPmYH|-UXA!R|`v@)&Nk~FII z(nWXU8xTkf9nd}cqK!i!Vsa9?w8a4)~J0DSQKO9xkr2l$XRD? z^V#N_#PfoY(yM11R2ZR@Jya4zA!`|Qz$|#!ZUkNc-4^X zR^q9qYb@z~@jFOg0fz{5UjA4llQyhsRbDjX@QKQ=FZ)fbstc9c*@zjQ+MRlYZExYW z+9N)(kYike78}g_tv)GU<(k_yMN8Upi6aV$Eg2LF>>6;d8(ko7Xc$KW{Yf}biZXe&@%N=< z{CIG2(t`y?vflpD^!jgicZPgC3v%-Q zpXId*>wvNr-DCfDlmidGk3Hy_w3)v8x{edIiSXh_=+pTA!s1eWsH3lT1zyN6Tzt2M z)UWv`Fg^aRgnir$c3NsNWP5&o5hh3HKi&eT-$Co%;bV$xM?Ja9WwVnm3V)-?lD-Q= z5sOZ77(h;OH1}~G)HZXjubsT|_q=*utZjCj#2{N6I}E}k_KB_#5wpM&!|DbB6U83qZV9{c8Gl=HYwl_`Ivq`N&HzBIJ z279<&Ydt%rQ2Dp?!CDc4Mky? zBO`2nT#t~_n?w7q3YB=Eq^^C=#UxOkNH9C*C3V&y6|AFFL$|}zR`|{D<&gGsVQDl* zDyk#yOg>Qtw42VbWlC)ZS!G&p^tKj^*KVo%QUej8Chb8$t;N`n-s_I1hGUN5(QZ!;O8zHJ#(Cc zWddD(KN(3^cSwN57ib9zYuZBoYclJJ^=9Ga1S#HniTcq(`t&4vz&R4l%~H@Da^9)O z9=S(PDT$n30EHX4UzRUzMbng0m@mxoq^jOjsJNq}o)8y;CaL5qKvNWEQ?BuegQ%|b zrw;m@t!%*4k5Xzn$h{2be>bMtwJCw%5KvslIJEoBwwY6-yQtfN-`wXNot+VNIX#Z{ z)EC>F8D}g)#1IY~=OdMoCp`4DIwFh|OB%Y8t2f=%0>Cfr|{p_OBwjoiC_Hk2QFce2AZe8u8?x+{ zitHfyJJZPL#BO8pL~3h5*iduaI1x{^3rCwHFelVJKdf8@dd5?gG&R<7tq=F7xS4S_ zG11NoYX~$lJyT-SA1p6LdDuA!LO#(citRNvGsxR#_K`ljU9SCRbo~C1NHQC(T6%u5 z;+wb|-nO?ccS;Ci6{%2XsM2wyN!9m4euWg_`?olY=xrjEnwnc(V-FjUgPVO=EX9*d zu1{s4S^cWfIQIML&wR?}HUelHK%5y8nZQDg$lWlJ<$Q9vrdJet9sqgo)0XEO_dM*6 zG_Vm&qw?{M+y*oa&&qH#qF%%B-Le9(5W_^gIjI;NlVKMnso>jEd9z6(cC(+jv%0Bbw+6OG9N|-(KW%&Q`5m zt}GUk-25fx6Q8wEhtae7jwZ}<8uDWE9(7r-3tgP7v#NT^&VG0T9b^m%@%1?gd0tOkD($For}9ZO}@j+fw&`z30-J3sdqVsdU`^$DNBmO_E;ZyWD72k$|eqw#2Xh>OvNQW`n9^E`G5^rg@Kz3=`ws`V91!7f_ zh$I-O1SJs__DSnKRAbdhYKrO;%GB}QM6zQH=UNqs$fgPweyOg;l-xAhPWns zNb$eGL%XLlgj`bF^v_vk(m>0PO6=nkTn{^)4s)nX?cS>t>&I`Y9DiXi{Ly3Uh{i4v zu{)g&Kc;vq85`v5qN?ovbyZotSQd4`epjWt0fmG!|N6w?ll=Y(OW5pzB-}s&I)~D+r<)?p^2L#P; zJS2+i{O(e^Pjk*#byPD`?wouh$1|3@k=FxCk~DINRP;!n2@ho&{5vy#*jh$1XUKv( z0Jqt~Dzd)TlEK#;@#eAo0Gz5Q{UT?kuOid^r)<&F{(?YY{^-lYXM0nZzjt8^KKJ43 zoo7B+Ieg)iYW$$}S6@+Kya{>D(PecG$m!j9j+o2L6+#;O2Lk*}_s>-@d58R8`0I|L zN#dCHqLNh3Wo|60WV{UJ|rlt6Q&7%W>aIj-DTDDL{F0A{Fv z2B}^qA+L#5wo{pO29;vH$dfEerm?Zny0Nj$;uk444mP>z-$FOv5xC0yOs4gFv~le> z(d!{;z{Ph#k-@ldhc>COwdU2?i-wnC(T}2th}wt3>_URmYMl++{^L6as>{e4h}STZ zM7A%&fCnuWh&~6Ya6cq5#wE9e;AbL_P;4zhER(M(k7Xz%B`LZ=vL{uzI9WN#!jG7%ux&~1sA~6xm zUN|dUuFE5^cIt`PM8;^z|xoBXeUdOHbs(^Qxv}(7ira(P2DS$Q`WthD2aV# zPd44$jii3RqA5xt`&p(IEccg#s77AL$NTI!j5XPQy&;($^3&5>=!Jn}qc=lTG@lu2iVDIrgWVynvv6`t7Hrh_RQeo+u##9IEU5@}jE?SMz}XYPM^vO{1s6$RCWF&OORE>ch7f^u z0dFrc<#e3>8cu>NHd3bMu&BnAYmw1v(j$N1v7>IyM{#u+1TU4(D#7*HwsNT9ka9 zZpuUB=i6fE&XV5V`LctnE)&^ck_pka>H>u1*7b7=U9dtFi95OPcV=Y0wKNZLTU%BY zgKf@ce`aRJfvdClox%{z%L}h0&jG(=oWvH?9SkhYsV$${uj&B%-&?Y7Gg>UCMA|M< z-l}F;?IgyCPIojzQpOB9%hevpC6~yrM6Oe0WO?*gvN07F3+d?6bKvWHZHyH}kyVFi zsfYEfd#4ZG!M)?ENNdFIE8PAb4%HjfjUwkSAKm;XCP~8u4it|aXArV&N`goeyMA(i zS)h?s{`Ek9YMOyi9dsEI@tSl<*c~Mt6M#AJX6?CE4I54Mj$$S`-B!{@9lcWfW z3Rx6x^yTNaSGlBya>eZ75Y-}uk-B4Ba9_{*a?noFb8X03f`+QAPJg-10aqO&qJxKl zC#z~yYzcgrz$Axh_dAuz2=g!S(v!`tJj{F+RvIeMF%_EEh)hw(3!?;19q^HlWZHaoYJ3#RF~b~L8-;` zT*>d}@}?lm#*54dT`TRQ5%q<1pPugAala?N?pWOg#*%8nFDUY_-OuGhGu`F!|(GBhF@IOQ?KD>#OMb`?0yA%>bb zDSLBwD)=dK;F&=?36v59VGk&0b^PGS2s{`uKS&DT(tiSH6wOh5?Gv+De|*ukzd~SM zRaZJ_lD|yWrcllHS6V~7NyNV9*WK_DV)fk&U6=#k+p`e1CRqoK`bjrR8M=mGUcz<$ zUqvZ2HfmuKnz#h62>q0G`%O!H>zQJ(5v=n1A*M?<%k>DZ7dQ*q-9fCWjQk`upapsU zaMc0dM4II8Mgv?K^z880)dFbAlRT+u2;}=Oc-8gdLW*J*mU!kb26^I`XHS$a+Ch0su3*)|p2s(# zK`2<1P5PMIG*_i@@9XD*o#vM|HNUD$k9X~SLGW~S7-_Y zbz^XaJwtn=Pzt?jbr4GW%uTasyX?EPt$avtRbe2;-6-0I5{Z(D;IBUcvZ$2RRjHS& z@P+8#zPgHTN)PA+iRYxSZ>a30>`WVJ3~|C4vg=0mXhD>7V9%=@WKOlrUBxT< zg(k+RDvBq=5p*!)lH*Yqlsn6DyPBww8dcVLl?#_9|6uN@3@a!9TP{^&MUKrLGG|gw z>z|vVzHxp0$>S!ku$=~~yu$KVVSBuiG1*TMeDnbS7Ou^%vlt5-*86Bj5 zr}sqCj*nMmT#tT7hGyNqQ|&^dt($AbX#Hf>Gp9PV>2n9B8(NRapPcb?LH_AB~Y==kSchn~QO#XL)sZ;=plzk?r2t9r z`q*0@@24i0P|4pLFD5A|N*3|6!UR`KFH7h_l1uIM|q}@LKp#$QT8cMUZk08)VSu9R5j5nf_@P z7VI2XIKj6$t8;$`%$tMXBm{yl=x|tUEl#sf7n=wm{`sHLPunlbkHeM+U+10A3BtP} zL9?bdL=587a5q_9uthf{Q|ENT?;=cZe;S`@flI?*oA$%_=6p*2vMuiHLZQxW7VP9+ zRWeO~l`A*VytTXpVTv~xd!ajWXVK@4l~7a_113z|bF+_f+7*(REF;tS_8!^cIXjL> z>Rfb^yP@NV5WlQmNw>Z=A5W{t@Jm${J)JIJZr>I&U}Z}$O`n_9JpZ&AKayi)V379E zb#O+8Mn9P|_xnKLju?nE8TLHLip{4=D;{(2_GT@%oLS(Mbe{bpweNa+H6QH@9u1JPa^k!_ALBUwFA}pZ4yPIgv>aF$w^{RpNpsFgKQi2?iQ5;(R<#cT!ynNgR zbV!7c+%|SnG9%D?5^Tzo^Qtb@WaI&lwWeH+Qz))`Qai~k>9CP?d0q8*Q{xgQdqzdrGT*toq1U%hftb zB^-NB%`ab;O0LqYoU}kRG&&>4)SyeNTvYriqFC!rWlU{aL4#e*Y00!h@ZIA~+|*Aj zxz^Ff{fsawH288Ero4g^Te_SUpKCov*xtS{{ zjX8~Cde;*v1g$cnOUrH=*-MMP&Ro-}1NxS%sv;*cBV4Kl%Z(vy(ap#Cro&mp$0fma ze`6jcf3Wf8wdZncLgqvYEkL-hX&}1gFQv_c@$Xqi$Xb&b9W(+s5-GG)T_5Dne78`b zp4MNnKLEEdqjt*$5c@8lBN%nY`yf0~R{Mzl{V@2JLef5Ta-L;~KLx2k8-mziDOv#M zBKQjX2pr2Lcij4wgO{VLcj-WXTi%LqlB=R1{GjF)>ht;9RE2S{34LOs(QlnU{Nc09 zX87~YwqM6)T{cH_sG8N&WhN8cdk;mp3Rv>P&gGYAIODPh-S`KQuD&tE#3se@r=9Po z>>@U6(HzL2VP~(8i}iZH`|`$}%Nn*<3YI&x-WF2%JN~v=_cY626^OUk;$P4ke*G-a zr{L$sDTDP}ETNF4kK})7`sT388))mAY&V%FO}07Nwr!h}ZF{n9+ty^;w!ZV;d%u5r z>O7~X`n9?C+G|VLMmd?gPiEtpu8!6R03?7W509uioG{e}Kik($eC5+i25UBxA*jS6 z&7?7w?zXc75*SG8?9#-{!O?$DZws<>t3rR%_pAq_ihnQ3xjtGhy0f@BxLxdFV+uCKphUaz5Cd9S2tJZVEZoFDab#3cx7EYD3 zN78eWD4At63KQ7nZ|%j?7I3|8%q=s#SM{Ol5ToXJl5ORoT+f=@Eev zq|6CU)i8QDc@G2XFr?K{D-#;*H9RSL3rkitrRGX=`Y%y}{=l<~<8eXXirs1<$r^($ zS}saMT?vDMsTMtKI?k44R1q5O0&OQc?4B8y%ttk4TvN`-yDS+b<802qv%hJZZ!u5*@KsN1V>z~YD?xM z$lp13w^<@il^j6>>u(yACr8I1I00PovnemA8p3MLx9JUEBIXx56*p z!=f!J-Sn(%G4EpBlNhVRBM~BQKMs^(6aD=SKxDa_xN24`O6U&!Lvx6-t8>d6+g1=?4)9*he;H0E|9>ukM-i@+xLwGiX^7Y}7_!3Uon0VmiZlG3rr%E} zw=QTKIl%?gFk1VcRYvo-gs|umdp`H6cipdI!#@y+tp2orr%tAj=fEpS^;_4??AIV+ zu9~#xMpHl?gWX#&<5jPy3UDy6n>M(BHkRXj-_8p9w(yNLKslPA9__hDw6NQO;TpEx z`!|$NGdh|6bTV~oxEA0CdM$A5LPj$|ru((a&A*orsoK7eroZm0vxWI5Rd03D$_p{> z!EstVVMpLvVBq}Yme$SwGqh&6aFy3&qCV#;e0!$j>Xr9Zh|F!WoRyx5iA$H z9mGcIh3fecNhO{kBEPi_gvU=r#iJr59fvfPQ6k!XJh?E_jEvbb?rD+wR5AZu0bhr| z{fE0qyts2pSTe&c&>{?BlJ9Fq$LpR+LN!?>x5N_AD`1k#xmelsT^(>JxDe%=uOd_~ zx{}O?x}~0Mq)_xc{))mN6ZPIgDJY6|nOweUcM@B5>P;Zfzy1lGc$tD#Fwv$n!HuZyZzYc|J60vWdC z!Q;$@ei5PelE$f}RA@wPtoWe)_1lN2uLetR`dx9MJ;xy})i;%CI$-(#or=icNjq&z=W!_Pq2q{cPNA^KVtwG!dpb zHMADgS{}G$H~ox~bZ5@KDENDponBvzl`e3aTF5=@aGGkadY;yBAN9`G{cIkWProZ! zg-ooMY@{AZZ}Fz^)5k)P1c>oXUz%j`wEkSm_9JgK8%S})*!S2U?*A}=qtW{Hx`{8? zS!YZy6F~$7{xF~?Pz^->QeA1P0Qw_>$_Jq*WJMu}Oh@sz=*)sH8&>X`{y53)Y8BC|D z$@ix!mkEwD@d1 zD?Zt=xYq#~9oLt-AasPds^at8&_%v=^E}A zRVN(=xhMaXs*g;UG^@n*C;-1Y$73TnOfAF1$$`IJWw9)dfb-{zlO?!IGWnIilB zkEbbRM2$aof5m$94M!@qo&B$wvJOx>YGh_e=2b3wYJ+CvyuZRY1>$B~z^uE^By-SS(e@ z_s1av)xxoCwQbHWS;S`}*Ad2##Jp>Ml0;gyHg>rfQG7 zU(~;rDjIKN>$;J?Ier~uy4Xzk$aBekuFzzSGj=?^Ngh8-&t05GPu)dN(LG*QRkfd^ zxh(6h*Bokjk>Ipce$c_`5x~)}(B42?2zxBt`u>I8ZI1VTFXG5U{sCaOrwp=s_dht| zL@5G13lRYqbtF1JMvV17=hwE26=$|{t#n9W{Nsc@Xd@**&y13uV-u<9YH;OZc*JWM zIZV2q-E_r-t6PFl+R*>Kg*wz59hz~YM3d&;3*yl#R-1k9itR|_L8j7wq6c|@oyJ5z zhJJ$zv|m65)zqL~l-Lf9#gb~85odq$wJ`7oJ#OHm<%o^q%&+tF9zN!jQu^iE#17tH zwKZRs4R&V6s&HL#o36{gdf9qZRvadCtDMjK^lcBP7b2*un($ot)GRj750_cOsd`N2 z1m^+|xgm1=m~g#U;A{@O;(%TkbnAVnrtVb;Ij|p3)^a!)qab~xkBB;r zqtY5Ctd>YmXjznt%dS{+u3*c~kjb>1E9PvZ#;nQ^$vm4a*K91uw26?ZFr6z`59Jj{ z`eP!O<=bMerdyJA`87f5u7mf(>$6T2u1^YjzJqYp23cqwyb0` zTCl&R9gLO5>W$UFqn!-!nFsC}F;A+^HroTu_+G$TDps4_bq4?3z5B?g!A0E8{^y@j?Xc_%S~vj<&O*jc9%y>_-dec_XHjl# zE;!Usj@$*ZJEP0>}^ZxXhDBegk0h{~w;x+4%sONDIQ(Oj= zZObV6mrp6F5|akhj~Ypo>l!TK`{FZwa;|gfsHo)n-ZSjyKJ8V zML{F#J@3oOBayR~oMTrlB+IwI>L;Jp;duN#G&>7jW4!_^pe9BgIAG-BV*7H^aW+oZ z=1vw{<4f?zKW`2gW9mA=z(|dqJlfmWfR}wu0!)8Rrf&|cs7It6jjzmTcPFLlZUvd7 zt5jZvgReXOdeR?DAX6VAs5C}MkvuaVj`7WjRgXN=2n?qFr=O#v-ub(r=r`#(tVzLm6-MyWOE%^U-BP3@0l}T9F|f{jIwD>b zkPb)bq(GXcPqwRydBQ{Ap&Ws$^VM4TXpE1B)|r!7rL5u<#~B|1pcrS z1Bm4T{^Y&!;2C1GltM&a;BhoFtFQ0|Wt@Wogc)Y^8Shj>;PQsdTbT5=KHY8+F_L^LRb7x1E6KVwc zn{{z8vy)PhuU(Hi2ZnCRaQ}DZ_NZaG~JgiCk%7s!fEw0JtWQVHJjbrO6 zmVAARWmQ^bLd!$c10aU~U@Q7!8w`&Zx2*9{%`PK7MPqCxiP2zZeSq1%aW8%l<_~oW+thMyLvH^iy z6mN(vdyc)Q?u3VSLw+{B4swb4+DY+#RUd;*M6t}TbU>ygu@EC(6~t!aoJ?l^seB`z z^QJKShHFSoY`%#SOP=}KfeZtDXn=(h1HSddU#X}>CUjTKf&*O2T=L%ywd$y{iA~B+ zSfN{A;s!Oub<1&xRDCOBgl= zoKWF4YrREr9VvxHp|93DV|k6J2z)LiRGY;F7sj`(#A;%1ldeD$kb#b)^UMd?`yb-i zzdK8ebl;AoJ@j8f?b}uGby_}id=2nuaJP}8&nWi)2lFPsk9g;`Y4YMAkBY1XE(53C zByIducvcRA%9;nxS{^VFGW^h^*7QDIiMq7?iWfr9# zkT?<*t-}uo8^SyaN>mvM6&q%Nj|}|$4G$lkS7tkDyAQOXAr_dV! zef;CV%x!LURTO}?*YW+z%!K_MxxXll=)03g5hg-9d2zMVyv-*~cm2z59fltW~9I*JMrmL{$AX-hb{%ZK|lHgvZY zo!4TAObn>*xw5ovKdiRUlPdFxq(5CfnySWYsHpgCB{g`+SDEfgz;;U9C>e2PG74$Y z0q~Y5%HkZ%FEnX#c!#xw@(e|!?sO|rQ>$+Po{5FxEo7U>_j*-{$dwtd-Z0r{`LW#F_P5p2Pe_NO_nRoyjT(ylK|2i}7O4hb2!N|s}HRgzH(eF$NvCKeBG-0w?fV_!3c;(wb zVx?A3k##&GGEC^}&U=mV?wAwE$u;4g82ffC>>!SAotNF~w%_?$E8zw1K{Gc2dG@p_ ze$UHZgNDab1XEu@CR@m9>$+(8i~n4_L37Uwz_V}0SzOqZPUKAcB3@WF7@o<-c_Ob= z?TtEv4dR-I%ut-&<}O5LI>=#xuY6{`13q0R6pKWoRAW*ee~mTLb*WgV!9vA)jU~0x z;t`$QCPGB0L4r~&q%$2~H4Pz7LJ%_Jj}_W4ek`zodTfs)k2RI_xC8yQIG{WzpdU7+ z+@J!71h2n-qmJz9dOyZxodF(A0`+2}Gh0W?ZC4x>3aR~K_-Kk7ID*i&|8iVgKU2X4 zfs@ObVE;Z4&GE721p^ysY@ty19adB~)-w?th8A z`hl#atuQ`_sjQJwgYPVnMsP|*eOQuCZwHAddWmh_Fmph5|Kn;C!Y&tqR0M0Rqj16= zR1)zG#_UgeTl6FXf|DyiLaj8_l<_TOt3aGXC{Nz_x|=V5j^U?U?qHzZH%QOhYnM7A z0PUBiFr+l*I#b&%);`}vAFv|7ZAY6rogJWV|ddr%U#8_S)KXq84$QRUv zoclKTuJKs-zQw{WhK)xs;9$+QwmCEWNZ6KgthdseQL>yczQ_-Q8Ob$f3_DU6J|r6j z(_^=3uK8yl(eHWv^P@-Zft}|m5=Wb;oG3UJ3 z-JFk4|`RlluX@MU<32RHt_Z$70zjgdR-PKMQ2xvYv zzIzl<8q+Eo4Ka-+$l?@=Au#D~B|oI;HX~4D#?XZmN#u{?&}?DnH&q%spl!4?T2P*` z5S-$}O-dthDpG2>t*R0jU^rCr#AJ)PANK$Hll^-ohwBjhj|B8Ucs@4@kUAiXwHYq& z+>d^VEW)k5MYmGq5*C1wuWz*k>eInDs|{oOeyIh!f^L6taBH$XzmwI(JV%j{ZL2mV zOb_u6QS3H(GajL4!l%a%oX-E4UEHp~n1DLlJdD-r<_Y z3@RGrYK57&97{n4%=x>Sa_adgQJQ{Ab6TsM#NFE1mCN{IvlUBbN1sRG@Fie3X?>Zn z;vxE=r(4v>ZNn2STEx=1s>lb5xbP0B%s!_3hPRm0al2x5tj^nN8`aZU5=%|@Lew@2 zI{bh_Gg7D#%}mXb`%_oiVbwa_?)x;t=#i1%V2Afh<=FR^9C+E-=fQTdU<}pB#2AZU zcS*S8@1?`EjPma_Z54zwxqp6_P1)jY1zQ*PlAIGKD`EJwq13uCN}eR{udAZ_2qv&%7@^mij|rymaNxQA|VtK z>2p|QuInxGl@ehj0`f2I8SLh(#|Hu7G;BugLl?CH)+eJOvc!q~4m=4CBd&;u?Vuj( zuIe3aeDu=Wy$n8J1C#W|7P$*5x%Z+AM3pE=!L4uQk>v>uJU4&2>kLHAI<%0}FX?oy zos9%fr{dPY)2FJ}#Yv4@I^YD%Ayu7GJk9yqW`E@HtqiS>J84&a)R#+W;}~{7{3?;8 zzs=lBZL>w1{74%T9HOd8fFw5U0OQw0U(Q(}e07@R;8Gd6StrlgR& zb~ncc=pG!mY>Dw5xA$Lo+xWFzqRDml-iRk4b$?4vb3eoD>bA5+3{pMzxV+oGb*5dB z{jB!htht{P29AC?=x!#PAA$hVFn%kUhlJO>y}7#UO3joGc%lA%DF5u7%|f)j#~=zL zm^?PS#?)6X2I*4JRZtqti^}>FcAW_7jEsKn9U4LXrrQnvzZwiFirmr zPHU)Zpi$cynQaUTYUxG|%& zBvRrp%c!5f3nPdV+9k`eNQ$fgk-zpuqKy?u%POLNDvP=#_vK5@b1iOAvc)~8fY^cC z$=hK7**!);ZTvg*g`QH0?E*|qKAe1te--Vx`ku~++*|~F_Vi!D{lD(InI&pF75?@xxwerr zY?)s?8r}8S7FngG6#{4T(R(`7n%J`G-cQY!m05{@D~yOE02~auJMmp!uNQ4L-tX;+ zF^>##&`IdfcUm7EoYyI(HMadt`p3X7Q`QLi?KaFsVvpx-Nt}z2~ z`0F69XVU(cs-p?h=;d{I9?4TTueZnaf$`+?Q}dmrW|2yRxtjKQW6;r2yTyXFP=f3< z4HGHA9BC|VE7it!kqlopq8$E=yrvk|IoNnuIx?L&xypk5YUxbIvfDs4tMgEQnISnA z!5C{%W;!gCvw=(~Ha(cv>umTm6roeKmccl=Sv@E^^XW$|PkHZA6_zHnj_TYY1B3`E zeXby#BU^B-FrX(9(+ACFSkzFp>l_he2=rhN1SLm^d@^*1#xu$Dt~4mIu*zHX-A%Va zu1M*uNa>=cR0mz3>(wZzUw~^mhd7;VlL|Gyzi>&d3K%>X4d;UpjgS!0wpTMZhfkN~o1KA* z*Aubo-IAVGxBidy_^{!dQLLG+THpCP?wDu;<-I(!+@>h2n(m~LuN&u8Lu=>8yEYZ{ z^o0~!lGKR?qh;U!!#l27C>6_Y}h072e4xL`5*tyFue;M zv$Z$E*OzVtiAG~>Ku2CZr{Yc{#vW+i6lCmJccDjSR>oncb#Tr{h%de-971N1HBB)h&r_cS8`5HXDJ02F_yY2AcyReD5b2 zW-i1*Ya1#0z0{f=4o`25cpHJ$4hKhck~A4=C=Q85`{7R6GcKBp5><9_MVcosxN@Jf z(xg-AK#YPWi*3oy(5L|-qmM!c~ z%-pG59p*&iX=5I%gvoSkzpq%VSDI#%KRVYjhNR5qIYJ$V@$#_7Q=2X>LH7JL*BNvD ztGeXx`TI9B>lCBgKfG;xx$!gJvE7X+2L!ebx_D_DJ!L67?zx_yizKjBbtDJVNN!Up z+@#+;PZA^r=SRngVHKgNl>z9dC7jNGTASVGn?rHOvPA9b!J&1iJKCm=-Q41C4Sa!pR_fm8 zm!!mx2f_DpSGR^;UAbDX6DN*2T!Z-mP!S7CnNFQkCg?ds^Q-SK?$0eOBj&d<^sm1H zK}yT}OKiOf8%(F%H?NoGg#L!X*lX)seIL-=2)B0IAVt`#AhX>>+p=#; zU&&ldQ@HDUtF~>*{FWMDUnYC1ue69`W1T2jbGy5b5AG}(Nd9$1n+%speNoT<*GcA_ zz?Z{)Jjm$0h2?EwQ-ECoYE33Hk_2p0O_4(+c!G&D#RNl$pbV7>PpCN16Fvb7tG*ZB zAEOxs1#XB`g@l$%W=dr5_6KdbpUef>MA>RlX=yDIl^M$3$HXn-xhV8daa^#YWHA?s8iBc?$U%LNz&K$E85Pvx-*o z^5Sf`?3brRnIySefj2Zq_v51# zcHbCj;AU+?v|pmKw?l+n(rz`|{sy*% zOYxMpW$$XLC3xp~&2D&zsopIVB zx+lkr8@_7@sV~2(r- z)Q%4%WF+QHY&+-&MM#5pZAm)Hz~0aI4KxXXM3Q($YUfFKo3oiTQ(W_A!N}7?^f53A z1a**kp0F691?WLh0lp}-fM9S1`fuq20HmVA*D+}7f%?<*#6;TyEW~qG{ELKelw9A| zyWl!fxtOVOJCQFjRK1gbc4KpMl0=$IjWtz?pnUC%{W*Zr!TbMn0d!$iQ`8dZ6qpJ# zvo@XoJPBo0$F1iE4sO`PimW(WY5?p4Uba4=CWIk--GBX7Q{#T-y>V}oR!Cxk0S*G1 zZSu}VIlmv*>G(r8JG?xOUe$`dJ$#>UESYq3Qjt2JVgKVvs*Di68~H?p3n`3HweDv# z76h(ed#9<>^e^`~l4cYqNEg2>)e!0%TW95NdlLHHPRXIc;%}q<-G=?{*mC4pVHfAq zGv3-rAH54l;Xst5fD#9LFh9L>)h=t%Mdbh_;lYuzy@X-$UrA3N&J66cP>GlZD^YR; zk)OWtsnOkWg?8>8)?7hX?O zq!LDHDlcJh#N`C>3V|Q<3Yiw@>cEjY5s3914kFL4e5wPRWZ0QcMNpEZ-d-)LIJ>)9Zm~!0kd{4Xp zo#rqC?!Y=R*uY7RVi=D2d(Y|L2+)h;ctPr!po=@t7`kfl))l-%6Bh`+!5{aUf)xHM zkkiycR2PdoQ4sEm5W7X;^Gz7B+gGLg$toP@R=F68TuXbMK)v3-X$|ML94!cza~A3r zy!l+HFF2>@{yf4zc5T~*G0|8{6L{NLks~0!vtqr*EPXADy%`nfpe$SplTXWG1m!@} z1JNU+BqT8M!&yM?sm~WVo@023y`A+RKWtzJgDhX}ce)pOxe>||bL}zvpDFcq@2^=` zR7v7M65iuO=cchVHdJIKnSxms5KSMZm$4mn#-IfV zMg;Wj;U z8t80mn25+Ii<52#Hvr%rJ^R^9{FDF+=O}kWO=U8pi72czy@SLUkxu+r02a_#^i@6> zJdP(h?_a(7!S@_iLsTDQYIX;?Ux(z*oBHcdyjc!!dlu2h>cq9=UT6lu7e1b<#VWe- ze(Kp{s&#&T7;#mf?r=Xp-&KX`kh`Yr+Gi;I#ZT;vTM-JXLV0H6=FYbt8ydKkk_H@!T=MF$?7-DK76^zDP{>d<+4@dvdskUX@3=HB)G)_L#zm$L6;8U!>3 zaJ3GBD|R-KsnY7pI2p!j>-K$}vy+UlJ`M}T1D9Lly<5=O+>R#v{vP@=H{~Yn=}Qdw zdT!~47ts4gZrQwLC{r%3-g+eLP9cjtmRU;$OG^=(Ubz_!^wVF*T#i53n4mM-qAJPf z#vmDa2*dk!t(q@c)C&jkq!xT~U9X;smLZqN4ys-VmnyK95t|PvJf3bJOdaP@~5I*Fa_Wy8GxYz58xl z^{UMbf%lTR?V~?(6!71=*6&YjUC$zf^%J4G0rPXj1RPpJ_D1N6M@K&I@52G&ME86F z5(T0;O#sSFV%Twb%MnjLnUvB3qjW1>8PSk8Kw^y-)ogf`G4HmeS_n)Qz z&#;9I@d?3k9Um!CR?YMnmiV@LV%p-RgN`%j=*r7?#y>FdkSG`#RjvDyhpzK)0{#8h zHg8{unw!2xGIq6%4Gf>}XUDkU>X7CyZor-m!A^?fbW&@5}i;Gf_drhWNQyb?CLt?BAX|+ zTODLgXWZP=@7rkx+wbh?2u{AWZ@a=kiLO$FUZ=a8Es(39^orkREeOV-O=gV3#X)W2^x&aem@t-0-31qS{~OqhJ%Prsspmyhc-BwJ)VLQfVWw1C!K2SYqzIj8|`In9dUcR=g4Av9T zzSXSw$9ArL9ZYoEELMyYAc4Dy%JW3b#rGczS|^+QAq>i<&y!@#n#P6vVvK$d{8x3Q zyYbWzdoZcm5`a4ZDrc&>gmG9$!k`zNyM&p0zivMTU$UCL__bECX z9y8bHrame5n^0SQWfPbw&kd`1-?v?Qh8ZcDMsk=cY`?c(&Y~L|S&{?mqi5&Q_h-6a z%cw@4?7knJ=1*zr%!3x>v?jN@x*iZ(`(ow%_C6=GneSb+9f010$c{I%o07XZ)jJE_ zEQ0~$Tt#8yd%xZjmQnXywjkBnjO3Kp&v@T~{l`l7_I|3whKG--_c)&}hp7}2|G3oi zGZ2dWPyym8XRP$HTu;+as@-pgxWztIVz|vpb?OzQ9@eQR7UCeU&5w{C9+EbbM#y;> z&C$$Tv7?Cl7d{x&slwv0PTL-&bw^kRu>x~I-oE&cjeoyHGo}a7vB1~rSTA#R> zwC)BG4`gfP^P!Za`eYNzV_7Zx-#A75dpUUw$W6x>XLlk-$BHS>(x^<#Bvt%%cvyD6Ei

ehWurFKv!p(r27)CoHlA7n1I z{G7UU5iDhuOiHLq)eA{S-rZ%y8!_Fa*5dR!EJJv=jS!hQ%d~Em&w;r{8h3SV?#{E~pN|t3nd?JI ztFc^0=4=`#SSX8Q4?+0USmBw9TV6N)-Ej0vsb4Wz8=Hrt_tLMkMVlUIOpi5N92WCV z>&|isCw^^_B9VffN zq|XSo&{WcNA%Ha`PFEK{Lb{bwXJLWN=EhN@8TN2Ie$8@zTwmoYZm?9Q;7lOP9XB++ z7k~Xxz9j0kVr(&LNW?!GW1ZpkHhc=xvQXA$lmnbv<5vgmXBzi4ZBBjYKSPJ?GZ?Bj zRxlwU7N64N(8L6{8Z=jE`p{N5t@k;IIw$H~;2!(%PDQ0-wu^<0>z-VfKjGD>C-uBF z%-l^joC8$_V)I97G&=tv;I`bZ<$yu&CU6v8Z$0UFjhR;HxRZMkfO2>ZmBAxM1HHXk z;(Y4j>hHGOOnRxFh{DU=F@JxoZFfU?3_xNsYjxe6TX zIootwm`yN#3IRd^W%h)FoH2tVQJN_+y=MEG{@A;Rf_Yy$L1Hn@9AIfLhAyItAt+$1 zmr?u-{LbpGv*(ejcAce7=oJ>0m(JD z^X2N7VIwSR0J0OlxpPl?NDw&HPfq%D?6HHNNs3zQ48>rK6#TT^rTT&rIoS!FYYN0gEi= z#ReutXgIjti#Al3o)X!g-bO^lK&3Bs5)x$u;HA$rKK}&|k;WE1Va}h#gBW>BmW-`Z zF}$}gI4)VGRb7cczaIXv+gdTbKgFiSopZ-HADHLJAs=xuuB^P1vV{v!q!u1^{=}N(yA~YJn^3^)jz7HUp@1dD6uKDg|hOEm;;6Hb(4I+?AE$P&PgGezv?@`z>^y zqN69xFD=N~+2zg{>FDs{@w%o!uutJVanLU|oE)6~kjMzK4ozo56$K zbwNJx&DH|@%bv1XgeKAEM2ubcWjyFP9*R(fw6bMYQoR|`G!S9?9LiBi{xyp#luHgO zZTewjdL?yYq`D$-Bo?I*nyVT(beg|qOfmkKv`5UtTf@4xUDq@&o?@QVBd;?WGBTQ9 z%@aZx!BcfG^w&8AwIFPlu9&PVj?9z|I;NVUymDSOKL-{Q2lTy6lqi^LMf4vKKcI3t zo9@T`I-iLYl$3N784pj;M*ve(YMWa~mTJYosq^gS=0vWZ$Bda)mu~2gA#kdhd2=J5 z87BpC?TtQ#)=js_=9mdl@gX$_nYM3 zhiepM3953qd;qni2SWpQ>%5%0J7FHN&wyR!oEK|aEdEJkKK|m*p<^pI(+D;Y7R|=1 zw>Y8ywKoU8vN9NRq*L`~cLJwlX_Zv|=YCPHc2DnrkMQ=vLEvSdc(z%b=>-(iKWe{w z9aJ~kpgT=_%!(=QZ{9jU)tik8X1-j(K$@-uCSKhGSn%;Ton0T12{88>wpUi*o^R2G zw6TFtB&8WqF=5Y}2?<`8Ebj+c=u<0)nmV7cy=uX2HbaNmw!!217@E^4x0bYn1qM!Z z3U>;<3lMm<35)3Oev#&O!lc}7^vY8%et@oJBb{Z9Nh?^Ep zr-J*}NCBAsDMJ;r>&4nkypadU;D{&+I=Z-r2|iGxyt~_1cfB=O@9V?i(XGX0U3WU? zehO2=eF1^5FXQP+R(GuyreY~%&YXyjE(Z8@uHaYw-Pt{)A>=bn{W@OSq@+MUv-pH01s2%f;6^*wdI_H==wMX`~W*Se@1|YfP$P~-5dxd^qGDp zXouQ3{8?d@bkTKDc`rwT9r<6dy8+ku5ehTtZp!D*?ui#3l9Cgtem>7hA9v--2kOt( zKR{!Nlybw6n9zqhSCaw-2R*;D2)Q#4zPI?cKM%3D2);8fx)1yad~xw*A@9+R{`u>h z#O?>)m0!0Rgm%uc&i@(UCmcM?aoq@kaLK}}_tV{1Lhe1SXoz(?f#iI-1nu-gj`gV%;Z<6 z=IRwbxr2B^9!87uuDf3BKddTZ(q_zrqj{jG{*u{7OF$i)lrCs(jq391QEe%!vO8Ly z>8{kx?Ot&UnvdWDzBjhj-M4oWhAXD=ea`q+rf3Yho|%1K+N~t=N^pG@PtxA#Y6BYx z?<#3(0yW!&YL<;!*6TEFvS=&G$wgrrDT)9JiokHh6gTE@GYf-9N1I4N9eDBiIet#c z!V_bXiyppGs|DvzMh2Jb)kX#19(JW({lk%hPfvO}8ayX6 z!+KVljqDL*yVs`rBugXwrJm#2qDBlp_kc|cEeV+k(Pe3Kvo3yQw3K*ZUQuW)|5osl zrO8{T*6XJk$_I-MjJ~zSN#E`dzw8kCF4O~PMy!+i)s?LhgW;$)5a9#-5#2oN0tf=R zNC7zX2Tk{y>PXv;zEe-UT&vq81@Nv*h};G* zM@C;B&*e6vqLvLH?;Io60&P;X{rhI4JM=A9qpdtvZ~!*Z|6p5cYO`Ip$*=zBy3-oKGLRWP)G#3 zJAy@Jw*e3h;bG7{iwo#NQ3+vT_Wx@1rmi-Eq+aAM-jp$IUYPOe&;J8``^CS?azJRRK>6 zrgL*9%YPyAqY#DQe~-ZM+O`_}GB6;%*nCf(xjQ)#$H7gek?2rUdj8b&+j_K_MA(zYP4M#JZHE)c zJH7HpMMD$u_P!wFVcyQ^5sd8F?QgQ-;%c|&!oOI#%@+}}Fg#6?_BqaTdrYmAdxkN1DROgDCny2>-`2%3y zZ$0i#ZQ36u1LQ!<7z=lJc+NDxRz6buc>q*8&7yjFkJW7ho?0$~CFWw1#+QG_M8OpD zyc4jC%-@qNhlJ38hva3;#7|tEoEsQXf8NoVY;#~@ZF)R=H0EgIv;YhS#UV5MJEX&$ zICHs7mN$uD-`^&s;l$($@{^lZ=~bZ88EmTQeaAJ&$R5q4$Mp0NzV~ode@t)ghXK=T zx>5sZshxT8HvR_^{6k5P-mB+YlD6zIhJv>GgpMVmOAbqVX?cRzSR9a@Ezur9K(3S-b1vz`p`j68@@hQ#Ar&7;} zn?e@z=JjXj343~J8TLsoGaYv{6k+Rc+UuwL+%No@t`|rhoh}XKl8luD ztbZFVSdVJ9l|FbngaiIv07l}c3|b>f|Fm-f^|ZT@ zWoBtWeM7@Y*YrxeJOXAScY0H-Pd|4}&Ns&~jk(F!S_geAeaS-j!l_UVzn>d($%2H6 z3$JWR6Skgw`Ze<49Qr@b`x4K{@3QbpF8irWOK1g;EoU$ zm!9DxPMtMVw>g5N8imKHt?Je;FIFzs(_Rg&*Xv?Kid7x! z=hb_j=s?EDt=ykK@f%&7Z{IQr?+5phqIRT#sc?M~8x0x`;pn`;gKw2?Yig-V75Cb& zRqAt#nGzQlUqa_M^XaJskNvjIj!MiZT2R}4vsM}habgW(_Cdu!2}o%TA*J8ZSWqAS zrvf*X7KDh0Ca5G=I~JIYUdEifn~$xu=}Iy*&K)_^+0D2@V~@(-5rNOp|J|;n zx;k72bw2+rKC~_4`q7oJuBXS2y3PsvG zevgw%zyC3(Pj)^|vz_5+dYjryk4_l_pXy0h#&R~F(qKweF4Frwu!)=bxC@8NNgUYK zpBV81X5E`M_ZY&|zR!wODFM;{&5VqdGkpBQ5@Wr8RV^KV>^si)>A)6McSC^i^OLT- zdWR1g68H4F=>xvCVH)hR`p;CPJg$b=Lz9RlBq9omWwAMU_1NIARwqFwQV||(N5`di z#kTbZ?)^L?hlep~Fwa@Jf2)L@D_BbduDg2{D4?At`f+_SD*S-e{~7q=PA|yYMqi4o zTR21HSpW1UR64sG4kIJj;V>|?O-DD7gVtJ~ddCoOM@|`(v^{z=Z?N)wxm9?)cU=Ob zZaN=Xs*}i}dKFuf-{?3-wEN!lEv!bbg(}rq0SC5}4lJtEiB9Qw`N{%v&$w z@pbZy!9jpUmStP$e2O5nORjz0L2-?6&9q#=G1}8)?a-pAIR{ww5iA46l+7j0{G2fAb9`&n0&2;ag_pG<<}pK6xKRpzjoKqGKt$fr-AucjAh+sxH<4Ic_~^*IO#`})?K9yK9Dh5~9Y zIvJ8?9Fz3*({XTbXOz_V{#v{(m&&>r2qj@N6a6P?2BX3~wW*iq#~4RYhi<7!UvBh| z4`&fNfyj?ARQZ*cIo%+>ckj27s75=Uw-*Uw#~;pfowpoH{Z=Djv>i{1w@9o->pbzB*A18K?D2dDLgy5Ah2&POk`;d_p+ zl$Z!)3&8cC86cSJ9mCHYRb2^qoqx6u)r?zI>?}yab=k>O>dc<06~yQKwkgjT8KmFm zkNb4fXF#Xr4|_}sCEp*4ClV17>AJ0{l|L7I)!CulLf4DxW;%xJi`8Z*f-s5ZBADw} zXTI%kKdXLam!%AtG@6y~4ik;ty-yx4Cm3nDyYKB0TZq&;@~}r`_@v>9cZ*23XY z+7|~3HSTA@5lS~Wmh8DsM!4j6d)=p=J#U5H7iZ@^l!9KP z#x&*SVa>dSG+WMnbs}`>ZO!|L@UJ-f9##z44I;TlZ$?IW@m1%*tw=_{-)sUw?_YP2 zfyK(2Nei)EoU&N(p1Zz#5dU$Qy+bpL@DV5Hhi!XO8q!yQGwSWN+QXCftWdXYXGwXH zIX$;^wZ&9nHHR(c;({~85V5opP|A8j8Oj6TzZ{4F;c}egy?b%{gMqR)vCov0zh5Y@ z)f@KSs-#i6p(vFSC%s5-!Ko4J!k#`5ERjyj$CwJh2xgcBX5(0peCcMXAHX@k>X;I8 z${z~zQ4eK@>$j$cu%2>7h_CM4ZfXK`Yk>kXxFe0!R@6`ddxflp#0PEzvJxcANrJnNd5n#US1$q9p~!X(}) z>PO2w z#9As>BT!DZ{`ppeoHe0~F{WvYaPm!H5U2a-J%I5AQkAI$-FJKiPkOFvrQx3%d(Fyi z$F?hpNl0)(h;k^nCr(zB2Zv&w6@54(a{bH)SiO7qKiwrIedD4MoD{I2@IgaLL5kk3 zjxZfz7G@0BA8F$grFJ^aB68YaqIvf6IZ=q?Nn>!=@}D8$phR{Nr5>}>EU8t2RO)nv z?0Ed)^|BTTCU0-Rn2BC4Yx3a4q>xD#e0nreTHgSNE7oka@b7F0BON$e{(PC3c1G>d zcD)k+Z1Wo`#P4O_+;pF$i_6p0)R}P$r=zV)wV9ReA^!JTzWko+1>czinHlF5V z)DgUAg4$9pWm)NkwOMvq{aUu$O&~~K+vFp_J+YoEhet*}vTJ{q$%SfgCbzCb_^pgO zIoffGNvPeu|Irboe%JQPb0V*idm^>$$HzjMkTwHI!2L|qlcnhM7HUNlz z+Hi_!%jlT6AMrb!WCs^rqQZq&_yx+FS5u0gX-V3Q32|FqP|Cr-Q6OiR0BtbduwA+; zjf;;zA3pWqjG2u!_t&}u@xzC$RVfZ|AJyzn9fFx`p=p(4uRT2Q!@^kn{cjEo4XqB* zJa6oS#~m|c%|{&`+Xxf|^Yh8Q{7V^~?lsyT(FpZc|Ke0r7Yq)scj8?CBeyOP4Q2sX zgU9D=ud4X=Yb&1X)j8M>w1AH6u76Xi(fS=qZsIS;86BR<0{#p!s$odhD$}h@+1r1m z2ii8+Qg$zas!_4Z`X*QIwRC6RNW0V4bdlG#9skIqP>J;A?~cxh1jKMjSJmeE@xj6A zK}hiw)r*wQNAfDcgKzmK&9P7|xT-?6NGmWq4~~xPFAPL@9K%ewDOP%*Z5tmk>0tq2 z_DG~FJWRv9iFJ~%jeYA@&b5OY%Y&}QMki-$QRf*iyoQU@rIs4T905uvkt+itAAxQgN=>vn))4jz@%?Yaa60{8r zUeMCMTyv;@)$DtPNgfT8Y6Xm5dQ)oaA1eRdnlIBLMoAF#;N0h(OK6CSWwjS=XsDv z3=Gk&AuBm`-y8)9B+{EMXA5ev;T=rMl|!55Cgq(j8LO97<>k$ckx@}F8N-h=Kj|BU zUXnqQdwPrDo?o;@&+lS42z&t`V0otUvmn{t`Zrwa*;@E&y$A_-VST?Un-9Mc6y}Sw ziG-7j5Z|r%802~U@E@1st1>gJB4u#SfCxkeqz0Xa(UixC(yhSkm z!^J2RJw;}pRRZo_pHWUhP3?`24upSVy*q_N_g>%If2yBto%^-lj9Hj#fwH8|HyT0& z;48j@WX;#$9Y-Is!l{4euiF*vGVQi+0q^4z1=>crqijaj$&q^pXaTgncRjm}@D2%j zUi$}~bx+?H4<+#9(yM(JOVAsPOLY4K_E1Bm*DcT*L0(G+4s216<@``K%4@Ic3SU2T z14-D#d^2rW)2myjU%zzX8B4wqGR3r|^-ZoQH_er z72eM0W>1-)8;=siNF1Kt`8tvxzgNj8l9rZD1nR1e4-JSpC#NDvVDus)fu>6KO{oM6 z3mq01Cmj)^dl&Fokcjk(Rd)Be6qTkLvDHN|osCvrZ6d+WK-r$)#K`0((E8X4q@uAj zK*#k$Hlg&Dy_vI`3)n@9Mzbk$*zNKwF>%lhCFG`=nWfTaafRkunk>3jC1r(ImYQ7E zOM@MP`8V}IS9ZOr?e!Z4b5DiAMD##<-J(T#nFBiIJvc?ws2iX0DZ5yY#lZT(hQXeg z_ciUryXM`404eF<%);+b(C+wA9~0b|*d`?OaQxgX*|)PZd4G8L`Ggpw+y=^B11qW3(=ogfxf?F+?L)ao9k%Fw4r4!;G9WiZf6;!54Ll!oO zd3noA{)|?0^HjzpQA9MrIrSS!q7LP%FE8;Pda?=ywBDx_bd9IEy2MOBk@*j-D&zrO z_3$vhW%OoGOzz^dfY0tWW;V~#>P2D%AntP;%N8EyOxs$e!LMHVRhuG1X)}M6t8GU< z3gz?m7UkqpWZmrsW$J5obxqLF`k(8|b~uJdy!*{tr$bXyMOZQrAkR6Sot>TbptP0R zx$ctaHO;6mw#)x$wyIQ8ntBWBxU2EI@d|qyDoyFpW{{@Q!kj>A_|Koj@a=Ln7IWqE zBS$iLvXttZu$nqD&)FuAFo{~#o4P)zpiL=Mjz%9~>REU38waL+2G1}iJm zCr{aZ#(2Gc5YASXIqpq~)&{n6M=mcs3mToA{9a!{NF&@Ft-e~~)svbyz2vsNJ>yZL z&qfiZysd=w6AYn&5nb3`sr|0rv|({xUQoTWyR`kt7?W?J)A6yworTWR6!KQUyQvGFEt;K&G&si<*GJZU@j+IAUzM?85zq4Jz}=b z`&hDh5T7Rk0(#@I>ed7VER=#UHZEbbW)7-XUYs+V(eg|Po7oIp`11D~6yLu*PGAO&!2Cw>b;|FIb`J(hz3Km*B5sh3U@I|a+k&<$9Dt5?Sqhy+s~FezM0w>bH0A# zpEo5**r`Hh>fSJi zacaUKR6a9>_~Y&E?KxfDMuy!zCc{hCAEY%wU0LZ%70Ggo4(pQAA2Y`0fc98S){=w? z;ux0l53jZ>N%t%gavmNX?*mkyhmn8Q=Bp)qh};@+o}CIGTGu%n-p$Bl&^p)!qix|N zh~=j{KDTM}!%5IBH5^r--PBeoLSAuC&B5WOu;(SfUL$*^SbkS&-hq|Jq8A@#W|h8L zZDTqsDypFgIOm!ANO`BHmm636d__BJ%}gU=tc?>Fch%Hl@zcJLl(3$zYRl@c#xb?v z2wi7adVFTDntSWCH7g${7&f}@uqWDg9kwxv1x9VR7&Es7sH<%&y!cpkYWXE|1}ndd zg{D>A)(+jWWI7cp2TaPy@ZE%bRa506X1}%Y3O}oa9Y&KZ1yb5Cu=`T}gFXDoSiHO> zI0s|sah~|UmrYHJg-@2Ee#*y1>9_ZNM$|@2IvG8OsEf>|vjSTE-o58b4<}Sj=L?nh zju2BW=qfaq_-;Zgv>t>XXLpwa1d!la{(W+4=?|~ThNg=M^BmVw6l zDd!t{SYcruSE+z^@T=x-97C9pDlGs#-N3qK3KoSEVrG>Lqy`tALrWRBfL5wZYAhL7 z78nZ5aJ&_!y6=7WZWjl1;#Z8O~j$L;e_K?&cfA^hRdh1qyrhi2$V&oP|acwLt41 zfWIB5shNK>xP(h7*a2HX7%BwwY-xF2VzDa(36&6K2&2J?b~#Q1AdD;w>+ZC0Fhu+_ zCcno4e%#1mtlQzQQZ>tqxU$ETq7Yj82z!*Y%9Al*!U1JAyH6I8tJIdOE$VVjRa@!G zXi23B2=wm*CIcq1k6t^Cim7&i++QwwN6h#W(SQPGOwyd2aq=WztN!9iG4%3pb~etm z+i;-G-v1hBr!9fWp|HTsTS2^ql(e*uAjiTw+4pNl{=Ei>z>HYrFQd|Yl1>ac7%SP4 zu>d8eZ~oCJ;o;_Dv@~d@hU-PT5_+;TGgBi(PW8qw!ICdiquR9{5FwXaJ$F^g{xdi; z^UmFvZ#z|*T{iz#&@0^fu)dyo&?Wy=WBEfGpF8E=&>)1eqYr>Dbe~{Ia@#9$a2fd1z z99&uQ^A1{fwzn}!qfhO#l@wI-v%m(o=uM^4{`C*D8f9uMd~AuW02kYxtda8&+ZXCQ z`nD)VfKNn3&&U*z7A;LkDb$qb9-jl{WF;*tcN`qEehf$`V8|%&EC5d<4)Wfq2WkvrqQP2&edyavMTU2KCw(km zUdHte3pn%p{rMRg0oK*Y;8b(`!~H7Vwwuv+^*!C*IyyP+2Z`1%lxo?f4T3u*%5{Vn zPi+2+@t(l z%lCfl6VPWrYK(Ckymg;cvFWHnr5zsGpE&XYleg)vD5Q7|?bIh9dqfYovGOMU(P zGZ&S&IH@zjA3ltL#iimYSEF-!3o+D?6Ob3JrcRYx9m4ygS4sK_+-X%5@J-UD zm%!O@*ew3pw4#j5x|chitxif427|TGXpIL0;}rst(BJPHt^ECa)5h_Y7r0%?Ht1YW zYYT5+Vh;aEZYqiat6(ucpcDe^N!{^)tOk?(SF<=&(0Jpi5Z;{DE2XZ;#_X18QJ0 zxlNvSVz?PD#zKIq=V5A#p}n~?1owzY;H_Qdf^9em46n8*xyclbX)6|`6q{MxA0=5- zUt8a9?$c>mw%cnLZ75)!D?W!*rB^|=w^M5?cJ&V*_xF!?He8Q&nQ2O=$~C|9b|nJN z609G}53ABAP*#&o;F(V|peJ0aCY_r{n=?+E_pST;Gc?!*5ZE_f5=}Q~T&krHpe5fC zv2eNUWx&)+?yBDT4|hfDU&Z~bV;88cE<>A#1{sWSzTms$JzN22Mi7@sHe?YRODn9*LK+x@EY#DV5nw%1@B<1 z@#jKtUf^T>694?=D~{56rVA*DJdb$n&u%(aH?i;39JA6S#wS6|SRY^(6Nly?h?AFFWGZrZsL8H(sPtKH>hj9^ z7*~~iFMrX}?m^M6mnOZ7;l%dU)}yen0$@*wj^`vc*}a)nxV)9H@Kz$UYGuO0iJ5mZ zavA0m!q_=IJ-DD0^Rti1k?5Gl)jj(zzZPOej*y1GxmP|eL_ycK^_wRGNF8a}MKq-& zi{%l%9~wv)u{T}_q!I!md;~RCQ2ow|_upJe+wyT_bQV|4!{Xp)1S5T$?U4ptPmn&qZa@R`^$4;w zkx$Og1=CNu2*&CM6fJ4#d_cI8)Rw%q&{|5IWg+?eIriH~JzGScXZ-X0&A_c=2Q3Z& zGsc;f&w6H;qMZPs&Nb|@iHL9_QW~`!=qUvp)vhjA*pa8}FyptoK!Z84U6)A~fUzA> zV>R}@AXeTv=y2UHkpIcQ1%hQ~&zV#7sWZ`)6AqD2^Q8}5OV zJ|G5wE1R{8HjCtH-OS-%?7}9(n&kgtx##h-CcXg9=Z{0t@DK7|dIuU-fgkcEEbMXP zoVKR(WErl|ewXiABJdBsDf;wyFaHyZ{tQt4X6|idi@GLedtb2s?YnDuoZa5k#^JrT z)O^w||J>A9tss1NiPIx(k%xyM{lMZsT`D>dwzdM(%Rc~J9_Z=eVI=^}6(wIOHXdS0 zE2Ro(X7FYmTd2?VcL$E~o*&?-vU_7eK)+O0!}xA$o3LIer|jmAf%WhR&n~=SqOtz6 z-NUQC!$3K?Z#BHD>c^^xVsWv;s^_fG;u6*kjqADVvpX@2EBLC6^v}Yzh4*8Vgh{BW z^F&tGdA56iyfQH%3-1v6Qw%T!57$aE&{%{mvId%7F-{<^pF+MML*uu)?*RvAaN-(H zo*6~lzI3qeDiX7{-WCrq;2BwTI`&U}`GJnCkKOGEzB5*9DR?x{KP)Y$OkqAY-E@5(A&RNPzGk|)ws^=Vy7Hx+ zW@n-eG5W{*;us_nyli<=Vj7ZV5)7vou?LM`mbgZ6%7@kua^3k^8B@pCY?9O_N5)sC zK)*~wBg&czoU~c;Jj(fhJR^RZlEp*=>7rimhgvq*$9IhatHKYk?DdvPf-tmYWj3$1j+q~)CQA7!Qk^6Yk&sd7u3`+7w0vW>k^fr&`AX^je{(#piaa-mT zIayP(Ilf%q+gK*BLF+R2?_-H+zQTiG6Q*Vq)_(!$raUj|uckOKJbYWJ&9*Kwl+3ng z|8;8uTQaKj0;u$uV*2`c54!pX4XGEtV@8>yaCYX^;v~{v%FTVPSnZ97WK6=tK1#=uhhf2^FNg}%Un)!72Bi?Z4Y(n22nbek8Ao$3b+cr9mazfP z6BV^EZb1l=(a0ki(8I}#p7y^vWUEor(@$q5^WW_^;WjYN=CCwF0$!+nUgX^+10GO{ zqD#yG_se&Rw6ittjjuV*U54GS%Xarxe_@ z$A?L+gP#KUCklVuonzy{77It=(h+G#SxgY=pC|GBCZG(1BxN#S7B^?>=JpuPTgo^l zenaDIt>Jywf`3MPI6XA{>rO5e89C5&|5c#ve*G+G z5H5dC{=wX?N-b{T`%9qLMLJisWV0H@A%<0Q5XBRJ)WgU>LLk%_QmQrLH-sS@2?CA0iq)3l9C}QpRPs zKYN{*w*e%cU&|D3lptpU9d?FsDl;P^Iyzc9I%cuIrqsgb6VutL=oOW>%L0Ln(cF~? z+FCNO4NQQz<|&?BR9VRHp#FB==?VX24&p%E0g$=v8QOM0VdOr>pON?1IfW zsONO_Z3>^pS^~#J0gjBED9y^*?5)wg%Q$_AtE70hk!pRWRNho1E;Z@~lFz4*+AzMc z9#ax3VkzWS7hbYwcDTr8A9yzf>bZ2U7sm8GEiK+1rIprugME!)qm_k^@$##%pQA9d->#opIrUt@E#5e7!G=!k7M(z|zC^dkzfoPTEKBNBi&2&ma1@VUm8{grq^ z9{1MdE}qC`FiLAtL7(={+E~h*onn$GC0{9pOimfTC^eR4VBxtF(we7C*x4(SSz97U zK)CMU#yJ#I?YyiAG#8U{X&Z^vxJwf12fv2oMOrqkz0kW^=nY5a2@il7uEcBmYu8>% z7(GRTdDVD4@mP4XWbe=YAxwx#IYo-xT!Mp*4n*{NLOZdVyKe6C~an zE4e(i{JW31Uj)N1m(oQ<+e-@@8eVGl>Gbc1z(ul*riHHC`9M^`DvAbfk+*M&v-fY? zR*U72><*I<0n36&XJ7b#=LBMLe7NZ!2Z7uI^T#8?FK5)9o!Kn6Q#0rozm1(6F&rKz zuOZi;%+8c7=$-$;U$J=s>{AliLz-h4FK}Ce&9z?XtINrbi zz+sS@6U_eE&kt|{pArB(2E03%E@+b$Brk7sVsBv8!Reeg`8NK}D*S3!0scV>*l-ZZ zPVW_!mttslLmP;uGYb=Y9&9{J`hr75|M;=RxZ}n~5CWH)@m?=%X1n<7ufpHizi^8s=q-L6|axY#gbiH<(= zqs5s{!dbOM{zz0*{O_}J=kFt(7v{A7X}Q&57xtNyB5qjuBW-2RssRhpF(UEha4{j1a@+~lLZ3uA%P_0}VRG@(O`pd6T5SS|HbR%-cj zfiiImd^oqzbbkBTNvCvTWD~FlnLpA3duWjLdaIB8$)+d?vAwUl5L==yl=&l0c?gX6tN!jpjkO{y&)mQ)i}h|bhBzQ=pdDjoMZ9pg!j%(2ynssBd{3( z`B=$Zb;k#YoE}R@tO}Fb^T9lI&Q2mXM>53Q+k#i{R>1wastC-W zSo49j0KUAJWIn))W0l0GKOC-!!{HQq05f<2*5U9d-<>-Tel5PSXWhSp9=a}D`N2uh zY9p)g9iafojjz3hB&knIuCIZG(fSr8wLy`DQDr-7;0Og?`^s6Nhd!mJ^yyAp;=$&? zTdKS8`JqbHJl5h;I-4FI2L+mCP0Y$t2Kz8NcU47P6?p+&b~#t&>D@pM5lz3T2P%V^ zgSI5-D{*e`rP3xrqwvKnEt_2{`JmmvYmM<%{uyn3GnikTi9q0NMmJr+zREUbdz-ec z5QG9^kd+OwSv08c+v~N;RB#se=r$BKqHP%m!+%$PofCy?1OY~s=jRDN0HueLbEdHr zCf0kAKy-EM>okC_LlO1>jO2`!WD(7k;lx<>v`=EU^3`UgcdJFQ9`}Y+Xaw4{NK8)d zb*mS2!90$Rhc}3YWT4^f&hVT<;D9e*NMhq3uDjWC3p{E4c#^B?Q)B6pDTQD07T_0xKJQ1fv}#gN&|z|e5uT<^t3uXZyNP{^ z&{d}Blh)T%P2ts7Zlp{oVVKyA zDc={@Xb>6;_Hw@6z$btD3uNtmD;g3K3?$JL(bEAp;IhZPeOqT@@YEoSskGyENaugn z<8l?_?u%sY+hQ&!tp#lURU%-RGm83J+>@O#cADE!#yyvn`$Y{dDY(^8psiG(&Z6@+ z&J+Z_zye_h`RUs?v8z*K@4f=QJ_bgnI6zZWU(YUkpuWceoLr+I=mbzkg2WAwig4|+ zx1Ajv&YAg=S^iOJ1ZFbl=(-cLhF_LXY^fAfbUBoPYW7=6usYk?q+qGHm-<*>_4-?M zA~Bwiay~scEA7(Tqj0dlWa)0_9bDQ%*@S&qM%jdSKIWy4L0a*tCvT-q1~ICzBZj2bWxJ4>&7Y zMreJ>Y+LCV+c9l;8(NG{0TWmi49#fHyu<`XI9~)bb-<)bMiL;7kbgHJ1P*Jm*?W(_ zbtclnZwF2rZi=5OKOCISyvBDApkG1Ri_<3+52XUrV|S%Go{q6qd-T6%zp3RW1ni|X z`>mgw`vn5@6|I$Yu)cY~1-4AhdP+im_ejy~asc;PmC)aW2#m-OfWgzfc79G#dARP7 zLH3R0mBtFJO^lk&lhF&KYO_cbcxyZ^I!Hw@$bp3_K0*ZZ5exn9S6Bq`VkZ{`ff;Tw z72+trOCI(dnEn=4jp>-U-e}g_A7g0ERkbp%MW;enQQGlThk|Ll4q=LhV<1b39kl}8yZ|l%JYugl@;-6Y^Hh>0&G#kqdXkuD|A3@DxRQB5z=ggmeSL;NZz5m_ZEB+ zYA>PRFWfi7*#qTg3{$ke~-2!z=8YK5`aD*w1~(+Wu+Joqe; zUfV<8BAMUiC_ohEe`Vo5Cs6;QU^^T73c5gU?@DN-0xZgRGvMAA(n$K z&b{cF5dqietFoXcsRV#c$@n%H6#qc1{Ut_fy62o1U*R!eU>J^2NJ0Od}74lomC4epJH8t*1a$nnZB3oDi-ceJVELJH1Z6;9G0)=Mh ztkV$L5hqn^x|Nkv+rz`WAmuP&kkQtsgTnjRpI9T%JsQ3JM){%Cm*TaD9WWI~T9zH>Ul|mMp zN^H^o!YPJ3H`$iaoN?&jac=fz#*oGV&01ONQPxne0YwCtYSO4|8+5oh#x19KrfNJg zkxKnRh{{bIe=!^>$wB_QMoP&C;meK1_#PskQ0F3Dr@}Rt8?Mw4$Tx9HSu6+5j zvcC^}Zr^G7`vCGG_|ZSFeG|XQ@y}ED9|vRneUwA|694Z5u>c&se_nh3ni%u%qyJxg z)o{2Y`j;l;H8ezO*Vi&iMmT0vRPgSv&O{o%Bds;OK1!!rMprSv z)=Uf%@w_JcNC<&w;Gu`A@%}cWycKg0`%solZ|t)eE_);W8aZdsBC?4;838m!F@%e^_*>8}962 z1WtlVhZU`ERntk?=D0waJgo)Yb!Wy#OE4hc7W3L`z*zZo=`Ob5KB*rW9j#9%W{^5N z!+_0xb$DEZ-5X9MsY4C`O#dKy$*=w%GQ9hhnI4UnMPv~I%Jch2%!h?f~QO>Yq z(N(otZ?`BFbbSy` z!T~17- zpBXlDC3jj13$G52I5J%C!sRnB$G_+bI;B;|S)V0GL6XKH}_IIW} zE>Nv|Y;SGSiMbp&3n>^<>#I)si(&sc8OE+kn&W4gLE>tw3Ft_^Rdogr2BdLB;_r%zQfAgp0aGf$FDCQd3X!15 zR%*?Dc$}0X%=&aN;Pkh!dEnthIfAEXTR4%Z?xUKq{JeIwG&VS zXM42{Z%B^GmNPzPa^Ke(%`UbzLpE4jg46`ADzm1B;drjnsRiY6UCgNT{63fKUdzQP z&9zrvUN5#5aJk6le(3C)Ix+xL1f7|77)JCp!JKW3TO5hMl!_(~8ww0ymr~Rg71^KV z6O$*BAvUV05n3=rvBv*sBC>0$o;>vb8KasqgIU8*^lolrqlF^3i{W z8tk?C{j2AZ2rGFl(X$^a%vug04^}Bp+yTjmUE0!em9311`N4j7DO;FHgNfvnXq^+)kzU2EJ6$;pUS z_v@?IZf`sYaL~#Y3;Pwk}I@V(9Q+$lkJk3{pxn*N;nVGQ2%>)kGj*!mu%kp z^wUDPQ0|2XYQ~w)`py-dlv4T(I~ex-zZEonSSX#tGH6l>4EN?}A;qTM>|g}^1a613 zpH1Y=LBc3km2vujH63bd+zYRL{_!aV(^YBPTi|hW;WGyvYJa4hp2yIm2x~hEuK5;jj;Q+ z{rl$;m7x0G^uGFYmD4quoFb50g*<*x=G_Zvz1UiZ8#lMfl2Y04_w0A|C3_d!zdqkH z#M6TBn|Pg^cpICTnm)TT;2jo&#TXOpF3 z7z}K5*g-Cjq4V^-Jncq#THq^#3qff*q07No3Q&p@oGZ4x3{g83Q&t_rn{zk{UtSq{Yz+2lM9 zTp1qGWJw$HNN!vvWLC!Esnq z6Ub)CDNwE~?XtOv(p9^i{$d%J5N!|FL#6#QTWj!NZT_w)juQ92+1kC0dWB_ny4qmP*Q{$gLLH*&8F#dSXaF9>6{IsgCw literal 0 HcmV?d00001 diff --git a/fig/homepage/qp$.png b/fig/homepage/qp$.png new file mode 100644 index 0000000000000000000000000000000000000000..4a1edd281d4a513fc71dfd3b00e060737e6b9516 GIT binary patch literal 73754 zcmeFZRali#_bt8^1Vj)}N?IiZNu@g!6a;B$q`SKX6c7Xyq+6t61r_2*ed}1maRO z76$yLcKnzDfuKP=eI%yrnz%OU;z;=W59ZdUHQBPzYvl<1CrdQiqUQG$KQrFbuhFR` ze)RjC*;S!1=+hnNX4V&v&F*9qin-&x6kmGH*)QQfJvll}oj~dAY#cdB=(l~s7w8Z> zIvVHVV!~jO>@~Hhu)Oas6!gDdW9&{Ac(4A~kJ%X<_|lL5`?8`!OZ)Ht9}O=2UwbRr z%K~pk@-Eh8MJ1(XBM}i1S6A24mDvCN)Mm==wK~&d{*@n;U%Z%q%aDv8W6~A>hQpwR zv!f1=lK=Jkc!fb{EC(sCEzR2cxDG{ic{wq|^W;Z%b{rTuqzD_E&0(q=3MG6l8x-YM z;~3AMKcDc}tnDi9v8g-8Xl`x}3JH<0w6rW)i!7Ms%CYJ*FXMIJyXU+%;tTh9`Sbnl zd?DI9sMnusW*#3Z$v>**HiwIx2Yva%l&4XKo%^D&r9VSvT)Es#L4hFI=iCj;4}lo@ zs)x2wy~>J1E&Q!4P;y_b-13&Rj11i>mB$)~<5F*$dbpCdcBoNDbXs}2s+Ng~N#Mo# zVLi9ifQF9SZz8QK`^Q>g@L=-t@{tE|QBiOFG4U=Vx)b^QXJ%e9>wK$hGobSyuBOk| zs=Befyd2?l@dsItf8$2~r&g=g_V)Iz35OoB^gaY)I8WXG5*kKn8#U2e)TCw`m9|ex zsH@+SOA!t|J@q6cBoxh8ZJM5@K0iNS>PyEQE3-h7HJK|2xNJO9QHiQtTU{N`^h-s2 zwHoKJ@9y8x+4)FTHqmU* zC6xA1`w^qJU?|)R#tv89Ou*e(|KcK0}BgF)YzC_-lcY=$N;k? zoO*gp;x1~sKh+1Vl3{hVwKWFdlD~id{-qsxnMUaO*@?lQz2&W)ooZ`?0aMPAdLNO4 zjmi7!_1%-$c z@Sq^9&J+=9R#sN-^^2qJ`8j1_#MZ%qb5p*&ddAGhN z3wg*ZD;w?~2n!2y8@(nbCcZ;Q_qcc?Gr_vp?41uCYMh4~lbRl+k67;9yN3nAI2V4~ zk7fn}k+@`)`E7W3%qjb?U%y^&ZEYD-vu(#4TD*M7!&A}R(y#UBU_FG(VmPQHCnu-U zeV@tG%S+Qq>+d5u&RcAYF8$eGr5pGj`mMHhd#efWE;BQZdo2|So6Zj@YA;%gdkV_R zsLa)2zNlU2sa`z&^XC&@Uf$H~Y^=rZq^9N&lG&xDAh{&|G^H##;?=YyUOVw}R(ME= zn%7xb46NGKucy6FaLGb3rlzLUDs6ACt*zyGpDpam*Vfh&;N#10I>2gd&$oU^OnkSq zWB)oJ;1hgRP;juHWP0Q8j+kbvA#<)z>FLeC{roO#)w-(`i%?TjADx`Y3>z65rWO|~ z7PIp6YbauNp9+P+r)q13_S}V7wJKj79l0v2s@{PS&9{X5tL8qow=XtKDJUq|cb0}n zY=DjPJ~9$5R1!^BR~J@0{Y9Y;?(pXU^Rg#To`^a-bMNfz%&e`wgZPGb@3OP^uDBO0 z%sboKiW(ZyB4*m7?(_Pb3qnDdU0h`FzCO&kcLV*>)vH&ZJbl_Yl>1_KVF3dQ#oqcj zat_zS%K%oOF-IxueQYe2fPlc>#w2;I`+o5X*QbKa-*FF3-lwF5zj}puBq{mH0$E_| z=vZ3yK}J#%BO)RK-ok^&gfD}IOO*6DuO>ao>e zp{~eNj4>FT<@MK8BHmiXmdmXlCML89Nk~FEDES@n6Y?P1X9shXD(!z`-?@92fSf#} zp+U55*TUS~_RpSKBEJ*%`dFF7qetk~POBI%ULY}A)s&R{VM%H3-@o?e%^O<>2MJM8 z1jLr+Cg*a0zoMauNk~U>a`Idy6c4-8BRj2X$E)4l-K2c>w0{2nbUlRy1ud{+-p9qg zEiQiG>gINvogM$|pZew& z+=x9u?wnrbuoztS6l&7D+Y$jUVHRWg95TCj=15qz-ZS%jjB|vP?YYy&5J6Af8b*nR z*jpRD3DeL@qU(8EI8!xu8!}EvM~(CP_b^HU+7k+HOVWkzr2DL^`|ix@#gClVi0-Qf zrM&I;_-5L7$CQ)o+sQ^vM9l8t7_vgf%oYW!PyOU)WB;H4}m~G`lLqn=p z#6yCEx6V&CLz1Fpub`u+R##UydXd78ohj)NE{Ww@fON&@u<&wgrcrgXqzHoitKGb$ zp`oGL1pS>mm+^%5-ZR@GV|T~1A>1QGv)xj=$YNNvBbYV$pHF-4%06Wbdm2fv*PANl zda_=jpr&@OfJes!HSt-=?$`(znaGosI5Iu~f%)l+^9p8!87MxMTu?k zTo%1$7B41zF3y6vWFhh2ui9|gn7H*^_?Wit&&vJLY5%P)TRRFrnwOK+P9tR&F>`kH z7v~-mcCD1fW_|lb2l2{^ihi5*7gRpy$60H~k`e(pjZh@=vG3gH17LQ3G>tR1 zP!^ zlbxfIt2zq>{zE>}ysX$MBO@cih>eX+@My+A)STFEd+u?Yy;oyNB*Sz4=3t`Wq_9gg z92^Qa(5;v=4twIQU@f*<$ZdXFJBJpw#h+b9!5(534a4nBDyPY0$mUkf#aB)a)|+1F zdKqndiEGcyz-<43LPAav-d}mKn{sgvLrftir_^y-vH973#L>aH&GtT07nxw>_Uk8s z<;d5x#v&}gSAc5T{VpT&R#cNA_rVrW;K0I0xVgCnPb<&`UZ?yH1#%^cUlVcFC@qUue}6Z~-NJNUB*l;e3D> zPswYG2q9sMMi-OSh9slwwNDhyri-fnVJ@S^M#0Fsl@;@EJMH&J>byK^xsrMSy*>^h zW~{qd=(!Jd0%kJjC_Q$Y0!l|qZed+8Jhz5m0S%T~tveS?7IRofA{X*A>PrsBEMWet z00isLet*M_4B{>(RFj;nIvm!tVGOItL3TtBYanaBer0bfJ`NB2hYX}OIPuLH6AJ3c z;-ZVan6G457P;*jfBg6nz^B}Oo5u z=YD>$K2GE>vGB}dHqQJW{uksbD*22Hu4_LvH~%8r*adJ+Z@k<}R=D5>)T^W8WA)lv zsL7eK31k!9W4q%`O=#DzUypEFxRt!mQ6HT%WS(n5PE16^HR8DO^TXX@>X)5EgM&Y- zbI=-~RDOtxG8$4dH#h(4d2Fx#epCy>wt3Y|ULIdQ#?JM3J5g@f$!s7sw*_T~)p)r& zrRdspeZ4-hrNa{$nI_l^w5##%wU%c2kisuCH53$lgM+biTg3lPp^q*L8J0?W1orjo z1r-MdFhs(KZ24W4_E4%F9XS(|l8C5zda&2~(gQ8YLnZxtL@p>~qM3_g_f2p!ILK{I zyR1gOig7f|LQ;y2C3SUo=XR%HX+GkUeDp}|EOCDodEZ#M`Nxl!W2I)rsY*IJRM#nZ zX;@Q)y)+JlQ7HxjycPC-U)FPKyECD~34fT|@xAy?H+T0tOia?-i7d>_KiVQ0ntwy} zYko!#dH5Zr05Rb5sC6mE*fIQhVKQvd*49?dPV~^QuyS55jW(DwB4ps}*Qv1OgExDi zxv*^iSlXP(=Rn&5SSlzqwCpH(ZLCa&1FcwiKWTcQjMvI7X<;-G8l=!Kx#m>w%A(wC z9|#NUaIB)50PPxQm2a#sR8;h!EOAq7LO0dZg9n9?*KLOu5FFijrq{_q>;ofg*vSm) zY*^yyVPjVE&cn3dmiANP% z=C%FWIXH+GLdKIPLLWGj?6Khs2$9!|kAXp|xGPN}&=#rf^0&2V2DE4gb#(On!Gi}| z-2$65f`Sx>(?0ccp)b5TOOyu9%k*)1#qjZ|so6BC@$p|RCVO0m?xc5h|LE*+IzN78 zvLAp=0Mw7$4<6ira`m;MVuQ0Eny`AGddo8a5q*JW^C>ScuhG3sdb3O;zva{8kRJ=o zCCT)wOf{ojXDfZMoUcaGUbxSW-LlnUcRJ(@CX2q_bh>1 zR*{?oU>aNfY4IB0l`B_Ny#n+3>V<@a0OogebmUXj9GADya}GmNaD{r&SL@+oXk?_f z+@BHr8&@)n;w{`sJa-7PPMxT}{-TS7pHF^havh=ug)g@M96@&OD5rdAjAPstPschwIywvKV9p-)er;><8^u&RX!hq8 z?N{?F=3v=S4ORl7=XD&MA4<=K=)93H@-^s=KVUlfJS@{-SzBLE@T}z=7V|pWM;hV2 zM}1{(+}W5#;Uxr>`_k=Ejuej#K6RRJWrZ&0Wn`#?EQ72 zCt!BSLM4ag+xO_FMt-yp0`mQfg|y!{egBS6LW0ajnh+8Y+=ePi@`F^^v(i#vprBb= zDKtwi2|HBsZe1PF6QkV2=uXx(>)b#ZNhFF}b?s?o%$!4?OGo)7Jw2TqXXBo%g|m|r zk$-A-LOH+fjOeJd^riEK2QC|0f@gaJb5&|uTJN9<3Tf@^>9Mg*8BU&FKxSv6i?bLN zYVM2}guk8skeW)x4 zPJk}9t@ZE_HpR=Mtyx<;J74Jxzk~#GSX#maS=bb+r`<3C?eR|mcXG+JVX9STe%tzf zZKT-fT6;%_o{dfT)tmI3d^Mf`nIjN0?`}OLta!-1Wf8V&BvVaL;Ih!DfFb^m0J_Z* zPq4DOdOugiM3#j+W9_+&(!t6AbmGnX5Y<3U`Bz%P_c0{9UWT6Q2>2A{sb^jQ2B8c7 z`bCL{(7K)3Xp7Nk92U6wPe#7g#za+Ud`n36;iML1a4MhEZTFOzm>5hk{8ZIk0tyNo zh+g%QI6(EZbaaga12-e-Wxv0_Es=M} zVOrL(#+6yLg)p`n!(;ZemY#=(ne93yKdm~lzf05J-hK;q@@)=|qNxRSge`DR7>6e7 z$5vEWWpoGw9i@SI>cD4lx>6sDS364{MwRz1JI-COc*!lKl*Gi4T)upnRs_-TolpPE z^AzD=E-Mzw!#I=V92XzN`{d*)X5>#{uRlMfFAok5zJxw^q|)v#?8HD#o7y3L^~e@` zC#N@%k=Lp}BqxXU_VyZA>6CvapLJhcTVrlpD*KIj6@@MD{P8gF6Ucz}bCh3ZGEA1s zS-O@_#<%s4^&bG+UG7yHdj=T|o1tnq35kW3EYx>LHTB7Uxu0&gvP}Jo z`S@|BX0xtow?|}pO^g>vo>6}2TE*~bu>6K5kzH{-TMtO^@n1T~H}wlgtq5{IT6Wmo z*(o+pf_;cS5g!7CB^14F$UJk9FmN%%^oWm2egM)qt+DJ_G@vpq%Pev80s?_-A~TKp z3bWUXkKJw=JymI&%8;TBe(E5qoP%p6I=6huUC067kguVsDy zjLuefr0-9eZ>7gr2?0i)L>D4yh_2Dp?|9gPP8a28Dt%Jq*j@g}cCPktstNjaLTZbg zf~m~Tp8FZE(&!e(xfl7y)S~MWq{TE~kVVt$nOZmFS3T09ON92uVD87;{p4J79_t@A z^=HG(t(=P0nWTkIT~4)@S9#i@%AS?{_IUCSA3bN`=Mv`RR4$jpN2d^Ul~?rX`>5D- zeK1l%nOf$Oj79OBmX_AmQhInhnL=toVf|1aDFfsHc$!gKYn}-m7KF_DXNEI z-Xu>khcIicwYj;i8_l7mg`umh9}g24momQb5hRkt%*+gsq*A)x?3taty?M05$l&qr zVzc|#OvzBPH;^uy;F|~r z^Rlro7@7;(KN28e>#YptkU}X!@#KJj7%>=KHm5@Wa;6;n?zQ^ES=%S0iO7!32Y5l# z5Ef%&8g2G6r`dn!?U9vC2!^=7pC3LI73;N6r>DC;OOrJf%rq#b8^ugcqtnvUOPlAY z-|SVcCg|+yiehG6%6b0KsqDw_^$Z zX+5f@+kP4qlgS}hG_R5N7IRuVmb~!7hWn;3Y`5B#$kI&PFmY<&W!$5;I`J!Q_B3IJ} zmcP31n>RE6gy|2tpyqeXUfYI_qNBQC)sY{X9BPll4XeSCyo|vKBN)RGq)*@K)Kkc$ z<1h0?GG4ufMzhMEu2h2scufEt8KkM4T{E&djmroMj=t<5 ze5Tob0O-CsJbOq#%KSXf6M6NDX#Ev!|1|ib72A{Md)0Qdw@2h#JX3o7pL%1~_f8%G zpzFS$pPwNQ3lR~KTAinxe@}Q=nCcpLo*~f7BUKJr`jR0ZmOq|GMBK>JsnyUDc3z8h zy=h`b4S7fP`GAYPeT35$2vkwHoDNm#0a0L#SJ=el6#{1n5LRt-mBS=wG^7a4-a;`) z8AP-u=>C}1Q-JvQxj2~)diU-%02nCJw7!`&8`UzW7l78Fb-+WQ+Iw#62^72C*w`p$ z$5eXkVZ|bb|J8g5rKq*fal?kcAxH*-9st_A*OY2`yezG$u52@EM%*8T)vYKmK+-!1XTOr+%5$L1?^}hG%CTZ+TUC` z7CPhnA-*-sLKSUSps49*mY9(&M+1IP89n;+=~JF}U4J<+rd?A}6RLK0c5^NrPw>&d z?GJ12C$quASSTL*!%*gawM9lasi>%^6&qfKxt>{E4E*%z4hOqCP%*$MepZ)eH7(OX zQBb1VnMFkhwx*`{U^)y(_W)1dzI{8CfB4Nck}b$P!zVgnq#WX>J`P%Jdbc$zAXA~} zG9Y+ABqfEyj(Cx$LP<(`uftVNTKfCV=Yp4jSO>LW<_(FC>Hey=yL+|aC{*RlXX^h6 zf@)S4RiH*kvaRQVTyg#NLCVrQIU}JHAPo&5E-jsU@7iW7mi!ccCv$(wSlKwtIiTB# zDiYZB{F)gL%MC*wOG@g*7;}Vj>3v5JQPN@4t$X+5$D=mcGf2$MmJPtyqPnh#MUehb zlsB*4+-v>iR}1mPg@&L~Mkt~~AQATYaFdJG&22i|o@T z7Fc+A6>1m#{m1=922$^+Me3fmh6V>0^Cs~_dj>fzVyzb>4GMt&Pg^-PDy)-Yub6)X zg1jjZ@4ikh5NzY0=_RulYaLx&VvNdoe!H7-LR|^*(R}dm;cYCxU#H|k?xl0(kRTxn z=JMVS*GeWWT9QW0EEQXm; z^2CZ4N{&mH3@u)+TqsW=8Tqdlz_%AlNAo%g9026tscL&8o$(z~{>o$v;A5aKtorT9 zWT476`dtCt?du=Dwzf6|LP|<%!f`-WRm29xjF0+v>4DDtx6kncrqyoR>rnlSk4#rn z#tov59Os?pfZo{TB<01S&xMIo!)D#dh7yjN2DYn_v9U(2tgCC~9H+HXmqJtDjor28 z7W9{$DF$^t!w-zw2}45HI-S-=2zD2HY@$0mmj|*my|f0o@+_`%aB!%m3;OC+w&x|aan3zCO6EC4_Betl9yoW>1jnfg$ z(v&LZ|El`sw#fN5-sz&qg&5>T7H^%jv^3X^szqjQv?59P4UzvZ$kwRGV25Tgq=%Bz z6ko@EIm6w5?b4-7uCORYj?4X$;eBM#$+Q9si=xU$4Im#3WIb=a#`cZIqmx&5SQ%hn zG^=*-hr3eWAwh1t3c2q!$gtJYdK)i*I*_fli{uRHinrRRJ=AI57=@36g!BVU2%3iR zc{+tLS6DEwQ<(00cRq}Ip2Ru&)XizZZ*ak;$t&t&P6X9{~F~|2o=t##K>oZ16C9ZjT2hPOm1`^c*EB#IHA(SOEtbnyod-cTKr&s-+?)v| z8?LT~0KMOSIW>A|-(BmQtAb*UWB=!;v4U>JvV~m}z>u&l4^okk1m1dRJb01_&=BPQ z)Vw?#1qFqz!$YXtw}AoUYjbWv=08gBqq%z*4PxX?eZ9!{@89DS?Z$4vT~LZ4lxki9 zfxEoCBVRH}K)Hq17zk5;xh#LU_Vnz`iWS#(Yeo#Esz*5W_4T17SSBVWl#~V^MM-J1 zv$Nb+-k|_H;{#8EypGOwOiawzuV4RcH&H8*H8aZ@Y5NA50vdLS62p5M_RIZr z-;Ng&=8FCO(WX2$g%avtfXD&~B*OmsHDYE_l*a|A^5Ed$_fS>7e-}rg&FSgs6|y#2 z1r3eh#6$|<#=#777i4CLPn+%47qYUr{R0CZ5)!U~&H>smFq?(yKD6y!T_E~CfX#wW zMrQ7CU~fr3@K*p&+ZrI@hDekJDhNSU5P3%*+^z zOG>t9{P8p59CcU~zSLtlPJ1D0&uqJ`Z1EQzg-KIcWORE>S z$2R`mf?8a=e@Zh*wB#Hss|++WsR~?%p*&;dj%(%7s<~HI9@u$60$4iZ<>Q-VPXWHv zt0c5}YKjURBBj+I;`mT4`0-?{m7t&?w9I_#Abk}Jf~!aJ$|7u&0GwNQs?On;^8wdh zyL)(4n8sa_flFs*XYD4#SAYE)T&kxYHV0lKQremG!2NCYrqz=tK|{DHsBD;>eJ?k` zS37()i3V$=)}z8OiN%QM?*uJv=Bo^#Wj#-^F0wX->380kcndlGdIFD)q^IY%JpWDI z#{hKc;_SJ&xOjoL$_;xPWEq)~!j?d6W6||eX+hQuWR>$`B=4uy-w-w+|Ki6$%I!;+ z3Rk7PNq~BtIhC5z4EFUA0Gj(YG)lkNlVY>Y0fq23E9-T*t_55fVB3hH-&D#_oP|c` zeI81S)5u}EM8NfZp(}x|0}TVm)X|s<0rAqY!X8iUA5_o>6fvl9bgMu!4t@H~{+|Um z+xPnbI03bHj5}ldW;pkt(=7*k0aMh7+8mJo6(j6FRN?#H5864{*@=OKRkD^4q>&87 z1~ZTHA`+E1W0=Wh9^ik_di?m!=Yrn~06M|3RCj{=( zr&rO%@X0xGs07s}roJ8f)t=#QD>4I!MM<^SsRV<-_XX7V#DW*Bf16H4eyHygxn(b_ z&_Ayo4Lp|80-+as4P7dlj*brakj!_4YrlO%Wl3=Iyd2C{$X5)HTq^wD)rEU;w(0XB zIywNHCI&MNzMU9<1;;xHr5Z@rzeo-B&!^Rlp{$mFvyWPVGS2@nb*M?o?lPeH_;l*|FajuR@ zy<-ypc`Fsq#C~o+G%N)^^q*Im+u!T{{iKl4Dthm~?-2a|f6HWg42{M-PvmBIT5&P9 zcJ-qMWNU8Ye*SMs^6;GYcBa!f+v^B+Lm=3x-wD%u6s<3%~p|8=@ynBE4vn^4@B`D!2$FUos=bi zdar|(mR|c&eVsOku(fHVF7Tv}tv6J(SeSha3mN?U$L?F3m=Kzm@&&FO4tme8V4^RXbeLe(?$%gPEy z1$l2FCjC3|fctS(Kkwf|M^NyeX3^rC=Zf6yktig=_nCVZc{xo*W$4JMgDR0P=5nnL zJ~CI?{BWQ5=vU{c;hKQ!<;RcdzE+e-$Yc8KZ6eY7lD_9pVg8MYzEChh5P&5y0BGpQ z=fdinUZrwbO$FLDVAu)*#lACSKfbJxLc@QBV0dkmg zV7h<52n?PSBHk1zkpgspmN3c?$m;j3Q$Bp~)%8BUvc0`6pF6ZPT`vOiUGTyB@23W6 znAgz}p4-hNOGCLamU-bq2c%)tB4IG9Jnb6du&^!?13g%$`yQAA+2$YpMKZ-6laQDo zL`|R2oO;zND5{3}qPcS~kJP$5O7_QvER744_nq9kckjE2ZoY(7zNFUT%uBo^IXto6 z+gw7|Z0ZkjsV&JX}x7E=0WRT*( zgG&hWGRlRnq_MRVTA^f7jr;5-DBpYeTz%M+nXtB3V!GF&5Lt9Ajv-FiGbSeeY!1$ zB0Rt``u@{46zS($sM$8&fE0bPyY z&Q_u^-@#y8I=jBlQZEVh<(iOy-A31E+`n^8Fj3g!YHg%JFC)|1n1lBbQIyJ%eC|5C z!ENr68)t8|2_U-thX(8ni4@JqS;hz##(LObj-9UR;` zJr)18NW#4u8)c}~Ciyf+FSuO8>V((Af(usd3gR8L0HMN%G+SruK!v*)cZ$<*XPlot z?ifhBM8mVmWN*Ji&UL;Zub}XDu=JTJr$%mhVNT9ta}9bBTCMmcTcRwB4&Htkt8-gT zAf4o&C3f|wo~J&Pm70jXZAv{8a^3g5a&cokRqZRNg<8??oAyXsIIHSbcV9h$o0b5 zo#d~?7NilDlthVY17Lg2q3Cwo6W~RI$6=?PaZ^8s?ioOLD@fbhz?J9&G)4JFP;D3V z<(CoI*RI(CjlazYwuvJU^nl+m1ib~!!>?cog9k=@PS$zsPCzuFVPnGs*NN+T84?tm zFz_lf>()^qY{BvN)HsI@oP^EfHs3BGP`(Ea&m-Fe-9j!9Ao5umDiEz<>Pw0Hfj7^Q~<<8+rxjNhRlwB*_vDsp}D58XnN-SAN! znz)yh?63-&oy=Tv%N44S;8uOc!U!W%6aD>#_yex!wUqLz=l2n97K*E@(*@uzhkUq; z@q}u;+Djy?g`u&h=UVuwL;0_w;9%*z@voH2FYIq)1vDh4r=#iDJR0NF865h^iG4kK z_)pOCjkmu(;<0lcN1`s&Hr8OKb9(I~2K|?t#4b=y8Bv$AMSGtIY6ll8qD>axLxO88~uu z+6@~*Qq2u)h{~E8{i!4+;Eh=dI@-IN7Im#&#RT4`@_b2Lr&FeoXGMPV<{Oz9#pVZ> z`Jj?)_CH4;!cTXvw|PIaIypmxQ3za-lJXF&*`h9P?dZ6_ec+LcK)7_(5kdUNUW`l@ zuTxO05s)s0tgU(4Jwwop~0U^}z*6f)LRL8OP{<~yoMfsO;}Nk$51 z=Y!tm(x2UJF@txw>UyX)tb+^t$QUjJuhr_js;#AL360{3Hn-U^K>ts1?;aRnm+9@u zoi%O;L-E`JI9@~ffnGlAEllxjF@}>EAhf@ai3uz%4X0O-m@>O5yu!&~fBuuud1vI&|??-#Bx35pk zPP2B(_F0(*F5DWj2}kxot>&V!8OG-!^P-&+YN(E6oMbE%l$0~Seqn`5hJ>^&s~wMq z_nK$i3K<&W>Rr7u>OO$Xde~trX&W2s>2XY*ED4MX_aw@;1f;m6eyt>g zQDbfG`X(g_!BDp#D1+6@5UC`4@U$OQl;?KYkZ&(Zm5p7Wz3tn(yr&wD%(k$k-E$Y~ zl;m{9mIZXYVHwXRwwZ}<@BDc#ze4Nwci5}@QZ{{+e0qs+q}Ag=Wcz9 zf|?d~lyvB=icqnNf`FNUQ3pn~8a?u0>GrMtFQG@RXXQ@*4=Yw`QWR8;v!C-pFM&ff zat)YN6lgnd)m6~Z8J!XC#Jx3vS-&Y51UNmdvXZbu_EudZhZaIMi^-yl-*ayn zFFSuoG(!=vDt)Tb$FvTYo!>Z#r@J`TR1;ZGVe`1xB3`^O=}dZf%Xb~O?AgJZP{8C# zSX#NNKN{xq;Q4OY`qe4km@Dlx6Wq=0XEpG~a}izt+{biBRvkK>Rr9KC9HknR*da1~ zH6}PMVN}eSzF zV4~&`a+*Uscj)$+G^@Nz9r>-Et1d2X@B0Vl#|m9DGK{vii%w;XTB0bnTig!ozpCEP>WqmAODaqHLU{&5`vCh~_V?ZB5Ky)iu zN2rFg67L_Y5g{!rOGM7?wBc!WpzB?i_By_>(DNXv;;`&i=;OzaQADhqT-X}N?z)46 z15s-#>^!jD8yb^E_@F$J7$%z755;qa2uWh}pKIfOU&^+9F8N;c2iNjZC(UE`Ay#h%zX@mq4A=6)Q1=J@D>t_k)C7L^eq)H6o#Z}MzB6Ngs#OkiaDAKJ$-fmw0gZ4)=SqFg-IOnCs zpp`}ICQ=F5xnhk?cj;NHjQtG#iVM%Tm#GMCUq%{8KrE(n@1e^OC% z@6ktE6>|RJUA{FscM+JdRwC^8GB(WWyl2lTrT%?zRAF0HLTTWJ^-K^^vDOzsqaY+{m?YwE?B90VXPw z#Hccv(2r9V1}|-9UL}i85;_emZ?`^AzO-_=B(qVTIYa&mtfGN*gWD4o&y>@W@4A=C2v_wTm`APAr#^3;kxDHOJ8B$7SP zzjJWasC?szN54b*^ipVB!_gKSID5sdsh?=31RJ z(~#FEkp5{r&cEqUQzq1F;f^o$X~3 z3!_yDzp;7W6XS-~@taU8?=dn)J`3lO3)!6F(m2pckLZ7JT4_y=Xh4qNv@-~JLgixT zEUx0j#l-YGH2)dex6H-5oRUSwA7ceP`^y1AVYiE|B=0#oP;-7IvEMD2>-pjEv$MN| z*ZXdiG!)TSAr1n9JH3`gIVxG%?1V&7L8EC(^r4ovy8qZ49~Y2~^<557M_l^)`cI^N zd?b3p;r!L8L#UYIpsWW^7d{IcTa1;is&T9yDR}LQHWpEKbZD3E(9)u#hy^%=8iG=o zhbgSCcVcC%7JT|7o#;(&C-}Q5Kpb z-$#zx;x9Ms8~x(?5u0RYV&avi_x!m_vro(6%kfPxyIc!`HZS21S@F&t{(zT_1}^C` zrD-QezD*3xFXPKuBM1A6jQl1qdFvDxIh)#eG1#3{>*gynalrY5=DkasUEAza zP#~LdzA-l;w_ST38ymlZBbtU?E}SQcSR|X(U@%cK9UQtgujI1cXjlc-Y{uUiZy}Y> zQ^tB7o$yTk@F3#3&CedTAJr-4XimQUnhZD<%TDA19mrwxA7qWuBje=f&n6$_)$=Y` zoN#CfK*h>x{YPqZ(eUuRSnC#c-FgWEq-MFSm#@6R%bN&KPnQ(cS2I7$ zR|S7+9+^YRRMFCa!0_CB%UEU+heE_YbZs|V@mh~V0$yoLo6}c2ZUlXOFG@&Eoc^`) zbE825Mu3Gy{xwUXy%|bXE0d?z+Q!92B~@V81?&#+qPNE6UF_)C4MCyFqM4$a)yYiV zY3n0{LjveFQ5~yfKpK8o!a-GKz<<2}2_gE?g{rwt3@L>K3Aekdye`9Mjm81`(ws?i zKOnq(0{)>6TJD}L#wSbYDi*# zyGpervs~3-G9mvTFMFl}=e%c9E>0Ps=_9yx>n$8HQsIz%iK9VJPv1@@;L^=5(AV7j z*wP%Xd)?Xj3{FE}pkX|bxk%zOxVK77nmu7*1W7D{L-6=lBlzboi%GKX zUDD9)A&25h+Vszwi_$S<%!$jRMVu)K90o;XSno`rWTAJw8;A0vsO}7^=*>+ zMRLs{b0FxDnwQn%w4pJQeq3jj*cwOrR+csSRln5O95Q@c7J27AwTLyl46+yy?byGG zNXLDVru=#LuDEaYVM}Ru#*G0SnO&rsq9T3H{3u1KM$*W#FzwP9n{rxOKy?Dcx!^c6 zmtMm6(JIQzXt?!1lX%^TpJuULre92F1R7-Y#j3oT8V({>*a?UKaU0q)qPHUa_K3Es z>TL|!P&v9=EA;_!Tp0IR8DVkGupndv1Uyc6ebtHUn_fI{kk=`gz-c=o(@Gp~D;Df> z+OWJQVRztJyjJo=(CJe4eY!r1uAF}bVC)mJgmzf`(`d%#2LjJeOb1HsL-z zhtuG9I(8;1dU{i4;Xo4+GW3>#={?IlKJ(AW)n=!3sg9f7AK|F10XVHiwI61|caiI4 zIFUk@fmQa00)H9d=~c^anz~b}qp2<(Q%)bs4|K-?B`5lwlfB^litWykK8YWqzj6jh zz8)hdf3~$@u3VNk>878b#H(*Ar#!Lyl`TkW&QFDj>4A<;nrX8s;Hqc|M0JePgRds_ zde5M4M38-&a*Fe6V|?*ml|&xw6pig{!@-SON(7V=GjoE#yiHe%u>edKy${uIr6Y50S+1a+gGbQykm<+y>~?5TTOS;E7KbWZx8q z$9zx59XPB%pQit|+gQQtu~Gc1==G1?qmqJ_oT3Fr0Gq$(%6LSY`gw$j>R*6%A#and zYz`cw<%ny3l_F3(5#6Mhzoa#`*Rz7m0vQH*I^pp{dWj#-R?tQ!{et}=St%uEn(;vprq=g!~G6v8cN^gR(# zKzyjRHLL4P@^VBG4n^-Z;ALpA_&v|+L+2d528|5jO%5(K{Mc?P+Q?c50{v*M!^zgk zEP6as*(~TgZIP0k%V+1rw+uYDCb^Nop+*Y;YUa-OKPo4cq# zn!ENofXrAGJmT7x;)b1Jg7&|#=H})Q2tdMWCC0#p%{7ON)KA;u9YF%q+xgWdpQG#w z=dta!KiwWyP*;Bo2e>~KD++tqz+uUyv7z2{eQ>6@&iG<}^*V7zooWYP46o}J4dP|{ zN5(m41ehg!QE&_z4{nqN#B#El$d85wcp?PKpaqR?+sffj@UDZkr2(u@+hFfCXnA)l zVxEiWq4;ujPF|_-D&5gj?`JQO7%9;T_FV>VTQi#j}LPQoOC^ zo8jG$fm+X24k!%g!1|&(ew*nI;q$I~DaD$;epjG^whA#fMb)0r44R|jA??#AY!Gh5 z{n57J_wR>H*2u`|;%-=FhnNQT-Nn$e*S5W8%q3jywEkMaSngF)(yU83&<2CU<}j*& zz;inf|IHFU18tdkZStX!(f5+@k<@DY@hae ze6LKy+NwD?AA&ELD4<-JPZ~|<~rN~z#-oe(CJX!KuVoUH3;5S zV|;S8I_P5Pgj$4; z6+WmaAhlm0ohEA!txu|YoNAFQ{kvGkGpAi`BF9)6MoKzyJ5_dld%X>P0N{@viFFb7)au*K7Um& z;#?zc6m``>;x1We^v7S1lFvr9E+^i~;WhEKS-wEbx8lx+<;?f3I&pGxE6OVecNx=h za>k|z@dbXB-z3Wx$GZI*UmVM>mJ!bSnsVxMk_q{JC@_*t@4HJC5fhWe>awVzs;!;R zx;k0Ib61MLZalur800s^&z_#ck%|nOd)=G%tiW6?Kvreo=8lFxu)u}I4{pNqqgE03 zbvP%l<$boQdy|wDEQ?aS&TFI)*QmdAK-IM5m zD{INg%OkFC=Vz7b=6x#0j?4)=s$A+w-k!+FuEZ&+`taj5Y03(xMm{%oh;*Mh7cE^& z8NaWRO4Qrp9j3YYbk_U5W$UY6Yg46_NE%XKzxkYK{O;1Rg(ivWf2Q8R{Y020JhZ;ts`7BkqphRE;Ntv@Qu9AOyE`uPG-O_)0zWCpZ1k#oyp3jWzqq-)4P$Iqlz428C4) z`}P8*yz~7U8#@?R&v#<#wQ2&iT0cu>!%F{MZup1iE4X#H;VfH3Vq&L5kC01#g2QA4 z*?+#}?`0DW@$vDo+*V|85dIDaM_-_jcP#B)WvsrOTYK)%&UkxUAs-O@_baWH!AL2U z-lrAg#8=301B*!9(Y)drX(P)`Nv&1m|8?INvKBW9u06MpL51CaA0{aoHI)DPbBlGw zc%jqX|9R1zdBi*U|9%3leU!xjBmdu5{NMjMt9}?4HHrd;hY!PbSLxvy1BD1~wdMs? zT1?`g9yMq+;@?Pl#+gHr)mO!_`t+V_tz!J039VK`({KM{7CsF`ivG_f`#<gJU)&bj7nwX&I9EW*Q*3Zwq0chOK2pgDB( z<%=pThst;N+n_7?^=?rMy+>J?id~HRf56dSnIHYMy&b~ucu4XZ#rVwEk6hS-a4d?r z(5dAA^chdceV#JR0hC-g@FVaq82Rbf zApLJ-RH_iY6?k#HbTE(+a_3G=Z@03CS@^W2%yV|J9s6cL~xiEab z=eHxK7X!E3RYJZ%T6^Wl%CS2wyFBz|G3znA(^Kuz9gH%6~?B`SR(L zlS2lEIja+Qx+aD@T-y1KO_HAn3Ym_u%&pK|12iF{YgFv*TQIEIpB;FS1`qq1 zKy)X&jfitWC}M8ApQQn9=U&C2_eOyj4glRiIg+>@*tT5~34(~)s< z;s)tQN1`A)^Ni%VA?W)tOH&x7NWPjZ@R_mPmZW~yJTBxNisIrZMGhyiSDgic)HH#m zrx@fJ=ov48fd@f#*Tm>(2bid|!*a-rmoJTAaT`RqM~;b@2X#?rRy&OUdOlk;mLg1_4Su(1U+v-^$ehUl9`_ON;d z<`#^{UCpdj!qIbN(Vz4+{uIqnoX!TF^jgrI{j44D5Zv(kj#GK5*dF(9+oMTfAUKR% zFK(61ydru2eS8!)?b#(_9ozrrIANv#1{DgTQqu;;+IkG3NKPeg6 z4y=^g!@5hv9PI4pX_kC|jx8Ae$>}t%-fE_T|AK?%<8vX4^UQ0kHaol3atLfb+Ti`Y z*MS@Zqkyjy zmGh@4^~TCDK^ghAmnM>fcdDFS_V)0XFFev8=zewhNY$<0&SQuizJBYL$w0@5jNC`V zjdSDGafd6$PWXb(G~17uYw|{A?Cg#h%61Mm{-kS5AuGLG3M2;CaBtZ4sz1=yzJAZ! z9BldyPbMzL*Uy04`~&TRjDo^E2=agUT%6C&&u;@7_tRlP0plwNUUiDVl7*aXfG-8U z#c^uQNY%-9`x8yD73A|vfknO59y?6E>gAWYyZiW!xj9^#zJLGzltmE6$stuKbAxPt zME~eS)UEDouym4;evKrgoB28wm#9)dv{QlQ9}w{LqSr&{vVE^I(dl-qTmJxxn3{TU z-2sh`XIDe@a6c*-lQ0`es~-JmR7Oljg__zaUzgCDQ2ihf_@00&4hqWGFl}cQ)mOdq z=Sbw9>TC3;Z-gta)t!!V^06eJ2kdkx(!Cn__JyQnQrzp~+uRK;6&2|hLG(&oX8UW) zN{4i!o@9j=6D|GmRD;Nqr!L*l5DA99X3WX8{1_5;|6$fyVQiJG>Co30s^1h}Y$Hy6 zjh;#P-nhBr?ehzzKgP|i3Fn*Xp2!C_-2w$?X?QuQ(;ShvXR7s7;yb0h#c#kbCQ`!- z-!3bcHws?b7OiE7N1d}7a`@px7SIWZJ@X7z^s0sC$xm-$9gVwSIoR{aE3&WE9XHWqSq=UQ2`w37wK03Ts4YC6y}JV+u~vR~1;KgN z7#i+#QAg*hL@=(PC>fH26$VXBO_2>j+i@XS82bjEf$!gkPx!#-50+Ve7+(A>OirZO z-=PgUJ?i$s*<7k#8yg>kg~!8@iF2HK&TgRw*0Ry*o!b9uQo;j5_p4w9Qv_wU%WTfA ze~^oO7pe=6@T*F9E1!9*Tg+;+4`1tL#c$ny&#s}9?6spVQ9eA?{PuA4gyDp@A943m=8(IIge<8NgyaXe~cO_1w;@aSGM z^9%dVaoOM;Ht>kczyy*VhX!J&{8&)+^oRO%eJ*@|fuBEnO7kXBs#Z=FW zzCR3Z5iTCG%G0Sybcn5>Ei|=qd_d3@s~{;!4$Y`V^qf~ohWpk0)U^9T-s3LyTXE!g z?h)=OTKUoU{EoNRYUvXMyvCUcD>03(;M)%4#qA~rI=$1g?&0cd$xk#}j`A+=nWujw@h7kv<*sH}G8FsPZ zBC4{H`Q+@*WMgZIXUWVv=^K8ZKHa5_^&so*O*$wR*4fDH>ACC8;ll$3CKD&!4k6p^ zvrC(&MkO0)r5;GIMI144jioklyg58kf2evpp!TOYd}dor0w>*RB=F4z{m zc0w++$@l?pz5SnQeE&iB=(Y89m%?>bb@b+vS*~zFVu;>?p~Uf48U^w1HDp`cBc+WB zsWj8wzwh-VdPn|E4(z9=^F=s$ouo&`CT@IwF-lsamnpKlB*T~qSM6S9;iYdP)}Dd1 zJ=gCw5{!X=*C4D-eE$3ygY=%lw$Fi^e!g1!>d-j732X<+ynLDZ^Cw|0YIwS~u=e;d zDDlBIrVsX8WN~;#s%m5b)~P${w8g@7(!h&+Q#`o=tzfFd4lBb8omUbP^tR^?M48*fgEQNiqz=z6mWI-=4PYMbQc4eX~IETh1nnTEir$hV!TwL7Q zCEBIU1%pK+cNRpuo#IP9+xKnvbH#ta?E!yb{)1GlPO0)38*?V%yYg?>pFQ&}_N z;ZEgoK$kW?3d{!6!>3qPy5G~&69`1K#GgO^+Om&upm(nF{ShDAOJ8*5`I*Q268(y6Lh1$M&FjJw zm|JA?fa&pLYOyss9KTyWbj6I;JU^Pe7qk#aEw_ycSw0Vun0W-Vw>M6u<5pV5P+LWZ z(D=#0?ZPLb2nTHEsz1Lam+Q=XpUHA+dR z_AO=S=Px84#2Dmc*UWOmK9+Y*cJ{)>xSr&T7ZuX)ic+$^Fv#+5MwF3#{;D!rRM@N0 ziEB1cSjD|ki~0Sb)d4q<`y3q9@QB-Z6J2<^6fJmJysakKz9~zcOfioGp8(9{>#ae6 zkboE|Iaj>LzucT#7Io(nH@Ty~z(fmAeAoe*XDeJd8u>h__y1hS$8WY#A9R@{Z&$P0 zIvR&tD9g*ErOS_YWqX@uBcgF5+p#&?+M`G#E(njHCI@LBK$u$cn3Z)7R`xDW68Icq z>sIQ-x*x)lO6Jd=Ef-T+ZXqz6YXuR(rNhFD<0mYnA!weMk{Vf9WP-gyYD7sacyw(; z+oFe3et{{X<#sYsSt?OYe|X8fDy~7(r4}et#&EYjIY~J3=MV{EoImKg*;?~x^?i|J zM+SJi!*#F4+65ZRbiJ2gLK!Ar?5(t*@_urfIXrxnpJHagtLM6|Y+;Ltq`^(5c7pm zi-^@A(u(2^L{Eb=7;kR)lz*8J5}lx@DyVT@7ie7K49|NvE*yoDYy9v*Dw<_Xrht>L zk9MZ{*4%C^btsCv`lwqmmh;4U+9*x4>^FPI%Di2JQ0c{P3ChA6^@3Q6<9yx_aA&v- zHLAR?9dyNA7R%N?dkr05RA!gMycH=_(DX(=sR+th_A6s|@T?v^rI}X~`JRyx{rB(N zJhfc+?mzzfnWda}da)8}`Ls#>f@ogou}@Y@B3!V*@1f*eZztf@3}y7!Agt3c{gOKR*C zrX^xHeMmIQVR!}l5YJK*nz=bCkn#z0d^lJY{^KIu2d-O|kFk#h8M$(^vZlwb%3_|3 zO_wD+x_N%bpT)&sG1R{MhL~K01wXNiXmzw?Ba2e?vUqvlatQa?Db@zQK<242U%Qof#o*7OEI|X=v+Gl>+z)6CU{zf%f$uXY2OJMw zp(y|s(tf%B5eDPHdJ78af3VUPu_nhDDFPoQ>|%PU z(E17eWtp2$ga3$gJWGW82RhW<8Q{uH)=iJZj3M^oZFWBa+@^c<|t(n}beihbrF`zQ9|et9f(iQ7Z=KsuYy*;k4j_8x&*Lw_SKne9mI;M0e($^AHBFjHn4Z zV{eJ+CdLwo>iyQ&Kl${bS<2l}(Fo+W%ca4opQ0A7m+o(%LxRdeRk78kroMN?3r2~V z;Uo_6R}Uddw&t~yixFq&1mR@a<{_(6uaXG$2_2aiO997ed-&mNe%e_ZO9G5yuaPr* zf*x$@y3QfmZ?m+~uH4_;AD?*!t_KN1Zz!((2n*CcsxS(i9q&&f7S0~AuV)-0u@p{Dwkz-c}TlMP7m2aFL0x$|cOY0@PL(%nR zgbj~A+3ti@E{?BFnd3f-KSi1TU-xW#s$%cOpY#SXgk=*b0#m20Q6O6c^h zmS@>RDfyMwRInCGwy_q6(sG9hU>$U(<_c)oRtowzXMcSuC^l3sdt=GQJ&1_k{Ww)u;s`_M=fhS4AwmndV$@>Wh1DaP->C-g&?!II2OH&(Lg(@T)8~0 z5}JgDRT{M8y_5w?-w4hT^xVLU{BEl-2UpRqNoC`xfGLR+&V%=Yu^etJTKSmK{7JNp zQki-JvB{F7wSJ7{;dvAHn=y)n_gwL7e^xAuxID(L9avh1koz9~sM=~=z zJfJ<@FRNqVAKIGS3*#L(N3;JoOS-l8xMpJch|+rhJ;>~2RVE22bfKQC|4z(`V>F75 zE38t*ZnI0czbg9D+FDpP#ujj{df=JA@fkLndj0Nhf1>6`{HQg^+Ga&W*2_?N$rL@( z{^X%@Hx1o{y|ad@Bsmo5**T=oVx_FIa;!V8e*0CDm@hl$*JNv=UF1V&f{sd{l6fBX z+AnwF!&*_)$$AY9T#I{DE}vr)6`gJF(Ro%UWcqAlVJ##`q$9Oxls`}DwKy%bE%%xK zg0&Dir;qL%dSktBkXFOTtc)t)hVoicTAGl&CjcsqQ!2H~x>x%itkG0eRj$^5kFrp+ zLuRYxGYu8m=2Uh0J5ih%R^Y8b&F4z}?&D8FBqHm??*s3|@h{X4dO+ZEnV<`eQOml7 z-U`zgM{p3aRF+fNnsWe5ZjGCUZ%h_Qe%g0BTUaa))+`9F1}8o$3$xzSiIG(%*~WcY ze>4uD;gsOPy)D?}_yJ~T-{csvmRD#9)}j3{;V^tk-aj-savLa%oY%QOhpgao`)Tm| zGRQp|x_!J_Sni5|y;_~m9*;29f-2_9v%(ZbQSE$0UQ@FayQd80 zqcP&4sta=I2s2k}QKvRO9D?PB(!v8fHcmQtVnIV-zdftzz1?`G$7y@g<|R|?BR0|m z&W@}CiL~*Z%j|;A(iMtN`?V0s%&rdn6j}H-LuTUd$4eZ`9z3m2{`qvHBd}3Y8ZrL8 zVF^Hn%^l9S6E$Ui4ty8Y6`UCOcZFvii~=He`(o4gRV<*qFJ=Y4?iTSQx#mPdCM@A!TE@~_E zeV#qT6OeDG`5{dFMZVUF%dBh8s6_LqJ@@7!n(6E5sS)>}uycn}*^a)ekCS!UY5=na z4Jqlr=@a^4{-uEo)(N9B=JX85*OqbnGD~ec-RIw+v41*1c`|Xf?n1j_Q?sX* zHS1D~x_xMSOdJ@I?$MuS6|ruk>BPWrC#FMfwO9E4j^@#qXmw@l9ER!u0~zTq zT`jE_>Do1_Wa>Q`@!E`;E2Ky@^=+ediOS$p8eO8&F7%x^< zuodIg@o`fLR;$=!EnqZ4-6#fjsxLF_;P0YyjT&^?O!M8fvqgcomeBK!ZwrQhLR+(| zQP6At&8~$Nwo9?aoE7eQ*}PoZXR5hw1-^)6&0ysfL<%(o3@o3%L?) zfdq64U>zPZa%2QxIp%Z6?DQcxLAa8ibO7e)-#QV{MJz#V2nZkOb!%2U<_9%e?V9V4(o{_`D z?k`_53o(A>cM(zVw0Drd&t+vU8lEg90e2w$Oj=DpCItas{Z_K818RBWxb>W(9@k4M zh=7r$ee{B+i^V3$^56Y;V-vK9f0Eq--zV1jr%Tsz zFAV=g`<{3hE)@91=(=MYY}6WmlSA&lTx-s&vY}@x`?$DxeR`DfnY~?&8VsiI{g}wP z?I%qWM+|z5i@3_ZEsZ4Q4Yb4K`9bP9vW5i)0F{?7alz|PFJuL5`+gVtoLK^tuQk>T zOC^l(V6%43>Z#1QPSSut5L{0BGUScHrj(U-0apIZwx*jp;BxS4c0lF zfMkM$c_VnLwbQta+G)Pm!#v2Ng9Q^0|2Vx|;c=69*a>(S%WR-&$mJ%dv(wq-XE6|6 zRTyoHm}+Uy&aEOmR3Uku$k)XET`q>r`-;HIc20V+J)-o%lITXWE7vfN#Cz;<-VNb}M=k*>;bn*3Ln z^JcK1PpIB2?DnixXOqAYM5J_ebS-oYQAap2DJG^Bw-NbM@YS=QxWG%66euA(pcDGN zMQnyzV~s}C9@+~_hYKLCD6U7R-~jYjDUP27OrZ0c2ws=S>~MyMed=+g&+!rEUU}xf z?p*HMwzMvW&UJlRpwx1IOV=aZ^-6Y0@y^gxb@aDyf1U9o z_K|g!dTG$Wh@4p=rE65Oh9sqm)Ik z-xB8D^3f_J^-JN?&vjj>l-E9R-CLTDw2u(+c;sbkFVsJHyr-9w`$TRt{UL5H z+8e)%Wp2EN`nP8;3e>n>f0FgxA|wJ8kt^UHue;_-@RA28TTf9d9xCt)5yG4p*esaD za})uiTp`#qT^+B{_7cl@h|8pqFbNx>R}Em6KzCxdx6t_{KsW{z6HxfeWu!!qX6ITQhyfdMR0 z+Vp+=^=n3_)EM-|){Uv};?UkI>1YL|4#Ivd9-Xt{N6UyWV8-3`K$t$Hu4ez9yI=U{ z2Ap4f%vJ5XBTTRm`dm%n65J{dtBn+3Gxs!h`R5Sz`}Qqf0u}5}2i>)a1f*G}R!0`>iJ8#7!2&jg+`A-4jgC?}=z9((4 zk#Xm-?O>%B9iN{GO)n>RtiL~ba7g=+(UQB7aoR^b0tV(xG5m6d#^)X;&qa;~VS7wJ zgq@eG*l16U$7VdthYTCVy$7@e=4ij20f;3rAUUrQDA)~KPf1U!2hBN@$eI!}`91UQ z#6G5C$0NqJ_OR;h10Q0-$wEXQBSWLRN&g2yrq&`Y>*}L{OaNtxCpWUnmfGZ~g|Z0F z`rLK*VLHbX~^VmYwvYfM3JR z`-_Wjl~z^BtB0Em3d-+l8wcJJ?O8^)3DFVtpk_-yBAp zw_nIC%R=Jf!uLLOWxQY$rCE_0mBTsC9W76J5FWYBXA)#-e!&k51NoNG z)5TZ->WI3uGRY=l2EY?5$^F#(!>#o|h0IK7)PZ*cDig%DIQWYVDvDiO76*Nb!ZNR( z!Hgdc)e(MIIQ5@DPYE3j1`LOeRW|P|R#!me9uvZS<=JIsx=p{l zg)26a-YhzX-)k+nYyQ&yeuU^ejXCu+X*v%t*DFoa)Z#Wt=udzYk1iZ;t0QCES4OK30>W@IaD!iF=W?c@ zSIXTw2$Pw@-l7Jb_)TQbKGNZ$CmP5u6u_YuByr(aV9rxfRu%_u{KD#O+&b_$q7ZU@3dlNm{pE=F z02_qK`2jmgZN4Zb^wF;SggxhQRZSH%3EI|u82{|`GmQGFmmlPBD(p9tUsL8jdQy1* zhsXW%b23C_K_Rb-nvG;3`h<%=c+4jA|H|10)i@JD)&ST2uokDG;qM2GhqOYMo~p%W z<>Y{O!gi$im8vlXQ&aAYH~~1Vj`G2lPJfu5EC20iW4fDiQ#VShb{tfg%gY_`rK(xF zg3eAgTwK{63#8);<%Yw)5#W&jieLWu<12i*IwmcrP4c-Zx3cmStHN=AB#_AfIXz%( zETzC>dBSUylg%LT*msi$A?osoIXAtuOYUISpLE0Ucu6jrZ32U3w|EQP%td`jefTi= znP%)~>}wIZzstyyJa&U|B7=k5e-3?N=Aif#Qh0_kdXw=pi=;+R5g2H7!fkgIZEeLj znpX|K-vg}T7yoB1*>TquVwmp91>LXvUE5Q}_5;$&?WhSjO}qqRZ=`C}uU~nQl>(f- zy;J@H&YrKh5(=wk9GEju-z^S!*(8y+MCp>HpO zV$O|U=b}m}&3B)hNPg@7Efgjt;pphE$XR5G)GBQK&PGaeXUgI2KZ70*JmG`aZ{Obi z{afAn^4E_apWgSZHe;*B$uc&&j}rJ<3y~YIEFDTs4$8uZ6QY>H_uER>KL7h#@WPJe z@b72*VUr@lKVNV9m-0T`xT$}Vo1FX|bzbbghNk6q9TCzuA$O6Znpk%(RhN-b3iRs< zc>bfGk4`}@o}8xH$SV zbEtKgmPp@U7}QTY27BCtR%+v?3-=>$8_BFkivfvsv{RwVZ}}I9j%*X?g?o)*iWdI3y)Q5V28oFu9`a(?gm;LV{Rlb^f1CI@tNk$j z>Ulor?ljHfB9NG5mv)WwXrLFJZk-s^4^qd`zXc{?O3#%sf~|r#tp5#mH4r96nD`N8 zLC;$V*_f07UvF(GH?}-K5nB3E&JMlMqes~(AZhLCF#rutlvaZJCyY3dw>XzwGZJ&u zB+c$g%mge-?z+=Jr?0z$_?VlAZ2fO+#7q^}dj(z)6aV+l{m&nc?vX?o1yQ*h1INA= zuQ=vC#r$fo?{fbyc_m&L=6C+@tMG@TJ4UYg|IdFvwj9HPmox2d|FhXSKmM-&;d@Go zoT>qN%KSos=897lZ#?E z7qeYY5ZU^0^CJ%z*LG_vSZ2L;XF;zxf%@f(Hd9z7!n!IcvFq>r-(Ho6!HLl0lvY@K z)x@q%P*3V0*uui(CH({brAezZDpH0eCr^322|-!RO@5C8xJ=g2jsnLoX>d^{L!Ls~ z(^D?w1jcdCni;I4qo)2?Do}%^`b;E^3cOLx)h8|Z0&7{=22U3$c-^ndMU=P&pgw@r{Mn?B5yp}s@SK{MXSaeHUGmVAcUO3`?pjual(&YGU%*G7N zE+F}qn2xS{w8CKoxdDyjV3{o)Y-8_kOeBNddMtQg|0?(L_7(vvc~ek4a&d9>Kw=W~ zof4QRrO|OKo)W7ej$3phauAdW`2xmZ-j4?!PL?AVhY=SB0M82TbZ~-M(}d+2W|jvV zRxEIae+0g>n@0lwNtM~0CA=(0L76q;va0SgVBxg5r^OPDAZcm_M$=@DWV3{6fsk<0 zM}A~E*FuH$!u{eVd>JFl@b-3xs}mT1XK#;>DoYEl`wIm6D;auZAj3@ds*SGpHHwSR z7g1n4X4Po%JwXvDP^JejQ_ zh_**3mT~N+-Zi#+!B6L1O`VI%VB@6B#YjeAy50&gn@3i4*1jSMa4$*ApJ~Q3{l`jQ zI9$;f_MP6VoEw;7wY(vzH`zqb?RGbcanscA_8YxnhALPYh zQa9`&Vad%(Ut7B!l;;?GGoexpN{YD>W?CX_i*!3XJN4RmY{&C(#P!{m7;gCDVyZ*& z*wYEh5(nJ=yIFdL#_NEA0u+e_XVeqv$r(Fz1j!^@O))F67mGFM@}}CMs*Q)FvYdeb zRrxqF!&nB79<_LnZr~ui88>bep(|t2_|KI+Z-q{-Qr#5Q3`iXW5d5kAAlC7o{Ezfs zk@DIuK(T@Bwf(hq$e7K-+JQ)B#n4M=MCGVOeZ9R@9u;O8HRV`CZ(i)&pm4muG(}W@ zVKKxAobvDA?>pk$sDf^=ua5bIj(LO|FXT>D#gGmGYQ zHu9PkpQop{Wo;HfXZ)ht1}$70VO<|BDlUS)Xsd%LT#~w&UjYiqHhdKPFpO6F-Ujgh zU#{7KT7+5I$Rn1=7&Xa5x{^yF!R=2M_rOK{cyt|_H82Rn^pkv71VN69<>l^TwwZFX zAp~x}C|jD&pN4Pz0|Q*w(7n#D<1MmtI6f27f%UNMrTSxc&gRF&K{tw62M~{0a~lgD zTHgwD3Y#t+S%VULn|$%a%LUvL4^dBOFHNpi=nPno_RjBHqBq2ptri{Ky|;E7EXGiv z3Hx{a!wBR@xVYv{7cjQxjuy?=EQL(Cq{T*H<_~kSaY7wBb7+xe zUWV>igmfb4>r(s|v#uvw9IL6LmDFt++3%6&tF81MwYP^obQZKLQl@RsSEITbzyQ3Q z%*TkWY}IGdA`OcMbT+cd@$Ok0iTLr(*~xxnoaDRt`&&?gKtKkNxdYv)a#ex!b;xx1 z-y+SmVIO!7uuef}q3Rb>g6K?5Pxrrilk_K)QEo!_>BbwOQY{-t6X*M7S8xQp(ZWzZ zzr@6pU=-3|y^L8{!;)hphP>IU7+W_s9vN;|SV)?BKUkRctQh5A6(8vAZ>~i*Z}%>Ki_!vx7z=d_T@Y zv&fz~plNRTlLIKm>O>xOR9XhBZ{~r5jVVOUZPdk2ygOoH{5cRBta`?lI&1hhZdA^e zQkXeleS2z?rJj<5#Nz;Hvkf)E9}_F967^h{mWrm;&mn`Dh-hHF6p#8XG4{35S!6Yc zA>mwG@h~?r%vlg-f1i3*_u^&n&S0i(vMNUSQ_Q1V>QxirbuHWtFB5Gpgl+TW{M=0M zq~gT0RiMl4PL}8sjdJRD{a+p+&$DwcOczLVOfLcM0n~Lzty!G9(!ag&@{Ocyxb3Yb ziuA2#E>6a}VQgHh++b2jnv18o)^EVxg`WIe;X*`1(>uS1Y(iZ~Nce#qXo%(5h=zfI zjDxI-v6&qd)}R?+(X4#GI(i=|D*Zq5&tK#k6V*-xE#YPc&LbJQJO5V;;GtXPB5Shl z=i)AO7gO7zRWjbJk0<9Seea#b_#O<&_;eollKs%19hTzPZjgUXS z&&(VG`5IX3?O|bI z;PzYs5%E}&QBj~K>I(XV=t^jXoD9fVF^zxU)7&hj=Q?#YREi#^AWZ${c=2D;UqC+b zDzc!6II}DJ!ZugUP)X^oaHZgfEyMz;dtQmw9k>&G&Q~0qbW?V2mZuI5{@T+9W%8gg z9UP65o3*p6M>=r*C=(gNGLLZ?GqhEqrp!WS{XFqhfI{(D4L9W=N8XrQNoH0N5(+s9 zM1kp&>uOZws=4h8O$N&WuQVK~k$*|fKa#s6A#j?j2b81%R$q;d1a z`;OThXs!Q$(~ah-b;o5F95nwg*(jk_eyQ$(@R=hcTyL0_3V<-6)de64n3L+A^V@RV zNXIX^7wG=}9Ut2#P!C7x>ot^A+aiBZ9kz^wfIxb45N)1^2;g)W8;f(A_~~+?+<_kk zITz5sg-fL|@Ya0X+$=9yFEr8zd;$N6QXMp+Uxh5Z>zDf3jE@y_>m=?$20}s^x}lgl9N_eVY^> zRXyuWV3~XV+8gx=5n%`-Nmbk+$r$HR=oA?_@ASrq`QeJ^DA`7RN3(%c7$lhzID4kc z*2XvBJ%lG{gOADOpZTv2FN4ZBxA@qv-JjEzMunWS)Z^i5g&&9jO&;{{1J;ABK`LfI zJM$;UN*5-zTGtlP0C9Wc9h{fRNimE-N%h#RV;%>zr{w%b@6eV9pi+PI$YGWD%^#WO z2_Zb7XQ5(2KQcM4$=aPUtPEI(g%m_I1<|fFhtafw;Fr%`$ZvU>5;5jPwKtEWkRe!W zCFkhz5C9f_KuNF~0y9AE9@RhNb67;5w)35zQ)bC~$vt^zz6%3k0|ms6rPx&$BUvVZ z{9pRhd@`&|&SK3<`7?O==`%g9D>yb z;99ZG24@;HhPcaY?{mY~-rdlxh# z{kGko2qGJIh9aH5#89<%bV)0d>~WfOkJEd)^nZ5a^cYaZ*>y3IRRgv3y6+G8UmXss z96u1q?(q2lhzV`{QXk_}PmQ>^8zp8F%G@7MSM)`m85m#%r*3Wpw1Z(16XRnLKOLvI zCxm-{KuT+S9a=zE3YbIPes^Q0P$5hQpg!7)4~^Iup$1GH#m3Sj48Ck?noEbe9(IKN z3?d@5X}wuCXk_lpy+e3rYD!rjKfeF;DPhS_Xsd~@>=gz2`)r|ufUjB<;nR6;UeYe* z^X`V&v_8U7yKr*-&4304vQH%zI+u@FVE-7r9R>YStiQwt90kZIs_DwS^Ge zF{IK^0=~>}uvo|2X~&|(i|pCEP}^E4Ky)#N*@BMe`lCem8gt zq%-?^e}vGLtjFach9Q~{S>8RD<4}7JGe3}jGa)}#`0bWiVD4HS^7-OZUGj}yS}}Ko zIXHEEMH>(MQ+2s;*Vrl)Pf5e5d2HzoE9mRSFSFi%dYW8P+S%>wA0KbpilC& zDwQVuC46uO%4N8D1QPXF``&R5dUcP58@xd{-&6mSFw{rE=}t%)_95%<+l*0XL-M20 z5nIE5jmp53frrdDyOV&9n8#)JRswGcUUr7fG+=k`3JCZXrO^;bhSOH{TpN>56#Ae{ zf6X25no#oBYhvBBx+o8DEu+}*JXBIMR4r+HvcQ=qJiOeX-q5NNKAhrS)0I|G429Vl zHbM{^<24ZHfmzLVTqG~R0*Oq!)39#4u9NsReEZXa$1E%+D+4#vXdyAG0dignx*q7O zt6viXkOB)L*7RDO>9jTkYOBC{dV9TUBS+NC#01jpn{Vl6{)ZP@+I$REQt$cq)u}(a zxtX!>Z1n1(hneR7`d7So>We2&T&~R7CtkV&COdsX7NnOOo1Uux5bSnSyn1m_rcg_& zASDa6EW9hs$QZehe8vudL5Ww(rO;*zXgd1LivRM(H_#Jwt8yN(W$K0p1;6~;(UDo6 zu0ylD+E!jOxYpO~!77-%biHvAM1d<;OXsHrVD*U9Pdu`VC>G|iiZlWT5c!(;BpSWMTsVqalwAGQKyw< z&3JCc<3BOY1$rxNCKZ4zfTd%2>O)mDAFXF74QKyK&2n3DK8IV~1#|?)iQ=X%!8_c$ zJ3F3XV}-AkSIv=Y7kNq~KbpEV{KO}g8x>+OC!*Jw0t-)OWv%auGN88`2_)2C9Ve^y~l(}%h}TAlJWdK*P!5X=+{G1-7(&6U-iu|lH=&>t99!8YzI&Sz(d zKKi`e+-}q@i6>^xvJx|pe24Xo-55u=(h(COhzY-ee8tvWk?j9Yhx8cX`<_VyX(?@3 z?=T*GBO5%%$g|~iJA89f30AR<`CPvmn@2EDUfTp2d1<9>zBXYYYYc!GX3=UFzw}7% z#1t9Mu)z}ugIePid+G=!?0Y70}k zUGIvchIz9v?ExAsvTKp{f8?WQEiNbYp4*MRplQ}QtI$UCoo{uS0q;#O%}MUF^R$aU zP{>PO)#EKt?vZE5895`4yCP-?-Ru5kB<8z2oPbBZu_=v=ouj#K zy(B!8bh3^+oGjp#q>`!K{T4DV@n~V==tb zo7gtfu`j!_fE}r`>RfgFW4r$M%w&g@3kuJA!>kDuT7yL=pU}rI%0eyd=J#dyrgFtW zgPK6GirIIb-N&4Qa)Ka8oqOzcysI3Tibf04mOl#iZ8iC0@bni2S7^9%C{9-Po1=T> zE9C`R-}$f$Av`iqr(*k}#ARYiK5=29zK$~5+O6Hghzbt}8?o`#pYZafKa4kD%KYQ8 z=9KxzY`x=L)*8VQ$K^*6cjY?aWkY}YL8?%8gA%J5VN>nsxqC~CXA=L4{7vY=UwP+D zN4+qPlQz-8F;C;5BKbs0LE#o;fIfu)H=&Eu9gN92xKK0-Bn>RBcQu#cys&HMa?^&B z=?d!?7~fJzGs;3)Kx`rI6NvG)E`;Hf=rgl7>HD6$@p5_4M^w27ls$g*Chz9Z?SXf` z$e9xBeH~Rv^QeDiZ9qT<4@?x%YEhUEpj?b#Pt_26{gNEug($*PAmI85R^*q(bu8EayHf{<(lw7TlI zlGFT4DjKvb-HC!^4hvlYz2+n(FIJY9KSddPFCy?0c*w5j7ucM}_wn>t+yT)Ws2AyQ z11^IFAZE2u@jnp*z>Ra zRgb=3z4X%W_g#e>NCdZ2ctmqH=bKM5>G1KoQzbla52NMBx<-)vzV*RfrZU3+`G{*Z z>#&hdPD&3)H>YnpB;W{W-DiA^XW_)q|Mj&Cmx7A-(djA=6?6Y9?4s}D1b)9KpUSt7 z$llPkPUI72X5+!l>`MP_@bpaG^Dwr+RoWYtX>?+oSJ(+s=Sa;5knx;j5zrWI1)#rK+h#e^0JTq6IIpMX^I{R=DrrJ+-dhO_G+9ZBS zy{miT>G=iPMJWkCJ*!U)U)wep%ZZ$6`r%{kEmL&aUF$n59jS09n81JDG(YTs8=xJJ zUSYUjsa{c4l?Gey3h24{`Cyf{lY@;7u6lE3uKrU47H;l4dwYA%x(m<~lMnvF_a81! ze3cxHNV=2qD!cLS52w?cWNdt^@t+OZtW#AVH)ZdZGAbmPt5-P!0fNaJ*xuRM0>8cg z@+S93F@}6PTP4^1kB^SUeryM7=|UC&v&?dDlc84n-Rp<;d!+2z=6@&0Sbn^&`2=SG?AJF<>@-RrXDo?3q{9Po6!*}l$ z?ywMRp_sF43e%qh0|hCNdtgPjbyV#1lsxoP>#VEcZ#h>F!R|kf4pC{}#W%Qei?i|m z$ZL1VeU(0#_JR!D{kfLBN@;7O?{%)S)fBIJGl4usJx-lhv{VG{LUjlT*Y zv6%TsW0&!&EsB9ik3$=!+)e@mt#}3n$z2B$Dyk3$*=Rmp7YM|w-XFFEOAYC$kwHi; zhM){zh;M`hU3M^|vm1jc1WUGTznrh7PS$o0G6ybWL5>Tv2q6`>1&PSfEOCO6+ilpt zOW+{D0`V~p%Oyqiha|X|Z;A;CgY0*IK<^Fye!9oDCJWxJ{a-wIgWrb4@`Uk$iZduT zI+PEqXUSq~FD^f;fCv3zIhrseOJ-q6{aQo>t(8@}caInRJE!t}FYl%PN+7?0mu`}aEU#Q7SJ&9L9Uf6cezG>6BbyI zXMxM2nQYyAcp4?|H=S0f#N`LZ4x)6r9X29bl;89P=n!79f=tP1ne=%n{b;PONV`w* z@zm_SQ!=Zh=nQeCmBE4anZSr2gQ)Kv9c@kG?Ggc#24CCf9%Mu5OP$ z_os#Q-F>+J$4odM*`9F+ZV1LHz0Cmc6)^@^Iv38mzC1})DdlPsQ`RC5GLy!E+tf1#qzBAb*^#?;`UYJWrkbYgIP@AoW7NR%s*nr zs2i2T0`(jy`}68-=aH^tUi@Sh&6^MfV9RrgWoyfD>gA=Boeg_d(mY)GfYziu_>X+?^H`Y#)TlMRiiLT(+RLV-`UG~UM z^Rt$bFipan-t4bCQW6grAL`U3g~4C})t^XQ{G1R^>Wg-q5kAEiJkg0}xuFGbcKnMo zD||(ZhIf(uAG<-^9NMEe21Gwp#SXmzFLmwhJ3~ z%E_&d;lB{B9JM2z^L6XnoajcUR952T)(tTZ%;aTfU&V6uU@kHk|3wDvo zdH-n--5zo;E9)PokICA0i6dHsp#R%BEuQnO{o98}5AS__X&5P!qErqdvCG)U zOhK30N>A}DmxMj%ETRv8&TSTzIQQ*e)R;dCmDs)cL^e(0xI1l0@5PgQX*2)UD7Sxdra2*?EO5y z_ndvcoOga$OPRu)ll!`_YmD(PrZ9sa$%e;&51zO8JFSf0p##^y-UYKJ22)t$ot;Tj z#P|jMeeae@absi&VYLJ94va;(5%B4Mx8!L!*jweM;~*>%ceM{!xhXlnB@L!%5x>*d z6<=I@nHAO|v$o%SZI@K2um-g4YS@fs%dg9uWsGzs5GVomvj;4^vm)KQ_wD&6b2Z%O*$pV5VG0*1E5-w||JA z$&7{Ak|6W&;9xRhLX45GytRY5E;vWCkZq=M6I>e%cNY_Go#sF#?0TQCx zwEcY-DA8f`_TW(r6|7Dd)Axg#UwC6HO^lRVCiGZGCQ>xK2TWw$F;9+|>c_NR!d89} zPo`;38hFKwT&at5)}>T~wHad#AK!d23s-$Ak((!_q{J*A|G3Hft_GuVNXhK^ZaPAh zm5{nhO})5-RoN-Ty3FNQe=yi@F%iiVeeK%Pj~rg%1kXrzwF2?u1MoVS?u3f9EsFgC zgn~l;D4dd*;E)ThG1Fj*84#*4gE0+AxgKncFc_8qXI99K3JR(R%gzL2XQrDs4FKyz zJ!5SAnG!Pr?6St)mUEdpQ<%Y$rg>#(Zg>tg{DX)j%?du0VS2|y*<5B)SE9_U?v%1m zG`_mNN%{Qf7$S+ZNK*;4cSrcrUzpzV}|6pC9vCfME< zffd+f*qp z4^`LxFs?8iVYzi%!qXGu+IPuek62e3fW5_``jK)e=y_2e)nl#O;9Lp!`xHf|2%%p+ zjs!6S4*KKAy#kJhzLXvs`@gIB2(FWnM%I~<4u87xelFdQV)fR1Y;X1srCbtXkqO)R%5r*KHpaVhJ)He-jK0ER`5!J zHiQuK^HV_Xi4&Tl$zvHAh~m8n@$T?zF5BkUfm8;-;-_b3w7fQ?;cE{3Cl;TZCw{ic z1nrka^8YGm_Vh!?AU15h?WkhNUxx*rFW*f(?n03-{KWdF>o! z{lDk7PQ=_9@gn7puzryKw_)FwQQhHP?$pdo-x7m^mjowpd`K_yzoQoZrRTZV=rURr z6Z(CcYhfouu4Ln~$Lle8v$*Vhe9%8pl$VzemRM3lNX(_nmlLdJAkO*Ir|TKZR%A3Z z?;t;dqQrs}0;V=LHpo{c|Lfyoq)7Gth`pj_+C%)azwq)rnv7*rq!_YaUJ>pEnpqor z(sHZ2j~KS))yxa1@X^5pT(Wfcw=(Y=8CC>2IV#^xhrnr}&(QyY*j?KPdWO4a2MUnB zR9Qd)&AL(xq*p3ozYh=hVQlHR%DZ~e&h8>HEiIag54+vg8yoIY1j~;E+VIeKAQ$zt zT-?J;^3V3at1ci}tp)T(Nv5y{Qy$$-?^aBy&^zX-3~Ejm+!`|9VcG9`XT968C!sOWYbF%Qpz?cAsls>4cM zdNXziyXdbN(W#nNEPuRq3NNcac2^R;orL=H&d{Jl|Ix}S6_Wm$<2J#}YXW(oZ{k}G z%v`^gx;v1sb2mr3`YMR=1I#!*c4|& z>ngjB-=P{RsI(XjVkzN6{qt~if#3@GrnF>j({)LP)T5`bgzZUD|89-x-y>`rW(-XJ z$t8jRE^Sca{9{85e@P__eTnM1|9SimJoRVQ6M@gQ_c3(7zj*H|v7^gXTjgJ;ajDfr ze&=tL`#j=wsH{w7Vmv^+cS>~sqq{pjI_d+C1qkpaqW;nOLrCO5KGMGj_Woa=atgkA z&Fn%#9{KzeZUkv~y=%GNaP^wP)(7azA^Di#Iw%AP^bny8Y(xEgPVYKXmCLEMDhrjM?R(f4N@V{YWAI5n7p}(y` zz}4Hg_w%Z)o;lIUCsz1C>9DYa1wtUb!LFKxnDDMIn*i*~$5;5|ms#+8)(l3GGo?`Ahv&@tJ`7=4laPQss#}W;@)OAy^d)6~3aP5M08VLex z`Ra#>-prinkDnbapMnfD>9S$822#3|GBpAT4sW#E%ul$9htc7!We-?M_Jas-*55yU zf9*)L%#Ih2`*U!ShU$~Fh76e+5_$_n+}Mc@Dpn5LW7En2hTQT_whWyIPVy3AEi|-b zbIJ@ntJ9G9;_`bfCAhTo<4eUE!$HV#=@pcW24K~ld-0NXnGG5? zAq@;~>VV)jY{>8%6#gwCv;wx%m@q1XZSEELFB+^27Rr|@$jLR;`(fKc&gQL%$J@&T zM8I^qXHzhPk;*^rf&nSJ->tp2s8PI7y%S*7h0L}^VIX`KgOI&eMxJyNbt~`&zf?p= zAfzxLriY&7EH^Ln`zgJ`wDO_jpFd<%zshAOtzu58X>z_V=L?-~%HI_YYkj2!XI5NT`RdjpyjubUi11yCJuNG?1h=_^~uQ(E5F_x?fU?PFRL3NF|X=n&G zVdD4nZE6AMQ?A3k<(GjQ#8J(x%qnRG1>f33*1fi&@Bh)k3|GGCy+rkg(MoEFGu@>p zyjEL)8TC6qtut84VUeb%tdoBvB|F#eDNDnI zX>n-@4PdDHhbv3_Mq(*4?;5yF^0eEWV$0>FXrEYG-Vr-oV6+%4sJA*OHXkSeLlqFu zU!eCs=KS+_m8$#>LHD=(zk0r3cvW z%l9%WJ;~Fm+{n0l(rWb_t}7_T0$`?+?Cv~IbZy72y5TEih(ow9N^XPii^>idyRbr~ zAP;~LhrzIb)csfJ2wYwb-VkkTakxw26qFem7~p3(G8Tj8O>?*=MItn3MWpvRI$L{d zJFDG+fJH>)7bQhCbO~a3Hn{*ZJ$;G1R{lxOKo{TT&d1k2=>C4i2~}ZZTdgY<2$){h z*0Nu!aQNQKOoWDD)(0;0+l{QaAd|y8aU3zf1`PV&ZQ@+S7YD%880z1P{?y3iQlCXs z%W^+2Vg9KVoA$|=W(FF!>bGrAmOCn?`<_4BS*0RAQ^DOU)Oo!^#V8%i_251my{fsK zhXW7#vB$>f@e4|RC(3)0=unisbA@6qsPS0I+HwsOfxvM5yQc=MK z{LRSc^zpl%lD3C{pX%R+^p!e~ocOH1o=zTx`c*|z-|`ZF9PTG*M5Nj7@& zw-@R_?d5`$AY;f}xtgC$2|T&!uaF{e`)pbmsWWHq^pA?VgxDN6x&qjYUhU4L`y+XL zu)sD-i(FOLhkmXzZjR^u2d7OIFf%qPt%iijIWfU25)x>HBlKc!(k(CW~s_ zo-_{|Rgp!7&m37Et$x;YXBO?|W=5N73AUl!TXl|-$mgGKAE3}(Kb?8_NG_mWcU^|x^w<iV|l&x#nL z>T!ldNefxb&kD3RBfSe$U@tA5Se2I7E_5McCbe1^zf&`0Fj~DwsmyLTC?~v5O8fHW zyez3-Q#zCb&>Z^|#U1``qCHMu6#@?a)?LTsiW0Y+$gdM|I? zzJ34EbA!pf#;m8d)s~_VY4Ap@`m*h|)J3**otE%NMCiFyojF829-dX!?ltYU&Z>dy z!K_zwrM9>FAyc>NXsvSLcRF5bT3XXA_#+5-9Sgv1|Ka0DUto;?Eiq^{}|a6>(Maui9F+F z8|dq=H@x)T>&9gnRjXGO${1PS3SX;2kId&;JD^w9w#dIRw&$Lb*DIc4BMAcpt{561 z^X@w=3UOj|nn4OVXy-psVgk~0@7jlZsu}@c%EMQaSaGSXcAp$hJ7P@6s_;a(cA4FM z(|;E6_N|I$3MujDwhkA1@@)@S?nD2#Nh8aIChSE2x+3+b6@6NFV0zoY%o6M;l<#ox zH~jMR`QiDS^6C0BpGDbpte1=XAaJwfuSO;-FO0_AA|E$6jhhpR67EX1U;9 zDQA%;%%mlR=*^~Z)Sj*Jp$&e#WUt7>^mEhm@#CdyiM70Sqw$~Dxc%zmFDx%FJC1M$ z23m-MmL71u?9-DY3_b#XO+d-8A$b`{m;99|_Xakd8hnKEu2e?@9+L#KU4cKU02|~c8>`9|+NtyhIXu%T_7dN*$U5?44+`FkdI`((i z0ROnrHqZblR*Fr-R%eH}VzOY71FSCrTNbhxQF*AMF*4UNY<0lAB(*B_qFp&Y47*;we=m;I!Ft+U{XB-6VA8qiavYy{ zYB-#vD_{96@1?j>mlzZAHh;h>p!K@$z*-s#rz<*|pCb!;9-1I@e{^?;>FzF)r``El z+q$klP#+6zCH8D#G|d zu$|7r{ElnV>0aNSqN*9ghliXen}rXT^ayv;O$_y5Mxm&r60D+Q)okC{LA-Zhx3i~T z=ego1$?0Ij^+}l-avKVb$DA_K$wO!r>dUw0*njuXd#ibz{e*kaJ=hn(Vm|Sp z<`&?MHOqV|hfNe)*$I=+zj7M864|-_3^=gc-pg3Pf!}JlOiZDa)H5KpKn00$`{nPF z$T)Mh^$+jgPj)Tb;^QNSxf9Iq!Q}3QR4EKGY)X19^=9nZ(ZLljLh`J>Kgfl}W;`@&E`QX{iBae;lvqq!^6lR)q zGAt2O&b40mRd`?kS;Wc(i6gBBE3v2!3q}5DzkzY1Jf33rfF`JWmP4kUl z-5jA~904y{_nVly=bTC{;l>Z5S zDw?>5CQn6UTqirM3<83XU~&t~v0X;p*63AI`y=s8m|3;ljKSy@biH5Z#fUwj(Yz2z zy!?LZU^^_V(cBKkN8_&67o66Gv4W)DWXy?Rl^?H}eEast^DCA^0RGYqcws1UXkHU7 zVjw@*ZVzX1>S*?=$>}I5_%`^16{_&HRCZA8mQ&*7Q-q@n*>O1LoESMvLXIWWT zvT6HA%$CH8+WVm;dM%}Y$-Bm8^Pf%%pki8v?>MxIvr6I4Fb0G_7@K=9qwq96I(@H%wb%SZJtYD#zO_zRREmWW-^M#8+GIQia%qhW9B~NjE_Igkf?(+@bhtCx9L?J-v+gk z82e|?FT?2iD=UQ!cgE&h`Lk0|nphG4*H|e9W%%hvWIjf2CEW{yJT1>HADa|&&CI0Q zuD(7P{rQtM7Ibt51A5_+aXR*gy#X-4k_}x;Ah}HTI=<>!j#jN?JfT(Yp*NMZ zl`aHHU4#wtN!kR-Id3y^>N8z4Do1RoN*E8?hNqN%tk}$x>~*VSKfod;2`a zBUlGztMrbiN@9*ieD|9-JX8`aC^}Y@C|*CVnuVC5W;x!Dh@gXV6yUw`Q8ZYYikob@&d>X2IVAY z2_HVZggNrRPMBu#TWE+Omtgo~=Q`z)$Fu8x;AMqRJ>2r*eO`KOEV_o09wCm_YJbPU%cJz86m{%q70`smNob07qZIk%<85s7_^rM12fNa3_RAO8=yp)@800q)Kn4 z^^}++g>b(i?QxnJ6k?Q^V-K^+{!Z$ z{DmMyL@n5H{R|7Qa4%1;t%aZ*-oX2b5>#+nxPMV)!iOsuDVSXEY8lp81Nr$=&2qmB=8layb&SP|Hn6-yCk9rUru#sjhc*P7&dL8xMhLY z0a%h5+4gGdi`~ZuYD4n;dw>5rsOhMGX)cX(!iYrr8y`F<=gGTcj&yzLc+@(LSO-3PPtc)iSpm^;C-q|(8zh9}L#3zR z=gdJx0FaQ&TvjXL2;kkQTRH(EYWj_TRuoON30HTz?a0<0;9W5_?SiCTzAF|tuA!9v zp!)(41`M`dHoc{o(K$P$VACw~sXd-$w>(srkeIXMqY7?bJp969e_1Tz=dq1!)>Hk} z;o|-69tGR|4L$H%W8mZ@ArjpthGx+x=yLl`dv$e!ck_*#H)nE7$9C$ORIm{!1VpvFH#aAr9?iUA`CKl`I4_aVC6F$@Hsgb7{ zhj#L@;H>Tul;HprUE?psDG2wpiOe$Npxiw8yQ9zj>87#h63^h$`uf}Ws(XLbmy(Bn z{<8S>x|)}<>dqQ>oC!l|Lac(+^$!n;DJcUplDw#3@$N$-ADXLCIt@lU52U1)k! zeUPnrFC(M$o*gpOz^&hh`8Mt1*+#(KbBl3&EWc=8x3&24(KnCN?PB$){=bQ0sg