Update Quantization with related to #15

Author: damian1996

pyproject.toml

@@ -7,6 +7,7 @@ requires-python = ">=3.12"
 dependencies = [
     "crfm-helm>=0.5.6",
     "dacite>=1.9.2",
+    "datamodel-code-generator>=0.31.2",
     "numpy>=2.3.1",
     "openai>=1.93.0",
     "pandas>=2.3.0",

schema/README.md

@@ -7,7 +7,13 @@ To generate or update Python types from the JSON schema, you can run the followi
 ```bash
 datamodel-codegen --input eval.schema.json --output eval_types.py --class-name EvaluationResult --output-model-type pydantic_v2.BaseModel
 ```
+or 
+```bash
+uv run datamodel-codegen --input schema/eval.schema.json --input-file-type jsonschema --output schema/eval_types.py --class-name EvaluationResult --output-model-type pydantic_v2.BaseModel
+```
+from main directory
 
 ## Example Data
 
 Please refer to [this file](./eval.example.json) for a minimal data example adhering to the evaluation format schema.
+uv run datamodel-codegen --input schema/eval.schema.json --input-file-type jsonschema --output schema/eval_types.py --class-name EvaluationResult --output-model-type pydantic_v2.BaseModel

schema/eval.schema.json

@@ -126,7 +126,8 @@
                             "type": "object",
                             "required": [
                                 "bit_precision",
-                                "method"
+                                "method",
+                                "type"
                             ],
                             "properties": {
                                 "bit_precision": {
@@ -140,13 +141,29 @@
                                     ],
                                     "description": "Quantization bit precision"
                                 },
-                                "method": {
+                                "type": {
                                     "type": "string",
                                     "enum": [
                                         "None",
                                         "dynamic",
                                         "static"
                                     ],
+                                    "description": "Quantization type"
+                                },
+                                "method": {
+                                    "type": "string",
+                                    "enum": [
+                                        "None",
+                                        "AWQ",
+                                        "PTQ",
+                                        "GPTQ",
+                                        "GGUF",
+                                        "QAT",
+                                        "BitsAndBytes",
+                                        "SmoothQuant",
+                                        "HQQ",
+                                        "Quanto"
+                                    ],
                                     "description": "Quantization method"
                                 }
                             }

schema/eval_types.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  eval.schema.json
-#   timestamp: 2025-07-06T13:59:25+00:00
+#   timestamp: 2025-09-04T13:29:15+00:00
 
 from __future__ import annotations
 
@@ -62,14 +62,28 @@ class BitPrecision(Enum):
     float32 = 'float32'
 
 
-class Method(Enum):
+class Type(Enum):
     None_ = 'None'
     dynamic = 'dynamic'
     static = 'static'
 
 
+class Method(Enum):
+    None_ = 'None'
+    AWQ = 'AWQ'
+    PTQ = 'PTQ'
+    GPTQ = 'GPTQ'
+    GGUF = 'GGUF'
+    QAT = 'QAT'
+    BitsAndBytes = 'BitsAndBytes'
+    SmoothQuant = 'SmoothQuant'
+    HQQ = 'HQQ'
+    Quanto = 'Quanto'
+
+
 class Quantization(BaseModel):
     bit_precision: BitPrecision = Field(..., description='Quantization bit precision')
+    type: Type = Field(..., description='Quantization type')
     method: Method = Field(..., description='Quantization method')