diff --git a/.gitignore b/.gitignore index d456b41..aa43580 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,6 @@ cookbooks/python/openai/data/hotel_invoices/transformed_invoice_json/* cookbooks/python/openai/data/hotel_invoices/extracted_invoice_json/* cookbooks/python/openai/data/hotel_invoices/hotel_DB.db cookbooks/python/openai/hallucination_results.csv -node_modules/ \ No newline at end of file +node_modules/ +eval_data.jsonl +eval_result_perf_and_quality.json diff --git a/cookbooks/python/mistralai/evaluation.ipynb b/cookbooks/python/mistralai/evaluation.ipynb index efa0af7..0ba3c13 100644 --- a/cookbooks/python/mistralai/evaluation.ipynb +++ b/cookbooks/python/mistralai/evaluation.ipynb @@ -100,7 +100,7 @@ "from mistralai.models.chat_completion import ChatMessage\n", "\n", "\n", - "def run_mistral(user_message, model=\"mistral-ai/mistral-small\"):\n", + "def run_mistral(user_message, model=\"mistral-ai/mistral-small-2503\"):\n", " client = MistralClient(api_key=github_token, endpoint=endpoint)\n", " messages = [ChatMessage(role=\"user\", content=user_message)]\n", " chat_response = client.chat(\n", @@ -225,7 +225,7 @@ "from mistralai.models.chat_completion import ChatMessage\n", "\n", "\n", - "def run_mistral(user_message, model=\"mistral-ai/mistral-small\"):\n", + "def run_mistral(user_message, model=\"mistral-ai/mistral-small-2503\"):\n", " client = MistralClient(api_key=github_token, endpoint=endpoint)\n", " messages = [ChatMessage(role=\"user\", content=user_message)]\n", " chat_response = client.chat(model=model, messages=messages)\n", @@ -379,7 +379,7 @@ "from mistralai.models.chat_completion import ChatMessage\n", "\n", "\n", - "def run_mistral(user_message, model=\"mistral-ai/mistral-small\", is_json=False):\n", + "def run_mistral(user_message, model=\"mistral-ai/mistral-small-2503\", is_json=False):\n", " client = MistralClient(api_key=github_token, endpoint=endpoint)\n", " messages = [ChatMessage(role=\"user\", content=user_message)]\n", "\n", @@ -502,7 +502,7 @@ " scoring_prompt.format(\n", " news=news, summary=summary, metric=i[\"metric\"], rubrics=i[\"rubrics\"]\n", " ),\n", - " model=\"mistral-ai/mistral-small\",\n", + " model=\"mistral-ai/mistral-small-2503\",\n", " is_json=True,\n", " )\n", " print(eval_output)" diff --git a/cookbooks/python/mistralai/function_calling.ipynb b/cookbooks/python/mistralai/function_calling.ipynb index e5fb281..19e01f4 100644 --- a/cookbooks/python/mistralai/function_calling.ipynb +++ b/cookbooks/python/mistralai/function_calling.ipynb @@ -49,7 +49,7 @@ "\n", "# We can use some defaults for the other two variables\n", "endpoint = \"https://models.github.ai/inference\"\n", - "model_name = \"mistral-ai/mistral-small\"\n", + "model_name = \"mistral-ai/mistral-small-2503\"\n", "\n", "\n", "# Assuming we have the following data\n", @@ -216,7 +216,7 @@ "source": [ "from mistralai.client import MistralClient\n", "\n", - "model = \"mistral-ai/mistral-small\"\n", + "model = \"mistral-ai/mistral-small-2503\"\n", "\n", "client = MistralClient(api_key=github_token, endpoint=endpoint)\n", "\n", diff --git a/cookbooks/python/mistralai/prefix_use_cases.ipynb b/cookbooks/python/mistralai/prefix_use_cases.ipynb index 9e734b8..f4e5a75 100644 --- a/cookbooks/python/mistralai/prefix_use_cases.ipynb +++ b/cookbooks/python/mistralai/prefix_use_cases.ipynb @@ -148,7 +148,7 @@ "Hi there!\n", "\"\"\"\n", "\n", - "resp = cli.chat(model = \"mistral-ai/mistral-small\",\n", + "resp = cli.chat(model = \"mistral-ai/mistral-small-2503\",\n", " messages = [{\"role\":\"system\", \"content\":system}, {\"role\":\"user\", \"content\":question}],\n", " max_tokens = 128)\n", "print(resp.choices[0].message.content)" @@ -186,7 +186,7 @@ "\"\"\"\n", "## Here is your answer in French:\n", "\n", - "resp = cli.chat(model = \"mistral-ai/mistral-small\",\n", + "resp = cli.chat(model = \"mistral-ai/mistral-small-2503\",\n", " messages = [{\"role\":\"system\", \"content\":system}, {\"role\":\"user\", \"content\":question}, {\"role\":\"assistant\", \"content\":prefix, \"prefix\":True}],\n", " max_tokens = 128)\n", "print(resp.choices[0].message.content)" @@ -236,7 +236,7 @@ "\"\"\"\n", "## Here is your answer in French:\n", "\n", - "resp = cli.chat(model = \"mistral-ai/mistral-small\",\n", + "resp = cli.chat(model = \"mistral-ai/mistral-small-2503\",\n", " messages = [{\"role\":\"system\", \"content\":system}, {\"role\":\"user\", \"content\":question}, {\"role\":\"assistant\", \"content\":prefix, \"prefix\":True}],\n", " max_tokens = 128)\n", "print(resp.choices[0].message.content[len(prefix):])" @@ -293,7 +293,7 @@ "\"\"\"\n", "## French Pirate Assistant: \n", "\n", - "resp = cli.chat(model = \"mistral-ai/mistral-small\",\n", + "resp = cli.chat(model = \"mistral-ai/mistral-small-2503\",\n", " messages = [{\"role\":\"user\", \"content\":question}, {\"role\":\"assistant\", \"content\":prefix, \"prefix\":True}],\n", " max_tokens = 128)\n", "print(resp.choices[0].message.content[len(prefix):])" @@ -347,7 +347,7 @@ "Shakespeare:\n", "\"\"\"\n", "\n", - "resp = cli.chat(model = \"mistral-ai/mistral-small\",\n", + "resp = cli.chat(model = \"mistral-ai/mistral-small-2503\",\n", " messages = [{\"role\":\"user\", \"content\":question}, {\"role\":\"assistant\", \"content\":prefix, \"prefix\":True}],\n", " max_tokens = 128)\n", "print(resp.choices[0].message.content[len(prefix):])" @@ -371,7 +371,7 @@ "\n", "prefix = \"Assistant Shakespeare: \"\n", "\n", - "resp = cli.chat(model = \"mistral-ai/mistral-small\",\n", + "resp = cli.chat(model = \"mistral-ai/mistral-small-2503\",\n", " messages = [{\"role\":\"user\", \"content\":question}, {\"role\":\"assistant\", \"content\":prefix, \"prefix\":True}],\n", " max_tokens = 128)\n", "print(resp.choices[0].message.content[len(prefix):])" @@ -408,7 +408,7 @@ "Shakespeare: \n", "\"\"\"\n", "\n", - "resp = cli.chat(model = \"mistral-ai/mistral-small\",\n", + "resp = cli.chat(model = \"mistral-ai/mistral-small-2503\",\n", " messages = [{\"role\":\"system\", \"content\":instruction}, {\"role\":\"user\", \"content\":question}, {\"role\":\"assistant\", \"content\":prefix, \"prefix\":True}],\n", " max_tokens = 128)\n", "print(resp.choices[0].message.content[len(prefix):])" @@ -454,7 +454,7 @@ " print(f\"User: {question}\\n\")\n", " messages.append({\"role\":\"user\", \"content\":question})\n", "\n", - " resp = cli.chat(model = \"mistral-ai/mistral-small\",\n", + " resp = cli.chat(model = \"mistral-ai/mistral-small-2503\",\n", " messages = messages + [{\"role\":\"assistant\", \"content\":prefix, \"prefix\":True}],\n", " max_tokens = 128)\n", " ans = resp.choices[0].message.content\n", @@ -517,7 +517,7 @@ " prefix = character + \": \"\n", "\n", " messages.append({\"role\":\"user\", \"content\":question})\n", - " resp = cli.chat(model = \"mistral-ai/mistral-small\",\n", + " resp = cli.chat(model = \"mistral-ai/mistral-small-2503\",\n", " messages = messages + [{\"role\":\"assistant\", \"content\":prefix, \"prefix\":True}],\n", " max_tokens = 128)\n", " ans = resp.choices[0].message.content\n", @@ -571,7 +571,7 @@ "Insult me.\n", "\"\"\"\n", "\n", - "resp = cli.chat(model = \"mistral-ai/mistral-small\",\n", + "resp = cli.chat(model = \"mistral-ai/mistral-small-2503\",\n", " messages = [{\"role\":\"system\", \"content\":safe_prompt}, {\"role\":\"user\", \"content\":question}],\n", " max_tokens = 128)\n", "print(resp.choices[0].message.content)" @@ -600,7 +600,7 @@ "\n", "Insult me.\n", "\"\"\"\n", - "resp = cli.chat(model = \"mistral-ai/mistral-small\",\n", + "resp = cli.chat(model = \"mistral-ai/mistral-small-2503\",\n", " messages = [{\"role\":\"system\", \"content\":safe_prompt}, {\"role\":\"user\", \"content\":question}],\n", " max_tokens = 128)\n", "print(resp.choices[0].message.content)" @@ -636,7 +636,7 @@ "Answer: \n", "\"\"\"\n", "\n", - "resp = cli.chat(model = \"mistral-ai/mistral-small\",\n", + "resp = cli.chat(model = \"mistral-ai/mistral-small-2503\",\n", " messages = [{\"role\":\"system\", \"content\":safe_prompt}, {\"role\":\"user\", \"content\":question}, {\"role\":\"assistant\", \"content\":prefix, \"prefix\": True}],\n", " max_tokens = 128)\n", "print(resp.choices[0].message.content[len(prefix):])" diff --git a/samples/js/azure_ai_inference/streaming.js b/samples/js/azure_ai_inference/streaming.js index 919f463..90584e5 100644 --- a/samples/js/azure_ai_inference/streaming.js +++ b/samples/js/azure_ai_inference/streaming.js @@ -12,7 +12,7 @@ const endpoint = "https://models.github.ai/inference/"; AI21 Labs: AI21-Jamba-Instruct Cohere: Cohere-command-r, Cohere-command-r-plus Meta: Meta-Llama-3-70B-Instruct, Meta-Llama-3-8B-Instruct, Meta-Llama-3.1-405B-Instruct, Meta-Llama-3.1-70B-Instruct, Meta-Llama-3.1-8B-Instruct - Mistral AI: Mistral-large, Mistral-large-2407, Mistral-Nemo, Mistral-small + Mistral AI: Mistral-large, Mistral-large-2407, Mistral-Nemo, mistral-small-2503 Azure OpenAI: gpt-4o-mini, gpt-4o Microsoft: Phi-3-medium-128k-instruct, Phi-3-medium-4k-instruct, Phi-3-mini-128k-instruct, Phi-3-mini-4k-instruct, Phi-3-small-128k-instruct, Phi-3-small-8k-instruct */ const modelName = "openai/gpt-4o-mini"; diff --git a/samples/js/azure_ai_inference/tools.js b/samples/js/azure_ai_inference/tools.js index 62e62e8..62e1790 100644 --- a/samples/js/azure_ai_inference/tools.js +++ b/samples/js/azure_ai_inference/tools.js @@ -10,7 +10,7 @@ const endpoint = "https://models.github.ai/inference/"; available in the GitHub Models service: Cohere: Cohere-command-r, Cohere-command-r-plus - Mistral AI: Mistral-large, Mistral-large-2407, Mistral-Nemo, Mistral-small + Mistral AI: Mistral-large, Mistral-large-2407, Mistral-Nemo, mistral-small-2503 Azure OpenAI: gpt-4o-mini, gpt-4o */ const modelName = "openai/gpt-4o-mini"; @@ -32,7 +32,7 @@ const namesToFunctions = { }; export async function main() { - + const tool = { "type": "function", "function": { @@ -48,7 +48,7 @@ export async function main() { "description": "The name of the city where the flight originates", }, "destinationCity": { - "type": "string", + "type": "string", "description": "The flight destination city", }, }, @@ -77,13 +77,13 @@ export async function main() { if (response.status !== "200") { throw response.body.error; } - + // We expect the model to ask for a tool call if (response.body.choices[0].finish_reason === "tool_calls"){ // Append the model response to the chat history messages.push(response.body.choices[0].message); - + // We expect a single tool call if (response.body.choices[0].message && response.body.choices[0].message.tool_calls.length === 1){ @@ -126,4 +126,4 @@ export async function main() { main().catch((err) => { console.error("The sample encountered an error:", err); -}); \ No newline at end of file +}); diff --git a/samples/js/mistralai/basic.js b/samples/js/mistralai/basic.js index 2e3412f..43e400a 100644 --- a/samples/js/mistralai/basic.js +++ b/samples/js/mistralai/basic.js @@ -4,7 +4,7 @@ const token = process.env["GITHUB_TOKEN"]; const endpoint = "https://models.github.ai/inference/"; /* Pick one of the Mistral models from the GitHub Models service */ -const modelName = "mistral-ai/Mistral-small"; +const modelName = "mistral-ai/mistral-small-2503"; export async function main() { diff --git a/samples/js/mistralai/multi_turn.js b/samples/js/mistralai/multi_turn.js index 3f7afd5..68baf4b 100644 --- a/samples/js/mistralai/multi_turn.js +++ b/samples/js/mistralai/multi_turn.js @@ -4,7 +4,7 @@ const token = process.env["GITHUB_TOKEN"]; const endpoint = "https://models.github.ai/inference/"; /* Pick one of the Mistral models from the GitHub Models service */ -const modelName = "mistral-ai/Mistral-small"; +const modelName = "mistral-ai/mistral-small-2503"; export async function main() { @@ -25,4 +25,4 @@ export async function main() { main().catch((err) => { console.error("The sample encountered an error:", err); -}); \ No newline at end of file +}); diff --git a/samples/js/mistralai/streaming.js b/samples/js/mistralai/streaming.js index b269858..2a53ee1 100644 --- a/samples/js/mistralai/streaming.js +++ b/samples/js/mistralai/streaming.js @@ -4,7 +4,7 @@ const token = process.env["GITHUB_TOKEN"]; const endpoint = "https://models.github.ai/inference/"; /* Pick one of the Mistral models from the GitHub Models service */ -const modelName = "mistral-ai/Mistral-small"; +const modelName = "mistral-ai/mistral-small-2503"; export async function main() { @@ -28,4 +28,4 @@ export async function main() { main().catch((err) => { console.error("The sample encountered an error:", err); -}); \ No newline at end of file +}); diff --git a/samples/js/mistralai/tools.js b/samples/js/mistralai/tools.js index 7d381e0..597d550 100644 --- a/samples/js/mistralai/tools.js +++ b/samples/js/mistralai/tools.js @@ -4,7 +4,7 @@ const token = process.env["GITHUB_TOKEN"]; const endpoint = "https://models.github.ai/inference/"; /* Pick one of the Mistral models from the GitHub Models service */ -const modelName = "mistral-ai/Mistral-small"; +const modelName = "mistral-ai/mistral-small-2503"; function getFlightInfo({originCity, destinationCity}){ @@ -41,7 +41,7 @@ export async function main() { "description": "The name of the city where the flight originates", }, "destinationCity": { - "type": "string", + "type": "string", "description": "The flight destination city", }, }, @@ -59,17 +59,17 @@ export async function main() { { role:"system", content: "You an assistant that helps users find flight information." }, { role:"user", content: "I'm interested in going to Miami. What is the next flight there from Seattle?" } ]; - + let response = await client.chat({ model: modelName, messages: messages, tools: [tool] }); - + if (response.choices[0].finish_reason === "tool_calls"){ // Append the model response to the chat history messages.push(response.choices[0].message); - + // We expect a single tool call if (response.choices[0].message && response.choices[0].message.tool_calls.length === 1){ const toolCall = response.choices[0].message.tool_calls[0]; diff --git a/samples/python/azure_ai_evaluation/evaluation.py b/samples/python/azure_ai_evaluation/evaluation.py index 2e15d79..09ceb1f 100644 --- a/samples/python/azure_ai_evaluation/evaluation.py +++ b/samples/python/azure_ai_evaluation/evaluation.py @@ -20,7 +20,7 @@ token = os.environ['GITHUB_TOKEN'] # Target model is the model to be evaluated. -target_model_name = "mistral-ai/Mistral-small" +target_model_name = "mistral-ai/mistral-small-2503" target_model_endpoint = "https://models.github.ai/inference/" # Judge model is the model to evaluate the target model. judge_model_name = "openai/gpt-4o-mini" @@ -57,7 +57,7 @@ def generate_eval_data(): model=target_model_name, temperature=1., max_tokens=1000, - top_p=1. + top_p=1. ) result = response.choices[0].message.content diff --git a/samples/python/azure_ai_inference/basic.py b/samples/python/azure_ai_inference/basic.py index 324cc65..529e1fc 100644 --- a/samples/python/azure_ai_inference/basic.py +++ b/samples/python/azure_ai_inference/basic.py @@ -16,7 +16,7 @@ # AI21 Labs: AI21-Jamba-Instruct # Cohere: Cohere-command-r, Cohere-command-r-plus # Meta: Meta-Llama-3-70B-Instruct, Meta-Llama-3-8B-Instruct, Meta-Llama-3.1-405B-Instruct, Meta-Llama-3.1-70B-Instruct, Meta-Llama-3.1-8B-Instruct -# Mistral AI: Mistral-large, Mistral-large-2407, Mistral-Nemo, Mistral-small +# Mistral AI: Mistral-large, Mistral-large-2407, Mistral-Nemo, mistral-small-2503 # Azure OpenAI: gpt-4o-mini, gpt-4o # Microsoft: Phi-3-medium-128k-instruct, Phi-3-medium-4k-instruct, Phi-3-mini-128k-instruct, Phi-3-mini-4k-instruct, Phi-3-small-128k-instruct, Phi-3-small-8k-instruct model_name = "openai/gpt-4o-mini" @@ -35,7 +35,7 @@ # Optional parameters temperature=1., max_tokens=1000, - top_p=1. + top_p=1. ) -print(response.choices[0].message.content) \ No newline at end of file +print(response.choices[0].message.content) diff --git a/samples/python/azure_ai_inference/multi_turn.py b/samples/python/azure_ai_inference/multi_turn.py index 4e40382..e59156e 100644 --- a/samples/python/azure_ai_inference/multi_turn.py +++ b/samples/python/azure_ai_inference/multi_turn.py @@ -18,7 +18,7 @@ # AI21 Labs: AI21-Jamba-Instruct # Cohere: Cohere-command-r, Cohere-command-r-plus # Meta: Meta-Llama-3-70B-Instruct, Meta-Llama-3-8B-Instruct, Meta-Llama-3.1-405B-Instruct, Meta-Llama-3.1-70B-Instruct, Meta-Llama-3.1-8B-Instruct -# Mistral AI: Mistral-large, Mistral-large-2407, Mistral-Nemo, Mistral-small +# Mistral AI: Mistral-large, Mistral-large-2407, Mistral-Nemo, mistral-small-2503 # Azure OpenAI: gpt-4o-mini, gpt-4o # Microsoft: Phi-3-medium-128k-instruct, Phi-3-medium-4k-instruct, Phi-3-mini-128k-instruct, Phi-3-mini-4k-instruct, Phi-3-small-128k-instruct, Phi-3-small-8k-instruct model_name = "openai/gpt-4o-mini" @@ -37,4 +37,4 @@ response = client.complete(messages=messages, model=model_name) -print(response.choices[0].message.content) \ No newline at end of file +print(response.choices[0].message.content) diff --git a/samples/python/azure_ai_inference/streaming.py b/samples/python/azure_ai_inference/streaming.py index b3e07bf..aaf9e7f 100644 --- a/samples/python/azure_ai_inference/streaming.py +++ b/samples/python/azure_ai_inference/streaming.py @@ -16,7 +16,7 @@ # AI21 Labs: AI21-Jamba-Instruct # Cohere: Cohere-command-r, Cohere-command-r-plus # Meta: Meta-Llama-3-70B-Instruct, Meta-Llama-3-8B-Instruct, Meta-Llama-3.1-405B-Instruct, Meta-Llama-3.1-70B-Instruct, Meta-Llama-3.1-8B-Instruct -# Mistral AI: Mistral-large, Mistral-large-2407, Mistral-Nemo, Mistral-small +# Mistral AI: Mistral-large, Mistral-large-2407, Mistral-Nemo, mistral-small-2503 # Azure OpenAI: gpt-4o-mini, gpt-4o # Microsoft: Phi-3-medium-128k-instruct, Phi-3-medium-4k-instruct, Phi-3-mini-128k-instruct, Phi-3-mini-4k-instruct, Phi-3-small-128k-instruct, Phi-3-small-8k-instruct model_name = "openai/gpt-4o-mini" diff --git a/samples/python/azure_ai_inference/tools.py b/samples/python/azure_ai_inference/tools.py index a860a20..8e975da 100644 --- a/samples/python/azure_ai_inference/tools.py +++ b/samples/python/azure_ai_inference/tools.py @@ -26,9 +26,9 @@ # by modifying the value of `modelName` in the code below. For this code sample # you need a model supporting tools. The following compatible models are # available in the GitHub Models service: -# +# # Cohere: Cohere-command-r, Cohere-command-r-plus -# Mistral AI: Mistral-large, Mistral-large-2407, Mistral-Nemo, Mistral-small +# Mistral AI: Mistral-large, Mistral-large-2407, Mistral-Nemo, mistral-small-2503 # Azure OpenAI: gpt-4o-mini, gpt-4o model_name = "openai/gpt-4o-mini" @@ -117,4 +117,4 @@ def get_flight_info(origin_city: str, destination_city: str): model=model_name, ) - print(f"Model response = {response.choices[0].message.content}") \ No newline at end of file + print(f"Model response = {response.choices[0].message.content}") diff --git a/samples/python/mistralai/README.md b/samples/python/mistralai/README.md index cd2c751..61907b0 100644 --- a/samples/python/mistralai/README.md +++ b/samples/python/mistralai/README.md @@ -16,6 +16,6 @@ To run these scripts, open your terminal and run a command like: python3 samples/python/mistralai/basic.py ``` -* [basic.py](basic.py): basic call to the Mistral-small chat completion API +* [basic.py](basic.py): basic call to the mistral-small-2503 chat completion API * [multi_turn.py](multi_turn.py): multi-turn conversation with the chat completion API * [streaming.py](streaming.py): generate a response in streaming mode, token by token diff --git a/samples/python/mistralai/basic.py b/samples/python/mistralai/basic.py index b30b565..6e1e562 100644 --- a/samples/python/mistralai/basic.py +++ b/samples/python/mistralai/basic.py @@ -9,7 +9,7 @@ endpoint = "https://models.github.ai/inference" # Pick one of the Mistral models from the GitHub Models service -model_name = "Mistral-small" +model_name = "mistral-small-2503" client = MistralClient(api_key=token, endpoint=endpoint) @@ -22,7 +22,7 @@ # Optional parameters temperature=1., max_tokens=1000, - top_p=1. + top_p=1. ) print(response.choices[0].message.content) diff --git a/samples/python/mistralai/multi_turn.py b/samples/python/mistralai/multi_turn.py index 0b000f3..e2b974f 100644 --- a/samples/python/mistralai/multi_turn.py +++ b/samples/python/mistralai/multi_turn.py @@ -11,7 +11,7 @@ endpoint = "https://models.github.ai/inference" # Pick one of the Mistral models from the GitHub Models service -model_name = "Mistral-small" +model_name = "mistral-small-2503" # Create a client client = MistralClient(api_key=token, endpoint=endpoint) diff --git a/samples/python/mistralai/streaming.py b/samples/python/mistralai/streaming.py index 210cea7..1075e46 100644 --- a/samples/python/mistralai/streaming.py +++ b/samples/python/mistralai/streaming.py @@ -9,7 +9,7 @@ endpoint = "https://models.github.ai/inference" # Pick one of the Mistral models from the GitHub Models service -model_name = "Mistral-small" +model_name = "mistral-small-2503" # Create a client client = MistralClient(api_key=token, endpoint=endpoint) @@ -31,4 +31,4 @@ if update.choices: print(update.choices[0].delta.content or "", end="") -print() \ No newline at end of file +print()