【feature】support n parameter #4273
Conversation
|
Thanks for your contribution! |
|
|
| request["prompt_token_ids_len"] = len(request["prompt_token_ids"]) | ||
| input_ids_len = request["prompt_token_ids_len"] | ||
| request["max_tokens"] = min(self.max_model_len - input_ids_len, request.get("max_tokens")) | ||
| if request.get("reasoning_max_tokens", None) is None: |
There was a problem hiding this comment.
这里reasoning_max_tokens的逻辑去掉吧
| chunk_object_type: str = "chat.completion.chunk" | ||
| first_iteration = True | ||
| previous_num_tokens = 0 | ||
| n_param = request.n if request.n is not None else 1 |
There was a problem hiding this comment.
直接用num_choices就可以，不用再加一个n_param，
| first_iteration[idx] = False | ||
|
|
||
| output = res["outputs"] | ||
| reasoning_content = output["reasoning_content"] |
|
|
||
| delta_message = DeltaMessage( | ||
| reasoning_content="", | ||
| reasoning_content=reasoning_content, |
There was a problem hiding this comment.
这行也改成reasoning_content=""
| prompt_tokens_details=PromptTokenUsageInfo(cached_tokens=final_res.get("num_cached_tokens", 0)), | ||
| prompt_tokens_details=PromptTokenUsageInfo(cached_tokens=sum(num_cached_tokens)), | ||
| ) | ||
| work_process_metrics.e2e_request_latency.observe(time.time() - final_res["metrics"]["request_start_time"]) |
| request_prompts = request_prompt_ids | ||
|
|
||
| num_choices = len(request_prompts) | ||
| num_choices = len(request_prompts) * request.n |
There was a problem hiding this comment.
request.n需要判空，下面的n_param = current_req_dict.get("n", 1)可以挪到上面来，用n_param去乘
| try: | ||
| for idx, prompt in enumerate(request_prompts): | ||
| request_id_idx = f"{request_id}-{idx}" | ||
| request_id_idx = f"{request_id}" |
| for idx, prompt in enumerate(request_prompts): | ||
| request_id_idx = f"{request_id}-{idx}" | ||
| request_id_idx = f"{request_id}" | ||
| current_req_dict = request.to_dict_for_infer(request_id_idx, prompt) |
| else: | ||
| arrival_time = res["metrics"]["arrival_time"] - inference_start_time | ||
| if first_iteration: | ||
| if first_iteration[idx]: |
There was a problem hiding this comment.
下面已经是 for i in range(num_choices)了， first_iteration 不需要定义成 数组了。
| ) | ||
| has_no_token_limit = request.max_tokens is None and request.max_completion_tokens is None | ||
| max_tokens = request.max_completion_tokens or request.max_tokens | ||
| # output 可能没有num_cached_tokens字段 |
| if logprobs_res and logprobs_res.content is not None: | ||
| logprob_contents.extend(logprobs_res.content) | ||
| logprob_contents[idx].extend(logprobs_res.content) | ||
| if data["finished"]: |
There was a problem hiding this comment.
这个 if 中的代码，建议单独提一个方法，返回一个 choice, append 到 choices 中
|
|
||
| num_prompt_tokens += len(prompt_token_ids) | ||
|
|
||
| num_prompt_tokens = num_prompt_tokens // request.n |
There was a problem hiding this comment.
1 if request.n is None else request.n
5 participants
Support adding the parameter
nto the request to retrieve multiple model responses.