From 3857e22efc2698205f248139d99aa3637acd61a4 Mon Sep 17 00:00:00 2001 From: Andrew Wan Date: Sat, 7 Dec 2024 14:35:52 -0800 Subject: [PATCH 01/40] added pdf context support --- .../serve/gradio_block_arena_vision_anony.py | 90 +++++++++++++++++-- 1 file changed, 85 insertions(+), 5 deletions(-) diff --git a/fastchat/serve/gradio_block_arena_vision_anony.py b/fastchat/serve/gradio_block_arena_vision_anony.py index d4d4d484e..f1df03b17 100644 --- a/fastchat/serve/gradio_block_arena_vision_anony.py +++ b/fastchat/serve/gradio_block_arena_vision_anony.py @@ -4,12 +4,16 @@ """ import json +import subprocess import time import gradio as gr import numpy as np from typing import Union +import os +import PyPDF2 + from fastchat.constants import ( TEXT_MODERATION_MSG, IMAGE_MODERATION_MSG, @@ -242,6 +246,56 @@ def clear_history(request: gr.Request): + [""] ) +def extract_text_from_pdf(pdf_file_path): + """Extract text from a PDF file.""" + try: + with open(pdf_file_path, 'rb') as f: + reader = PyPDF2.PdfReader(f) + pdf_text = "" + for page in reader.pages: + pdf_text += page.extract_text() + return pdf_text + except Exception as e: + logger.error(f"Failed to extract text from PDF: {e}") + return None + +def llama_parse(pdf_path): + os.environ['LLAMA_CLOUD_API_KEY'] = 'LLAMA KEY' + + output_dir = 'outputs' + os.makedirs(output_dir, exist_ok=True) + + pdf_name = os.path.splitext(os.path.basename(pdf_path))[0] + markdown_file_path = os.path.join(output_dir, f'{pdf_name}.md') + + command = [ + 'llama-parse', + pdf_path, + '--result-type', 'markdown', + '--output-file', markdown_file_path + ] + + subprocess.run(command, check=True) + + with open(markdown_file_path, 'r', encoding='utf-8') as file: + markdown_content = file.read() + + return markdown_content + +def wrap_query_context(user_query, query_context): + #TODO: refactor to split up user query and query context. + # lines = input.split("\n\n[USER QUERY]", 1) + # user_query = lines[1].strip() + # query_context = lines[0][len('[QUERY CONTEXT]\n\n'): ] + reformatted_query_context = ( + f"[QUERY CONTEXT]\n" + f"
\n" + f"Expand context details\n\n" + f"{query_context}\n\n" + f"
" + ) + markdown = reformatted_query_context + f"\n\n[USER QUERY]\n\n{user_query}" + return markdown def add_text( state0, @@ -253,10 +307,14 @@ def add_text( request: gr.Request, ): if isinstance(chat_input, dict): - text, images = chat_input["text"], chat_input["files"] + text, files = chat_input["text"], chat_input["files"] else: text = chat_input - images = [] + files = [] + + images = [] + + file_extension = os.path.splitext(files[0])[1].lower() ip = get_ip(request) logger.info(f"add_text (anony). ip: {ip}. len: {len(text)}") @@ -267,7 +325,7 @@ def add_text( if states[0] is None: assert states[1] is None - if len(images) > 0: + if len(files) > 0 and file_extension != ".pdf": model_left, model_right = get_battle_pair( context.all_vision_models, VISION_BATTLE_TARGETS, @@ -363,6 +421,27 @@ def add_text( for i in range(num_sides): if "deluxe" in states[i].model_name: hint_msg = SLOW_MODEL_MSG + + if file_extension == ".pdf": + document_text = llama_parse(files[0]) + post_processed_text = f""" + The following is the content of a document: + + {document_text} + + Based on this document, answer the following question: + + {text} + """ + + post_processed_text = wrap_query_context(text, post_processed_text) + + text = text[:BLIND_MODE_INPUT_CHAR_LEN_LIMIT] # Hard cut-off + for i in range(num_sides): + states[i].conv.append_message(states[i].conv.roles[0], post_processed_text) + states[i].conv.append_message(states[i].conv.roles[1], None) + states[i].skip_next = False + return ( states + [x.to_gradio_chatbot() for x in states] @@ -471,10 +550,10 @@ def build_side_by_side_vision_ui_anony(context: Context, random_questions=None): ) multimodal_textbox = gr.MultimodalTextbox( - file_types=["image"], + file_types=["file"], show_label=False, container=True, - placeholder="Enter your prompt or add image here", + placeholder="Enter your prompt or add a PDF file here", elem_id="input_box", scale=3, ) @@ -483,6 +562,7 @@ def build_side_by_side_vision_ui_anony(context: Context, random_questions=None): ) with gr.Row() as button_row: + random_btn = gr.Button(value="🔮 Random Image", interactive=True) if random_questions: global vqa_samples with open(random_questions, "r") as f: From 06d056be7cdcfbc320dc4077a2f38bdc851f43c1 Mon Sep 17 00:00:00 2001 From: Andrew Wan Date: Sat, 7 Dec 2024 20:26:58 -0800 Subject: [PATCH 02/40] These changes are in response to PR comments --- .../serve/gradio_block_arena_vision_anony.py | 61 ++++++++++++------- 1 file changed, 38 insertions(+), 23 deletions(-) diff --git a/fastchat/serve/gradio_block_arena_vision_anony.py b/fastchat/serve/gradio_block_arena_vision_anony.py index f1df03b17..1ebf5acac 100644 --- a/fastchat/serve/gradio_block_arena_vision_anony.py +++ b/fastchat/serve/gradio_block_arena_vision_anony.py @@ -258,28 +258,43 @@ def extract_text_from_pdf(pdf_file_path): except Exception as e: logger.error(f"Failed to extract text from PDF: {e}") return None + +import os +import nest_asyncio +from llama_parse import LlamaParse + +nest_asyncio.apply() # Ensure compatibility with async environments + +def pdf_parse(pdf_path): + # Set API key, can also be configured in the environment + api_key = "LLAMA API" + + # Initialize the LlamaParse object + parser = LlamaParse( + api_key=api_key, + result_type="markdown", # Output in Markdown format + num_workers=4, # Number of API calls for batch processing + verbose=True, # Print detailed logs + language="en" # Set language (default is English) + ) -def llama_parse(pdf_path): - os.environ['LLAMA_CLOUD_API_KEY'] = 'LLAMA KEY' - - output_dir = 'outputs' + # Prepare the output directory and file name + output_dir = "outputs" os.makedirs(output_dir, exist_ok=True) pdf_name = os.path.splitext(os.path.basename(pdf_path))[0] - markdown_file_path = os.path.join(output_dir, f'{pdf_name}.md') + markdown_file_path = os.path.join(output_dir, f"{pdf_name}.md") - command = [ - 'llama-parse', - pdf_path, - '--result-type', 'markdown', - '--output-file', markdown_file_path - ] + # Load and parse the PDF + extra_info = {"file_name": pdf_name} - subprocess.run(command, check=True) + with open(pdf_path, "rb") as pdf_file: + # Pass the file object and extra info for parsing + documents = parser.load_data(pdf_file, extra_info=extra_info) + + # Save the parsed content to a Markdown file + markdown_content = documents[0].text if documents else "" - with open(markdown_file_path, 'r', encoding='utf-8') as file: - markdown_content = file.read() - return markdown_content def wrap_query_context(user_query, query_context): @@ -423,7 +438,7 @@ def add_text( hint_msg = SLOW_MODEL_MSG if file_extension == ".pdf": - document_text = llama_parse(files[0]) + document_text = pdf_parse(files[0]) post_processed_text = f""" The following is the content of a document: @@ -436,7 +451,7 @@ def add_text( post_processed_text = wrap_query_context(text, post_processed_text) - text = text[:BLIND_MODE_INPUT_CHAR_LEN_LIMIT] # Hard cut-off + # text = text[:BLIND_MODE_INPUT_CHAR_LEN_LIMIT] # Hard cut-off for i in range(num_sides): states[i].conv.append_message(states[i].conv.roles[0], post_processed_text) states[i].conv.append_message(states[i].conv.roles[1], None) @@ -553,7 +568,7 @@ def build_side_by_side_vision_ui_anony(context: Context, random_questions=None): file_types=["file"], show_label=False, container=True, - placeholder="Enter your prompt or add a PDF file here", + placeholder="Enter your prompt here. You can also upload image or PDF file", elem_id="input_box", scale=3, ) @@ -563,11 +578,11 @@ def build_side_by_side_vision_ui_anony(context: Context, random_questions=None): with gr.Row() as button_row: random_btn = gr.Button(value="🔮 Random Image", interactive=True) - if random_questions: - global vqa_samples - with open(random_questions, "r") as f: - vqa_samples = json.load(f) - random_btn = gr.Button(value="🔮 Random Image", interactive=True) + # if random_questions: + # global vqa_samples + # with open(random_questions, "r") as f: + # vqa_samples = json.load(f) + # random_btn = gr.Button(value="🔮 Random Image", interactive=True) clear_btn = gr.Button(value="🎲 New Round", interactive=False) regenerate_btn = gr.Button(value="🔄 Regenerate", interactive=False) share_btn = gr.Button(value="📷 Share") From cc66890ccf325b82be739bab73227425c5f95f57 Mon Sep 17 00:00:00 2001 From: Andrew Wan Date: Sat, 7 Dec 2024 20:29:38 -0800 Subject: [PATCH 03/40] These changes are in response to PR comments --- fastchat/serve/gradio_block_arena_vision_anony.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fastchat/serve/gradio_block_arena_vision_anony.py b/fastchat/serve/gradio_block_arena_vision_anony.py index 1ebf5acac..fda3070c8 100644 --- a/fastchat/serve/gradio_block_arena_vision_anony.py +++ b/fastchat/serve/gradio_block_arena_vision_anony.py @@ -423,7 +423,8 @@ def add_text( + [""] ) - text = text[:BLIND_MODE_INPUT_CHAR_LEN_LIMIT] # Hard cut-off + if file_extension != ".pdf": + text = text[:BLIND_MODE_INPUT_CHAR_LEN_LIMIT] # Hard cut-off for i in range(num_sides): post_processed_text = _prepare_text_with_image( states[i], text, images, csam_flag=csam_flag From 85767e51f808fdb308fcda0b064635e52440f1b6 Mon Sep 17 00:00:00 2001 From: Andrew Wan Date: Sun, 8 Dec 2024 15:02:32 -0800 Subject: [PATCH 04/40] Changed file detection to magic numbers and removed unnecessary libraries and code --- .../serve/gradio_block_arena_vision_anony.py | 72 +++++++++++-------- 1 file changed, 42 insertions(+), 30 deletions(-) diff --git a/fastchat/serve/gradio_block_arena_vision_anony.py b/fastchat/serve/gradio_block_arena_vision_anony.py index fda3070c8..f589a53e0 100644 --- a/fastchat/serve/gradio_block_arena_vision_anony.py +++ b/fastchat/serve/gradio_block_arena_vision_anony.py @@ -4,7 +4,6 @@ """ import json -import subprocess import time import gradio as gr @@ -13,6 +12,8 @@ import os import PyPDF2 +import nest_asyncio +from llama_parse import LlamaParse from fastchat.constants import ( TEXT_MODERATION_MSG, @@ -246,22 +247,37 @@ def clear_history(request: gr.Request): + [""] ) -def extract_text_from_pdf(pdf_file_path): - """Extract text from a PDF file.""" +def is_pdf(file_path): try: - with open(pdf_file_path, 'rb') as f: - reader = PyPDF2.PdfReader(f) - pdf_text = "" - for page in reader.pages: - pdf_text += page.extract_text() - return pdf_text + with open(file_path, 'rb') as file: + header = file.read(5) # Read the first 5 bytes + return header == b'%PDF-' except Exception as e: - logger.error(f"Failed to extract text from PDF: {e}") - return None - -import os -import nest_asyncio -from llama_parse import LlamaParse + print(f"Error: {e}") + return False + +def is_image(file_path): + magic_numbers = { + b'\xff\xd8\xff': 'JPEG', + b'\x89PNG\r\n\x1a\n': 'PNG', + b'GIF87a': 'GIF', + b'GIF89a': 'GIF', + b'BM': 'BMP', + b'\x00\x00\x01\x00': 'ICO', + b'\x49\x49\x2a\x00': 'TIFF', + b'\x4d\x4d\x00\x2a': 'TIFF', + b'RIFF': 'WebP', + } + try: + with open(file_path, 'rb') as file: + header = file.read(8) # Read the first 8 bytes + for magic in magic_numbers: + if header.startswith(magic): + return True + return False + except Exception as e: + print(f"Error reading file: {e}") + return False nest_asyncio.apply() # Ensure compatibility with async environments @@ -278,14 +294,7 @@ def pdf_parse(pdf_path): language="en" # Set language (default is English) ) - # Prepare the output directory and file name - output_dir = "outputs" - os.makedirs(output_dir, exist_ok=True) - pdf_name = os.path.splitext(os.path.basename(pdf_path))[0] - markdown_file_path = os.path.join(output_dir, f"{pdf_name}.md") - - # Load and parse the PDF extra_info = {"file_name": pdf_name} with open(pdf_path, "rb") as pdf_file: @@ -326,10 +335,14 @@ def add_text( else: text = chat_input files = [] - + images = [] - file_extension = os.path.splitext(files[0])[1].lower() + # currently support up to one pdf or one image + # if is_pdf(files[0]): + # pdfs = files + if is_image(files[0]): + images = files ip = get_ip(request) logger.info(f"add_text (anony). ip: {ip}. len: {len(text)}") @@ -340,7 +353,7 @@ def add_text( if states[0] is None: assert states[1] is None - if len(files) > 0 and file_extension != ".pdf": + if len(files) > 0 and is_image(files[0]): model_left, model_right = get_battle_pair( context.all_vision_models, VISION_BATTLE_TARGETS, @@ -423,7 +436,7 @@ def add_text( + [""] ) - if file_extension != ".pdf": + if is_image(files[0]): text = text[:BLIND_MODE_INPUT_CHAR_LEN_LIMIT] # Hard cut-off for i in range(num_sides): post_processed_text = _prepare_text_with_image( @@ -438,9 +451,9 @@ def add_text( if "deluxe" in states[i].model_name: hint_msg = SLOW_MODEL_MSG - if file_extension == ".pdf": + if is_pdf(files[0]): document_text = pdf_parse(files[0]) - post_processed_text = f""" + prompt_text = f""" The following is the content of a document: {document_text} @@ -449,8 +462,7 @@ def add_text( {text} """ - - post_processed_text = wrap_query_context(text, post_processed_text) + post_processed_text = wrap_query_context(text, prompt_text) # text = text[:BLIND_MODE_INPUT_CHAR_LEN_LIMIT] # Hard cut-off for i in range(num_sides): From c49344f7b5cfe343413c099e525d2129bf2a04c7 Mon Sep 17 00:00:00 2001 From: CodingWithTim Date: Thu, 26 Dec 2024 23:48:39 +0000 Subject: [PATCH 05/40] switch to using environment variable --- fastchat/serve/gradio_block_arena_vision_anony.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastchat/serve/gradio_block_arena_vision_anony.py b/fastchat/serve/gradio_block_arena_vision_anony.py index f589a53e0..5b05864c5 100644 --- a/fastchat/serve/gradio_block_arena_vision_anony.py +++ b/fastchat/serve/gradio_block_arena_vision_anony.py @@ -283,7 +283,7 @@ def is_image(file_path): def pdf_parse(pdf_path): # Set API key, can also be configured in the environment - api_key = "LLAMA API" + api_key = os.environ["LLAMA_PARSE_KEY"] # Initialize the LlamaParse object parser = LlamaParse( From afbf7e65f9ffd32ffcd4af29c37f926a77af84ab Mon Sep 17 00:00:00 2001 From: CodingWithTim Date: Mon, 30 Dec 2024 00:14:08 +0000 Subject: [PATCH 06/40] new architecture and bug fixes --- fastchat/serve/gradio_block_arena_vision.py | 100 +++++++++++- .../serve/gradio_block_arena_vision_anony.py | 152 ++++-------------- fastchat/serve/gradio_global_state.py | 2 + fastchat/serve/gradio_web_server.py | 9 +- fastchat/serve/gradio_web_server_multi.py | 28 +++- 5 files changed, 161 insertions(+), 130 deletions(-) diff --git a/fastchat/serve/gradio_block_arena_vision.py b/fastchat/serve/gradio_block_arena_vision.py index b3d812220..c56685902 100644 --- a/fastchat/serve/gradio_block_arena_vision.py +++ b/fastchat/serve/gradio_block_arena_vision.py @@ -74,16 +74,60 @@ def get_vqa_sample(): return (res, path) +def is_image(file_path): + magic_numbers = { + b"\xff\xd8\xff": "JPEG", + b"\x89PNG\r\n\x1a\n": "PNG", + b"GIF87a": "GIF", + b"GIF89a": "GIF", + b"BM": "BMP", + b"\x00\x00\x01\x00": "ICO", + b"\x49\x49\x2a\x00": "TIFF", + b"\x4d\x4d\x00\x2a": "TIFF", + # For WebP, the first four bytes are "RIFF", but we also check for "WEBP" + # in bytes 8–12. + } + + try: + with open(file_path, "rb") as f: + header = f.read(16) # Read a bit more to handle WebP safely + + # Check for WebP (RIFF + WEBP) + if header.startswith(b"RIFF") and header[8:12] == b"WEBP": + return True + + # Check other formats + for magic in magic_numbers: + if header.startswith(magic): + return True + + return False + except Exception as e: + print(f"Error reading file: {e}") + return False + + +def is_pdf(file_path): + try: + with open(file_path, "rb") as file: + header = file.read(5) # Read the first 5 bytes + return header == b"%PDF-" + except Exception as e: + print(f"Error: {e}") + return False + + def set_visible_image(textbox): - images = textbox["files"] - if len(images) == 0: + files = textbox["files"] + if len(files) == 0: return invisible_image_column - elif len(images) > 1: + elif len(files) > 1: gr.Warning( "We only support single image conversations. Please start a new round if you would like to chat using this image." ) - - return visible_image_column + elif is_image(files[0]): + return visible_image_column + return invisible_image_column def set_invisible_image(): @@ -166,6 +210,40 @@ def report_csam_image(state, image): pass +def wrap_pdfchat_query(query, document): + # TODO: Considering redesign the context format. + document_context = f""" + The following is the content of a document: + {document} + Based on this document, answer the following question: + {query} + """ + + reformatted_query_context = ( + f"[QUERY CONTEXT]\n" + f"
\n" + f"Expand context details\n\n" + f"{document_context}\n\n" + f"
" + ) + + return reformatted_query_context + f"\n\n[USER QUERY]\n\n{query}" + + +def parse_pdf(file_path): + from llama_parse import LlamaParse + + assert ( + "LLAMA_CLOUD_API_KEY" in os.environ + ), "Make sure to specify LlamaParse API key." + documents = LlamaParse( + result_type="markdown", + verbose=True, + ).load_data(file_path) + + return documents + + def _prepare_text_with_image(state, text, images, csam_flag): if len(images) > 0: if len(state.conv.get_images()) > 0: @@ -177,6 +255,18 @@ def _prepare_text_with_image(state, text, images, csam_flag): return text +def _prepare_text_with_pdf(state, text, pdfs): + if len(pdfs) > 0: + # if len(state.conv.get_pdfs()) > 0: + state.conv = get_conversation_template(state.model_name) + assert len(text) > 0 + + document_content = parse_pdf(pdfs[0]) + text = wrap_pdfchat_query(text, document_content) + + return text + + # NOTE(chris): take multiple images later on def convert_images_to_conversation_format(images): import base64 diff --git a/fastchat/serve/gradio_block_arena_vision_anony.py b/fastchat/serve/gradio_block_arena_vision_anony.py index 5b05864c5..5c3870925 100644 --- a/fastchat/serve/gradio_block_arena_vision_anony.py +++ b/fastchat/serve/gradio_block_arena_vision_anony.py @@ -10,11 +10,6 @@ import numpy as np from typing import Union -import os -import PyPDF2 -import nest_asyncio -from llama_parse import LlamaParse - from fastchat.constants import ( TEXT_MODERATION_MSG, IMAGE_MODERATION_MSG, @@ -68,10 +63,13 @@ moderate_input, enable_multimodal, _prepare_text_with_image, + _prepare_text_with_pdf, convert_images_to_conversation_format, invisible_text, visible_text, disable_multimodal, + is_image, + is_pdf, ) from fastchat.serve.gradio_global_state import Context from fastchat.serve.remote_logger import get_remote_logger @@ -91,15 +89,19 @@ # TODO(chris): fix sampling weights VISION_SAMPLING_WEIGHTS = {} +PDFCHAT_SAMPLING_WEIGHTS = {} # TODO(chris): Find battle targets that make sense VISION_BATTLE_TARGETS = {} +PDFCHAT_BATTLE_TARGETS = {} # TODO(chris): Fill out models that require sampling boost VISION_SAMPLING_BOOST_MODELS = [] +PDFCHAT_SAMPLING_BOOST_MODELS = [] # outage models won't be sampled. VISION_OUTAGE_MODELS = [] +PDFCHAT_OUTAGE_MODELS = [] def get_vqa_sample(): @@ -247,79 +249,6 @@ def clear_history(request: gr.Request): + [""] ) -def is_pdf(file_path): - try: - with open(file_path, 'rb') as file: - header = file.read(5) # Read the first 5 bytes - return header == b'%PDF-' - except Exception as e: - print(f"Error: {e}") - return False - -def is_image(file_path): - magic_numbers = { - b'\xff\xd8\xff': 'JPEG', - b'\x89PNG\r\n\x1a\n': 'PNG', - b'GIF87a': 'GIF', - b'GIF89a': 'GIF', - b'BM': 'BMP', - b'\x00\x00\x01\x00': 'ICO', - b'\x49\x49\x2a\x00': 'TIFF', - b'\x4d\x4d\x00\x2a': 'TIFF', - b'RIFF': 'WebP', - } - try: - with open(file_path, 'rb') as file: - header = file.read(8) # Read the first 8 bytes - for magic in magic_numbers: - if header.startswith(magic): - return True - return False - except Exception as e: - print(f"Error reading file: {e}") - return False - -nest_asyncio.apply() # Ensure compatibility with async environments - -def pdf_parse(pdf_path): - # Set API key, can also be configured in the environment - api_key = os.environ["LLAMA_PARSE_KEY"] - - # Initialize the LlamaParse object - parser = LlamaParse( - api_key=api_key, - result_type="markdown", # Output in Markdown format - num_workers=4, # Number of API calls for batch processing - verbose=True, # Print detailed logs - language="en" # Set language (default is English) - ) - - pdf_name = os.path.splitext(os.path.basename(pdf_path))[0] - extra_info = {"file_name": pdf_name} - - with open(pdf_path, "rb") as pdf_file: - # Pass the file object and extra info for parsing - documents = parser.load_data(pdf_file, extra_info=extra_info) - - # Save the parsed content to a Markdown file - markdown_content = documents[0].text if documents else "" - - return markdown_content - -def wrap_query_context(user_query, query_context): - #TODO: refactor to split up user query and query context. - # lines = input.split("\n\n[USER QUERY]", 1) - # user_query = lines[1].strip() - # query_context = lines[0][len('[QUERY CONTEXT]\n\n'): ] - reformatted_query_context = ( - f"[QUERY CONTEXT]\n" - f"
\n" - f"Expand context details\n\n" - f"{query_context}\n\n" - f"
" - ) - markdown = reformatted_query_context + f"\n\n[USER QUERY]\n\n{user_query}" - return markdown def add_text( state0, @@ -336,24 +265,19 @@ def add_text( text = chat_input files = [] - images = [] - - # currently support up to one pdf or one image - # if is_pdf(files[0]): - # pdfs = files - if is_image(files[0]): - images = files - ip = get_ip(request) logger.info(f"add_text (anony). ip: {ip}. len: {len(text)}") states = [state0, state1] model_selectors = [model_selector0, model_selector1] + images = [file for file in files if is_image(file)] + pdfs = [file for file in files if is_pdf(file)] + # Init states if necessary if states[0] is None: assert states[1] is None - if len(files) > 0 and is_image(files[0]): + if len(images) > 0: model_left, model_right = get_battle_pair( context.all_vision_models, VISION_BATTLE_TARGETS, @@ -365,6 +289,18 @@ def add_text( State(model_left, is_vision=True), State(model_right, is_vision=True), ] + elif len(pdfs) > 0: + model_left, model_right = get_battle_pair( + context.all_pdfchat_models, + PDFCHAT_BATTLE_TARGETS, + PDFCHAT_OUTAGE_MODELS, + PDFCHAT_SAMPLING_WEIGHTS, + PDFCHAT_SAMPLING_BOOST_MODELS, + ) + states = [ + State(model_left, is_vision=False), + State(model_right, is_vision=False), + ] else: model_left, model_right = get_battle_pair( context.all_text_models, @@ -401,6 +337,8 @@ def add_text( state0, text, text, model_list, images, ip ) + # TODO: add PDF moderator + conv = states[0].conv if (len(conv.messages) - conv.offset) // 2 >= CONVERSATION_TURN_LIMIT: logger.info(f"conversation turn limit. ip: {get_ip(request)}. text: {text}") @@ -436,12 +374,13 @@ def add_text( + [""] ) - if is_image(files[0]): - text = text[:BLIND_MODE_INPUT_CHAR_LEN_LIMIT] # Hard cut-off + text = text[:BLIND_MODE_INPUT_CHAR_LEN_LIMIT] # Hard cut-off for i in range(num_sides): post_processed_text = _prepare_text_with_image( states[i], text, images, csam_flag=csam_flag ) + post_processed_text = _prepare_text_with_pdf(states[i], text, pdfs + states[i].conv.append_message(states[i].conv.roles[0], post_processed_text) states[i].conv.append_message(states[i].conv.roles[1], None) states[i].skip_next = False @@ -450,26 +389,6 @@ def add_text( for i in range(num_sides): if "deluxe" in states[i].model_name: hint_msg = SLOW_MODEL_MSG - - if is_pdf(files[0]): - document_text = pdf_parse(files[0]) - prompt_text = f""" - The following is the content of a document: - - {document_text} - - Based on this document, answer the following question: - - {text} - """ - post_processed_text = wrap_query_context(text, prompt_text) - - # text = text[:BLIND_MODE_INPUT_CHAR_LEN_LIMIT] # Hard cut-off - for i in range(num_sides): - states[i].conv.append_message(states[i].conv.roles[0], post_processed_text) - states[i].conv.append_message(states[i].conv.roles[1], None) - states[i].skip_next = False - return ( states + [x.to_gradio_chatbot() for x in states] @@ -578,10 +497,10 @@ def build_side_by_side_vision_ui_anony(context: Context, random_questions=None): ) multimodal_textbox = gr.MultimodalTextbox( - file_types=["file"], + file_types=["image", ".pdf"], show_label=False, container=True, - placeholder="Enter your prompt here. You can also upload image or PDF file", + placeholder="Enter your prompt or add image here", elem_id="input_box", scale=3, ) @@ -590,12 +509,11 @@ def build_side_by_side_vision_ui_anony(context: Context, random_questions=None): ) with gr.Row() as button_row: - random_btn = gr.Button(value="🔮 Random Image", interactive=True) - # if random_questions: - # global vqa_samples - # with open(random_questions, "r") as f: - # vqa_samples = json.load(f) - # random_btn = gr.Button(value="🔮 Random Image", interactive=True) + if random_questions: + global vqa_samples + with open(random_questions, "r") as f: + vqa_samples = json.load(f) + random_btn = gr.Button(value="🔮 Random Image", interactive=True) clear_btn = gr.Button(value="🎲 New Round", interactive=False) regenerate_btn = gr.Button(value="🔄 Regenerate", interactive=False) share_btn = gr.Button(value="📷 Share") diff --git a/fastchat/serve/gradio_global_state.py b/fastchat/serve/gradio_global_state.py index fafaec213..674d7e219 100644 --- a/fastchat/serve/gradio_global_state.py +++ b/fastchat/serve/gradio_global_state.py @@ -8,5 +8,7 @@ class Context: all_text_models: List[str] = field(default_factory=list) vision_models: List[str] = field(default_factory=list) all_vision_models: List[str] = field(default_factory=list) + pdfchat_models: List[str] = field(default_factory=list) + all_pdfchat_models: List[str] = field(default_factory=list) models: List[str] = field(default_factory=list) all_models: List[str] = field(default_factory=list) diff --git a/fastchat/serve/gradio_web_server.py b/fastchat/serve/gradio_web_server.py index 4f0521da0..1cc4a4982 100644 --- a/fastchat/serve/gradio_web_server.py +++ b/fastchat/serve/gradio_web_server.py @@ -183,7 +183,9 @@ def get_conv_log_filename(is_vision=False, has_csam_image=False): return name -def get_model_list(controller_url, register_api_endpoint_file, vision_arena): +def get_model_list( + controller_url, register_api_endpoint_file, vision_arena=False, pdfchat_arena=False +): global api_endpoint_info # Add models from the controller @@ -205,10 +207,13 @@ def get_model_list(controller_url, register_api_endpoint_file, vision_arena): api_endpoint_info = json.load(open(register_api_endpoint_file)) for mdl, mdl_dict in api_endpoint_info.items(): mdl_vision = mdl_dict.get("vision-arena", False) + mdl_pdfchat = mdl_dict.get("pdfchat-arena", False) mdl_text = mdl_dict.get("text-arena", True) if vision_arena and mdl_vision: models.append(mdl) - if not vision_arena and mdl_text: + if pdfchat_arena and mdl_pdfchat: + models.append(mdl) + if not (vision_arena or pdfchat_arena) and mdl_text: models.append(mdl) # Remove anonymous models diff --git a/fastchat/serve/gradio_web_server_multi.py b/fastchat/serve/gradio_web_server_multi.py index 7a255d59e..1b44ae215 100644 --- a/fastchat/serve/gradio_web_server_multi.py +++ b/fastchat/serve/gradio_web_server_multi.py @@ -77,12 +77,21 @@ def load_demo(context: Context, request: gr.Request): args.controller_url, args.register_api_endpoint_file, vision_arena=False, + pdfchat_arena=False, ) context.vision_models, context.all_vision_models = get_model_list( args.controller_url, args.register_api_endpoint_file, vision_arena=True, + pdfchat_arena=False, + ) + + context.pdfchat_models, context.all_pdfchat_models = get_model_list( + args.controller_url, + args.register_api_endpoint_file, + vision_arena=False, + pdfchat_arena=True, ) # Text models @@ -316,25 +325,32 @@ def build_demo( args.controller_url, args.register_api_endpoint_file, vision_arena=False, + pdfchat_arena=False, ) vision_models, all_vision_models = get_model_list( args.controller_url, args.register_api_endpoint_file, vision_arena=True, + pdfchat_arena=False, + ) + + pdfchat_models, all_pdfchat_models = get_model_list( + args.controller_url, + args.register_api_endpoint_file, + vision_arena=False, + pdfchat_arena=True, ) - models = text_models + [ - model for model in vision_models if model not in text_models - ] - all_models = all_text_models + [ - model for model in all_vision_models if model not in all_text_models - ] + models = list(set(text_models + vision_models + pdfchat_models)) + all_models = list(set(all_text_models + all_vision_models + all_pdfchat_models)) context = Context( text_models, all_text_models, vision_models, all_vision_models, + pdfchat_models, + all_pdfchat_models, models, all_models, ) From 8527b029665fe4b2ea2d7cd076160750a575ccaf Mon Sep 17 00:00:00 2001 From: CodingWithTim Date: Mon, 30 Dec 2024 00:17:37 +0000 Subject: [PATCH 07/40] fix format --- fastchat/serve/gradio_block_arena_vision_anony.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fastchat/serve/gradio_block_arena_vision_anony.py b/fastchat/serve/gradio_block_arena_vision_anony.py index 5c3870925..31f0f343b 100644 --- a/fastchat/serve/gradio_block_arena_vision_anony.py +++ b/fastchat/serve/gradio_block_arena_vision_anony.py @@ -379,8 +379,8 @@ def add_text( post_processed_text = _prepare_text_with_image( states[i], text, images, csam_flag=csam_flag ) - post_processed_text = _prepare_text_with_pdf(states[i], text, pdfs - + post_processed_text = _prepare_text_with_pdf(states[i], text, pdfs) + states[i].conv.append_message(states[i].conv.roles[0], post_processed_text) states[i].conv.append_message(states[i].conv.roles[1], None) states[i].skip_next = False From 8da825ba6fde563cb67f63174914a09a3ceb6204 Mon Sep 17 00:00:00 2001 From: Tim Li Date: Sun, 29 Dec 2024 21:31:59 -0800 Subject: [PATCH 08/40] improve UI and efficiency --- fastchat/serve/gradio_block_arena_vision.py | 14 +++++++------- fastchat/serve/gradio_block_arena_vision_anony.py | 5 +++-- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/fastchat/serve/gradio_block_arena_vision.py b/fastchat/serve/gradio_block_arena_vision.py index c56685902..09139a2ec 100644 --- a/fastchat/serve/gradio_block_arena_vision.py +++ b/fastchat/serve/gradio_block_arena_vision.py @@ -236,12 +236,15 @@ def parse_pdf(file_path): assert ( "LLAMA_CLOUD_API_KEY" in os.environ ), "Make sure to specify LlamaParse API key." - documents = LlamaParse( + document = LlamaParse( result_type="markdown", verbose=True, ).load_data(file_path) + + assert len(document) > 0 + output = document[0].text - return documents + return output def _prepare_text_with_image(state, text, images, csam_flag): @@ -255,13 +258,10 @@ def _prepare_text_with_image(state, text, images, csam_flag): return text -def _prepare_text_with_pdf(state, text, pdfs): +def _prepare_text_with_pdf(text, pdfs): if len(pdfs) > 0: - # if len(state.conv.get_pdfs()) > 0: - state.conv = get_conversation_template(state.model_name) - assert len(text) > 0 - document_content = parse_pdf(pdfs[0]) + print("Document processed") text = wrap_pdfchat_query(text, document_content) return text diff --git a/fastchat/serve/gradio_block_arena_vision_anony.py b/fastchat/serve/gradio_block_arena_vision_anony.py index 31f0f343b..18361510b 100644 --- a/fastchat/serve/gradio_block_arena_vision_anony.py +++ b/fastchat/serve/gradio_block_arena_vision_anony.py @@ -375,11 +375,12 @@ def add_text( ) text = text[:BLIND_MODE_INPUT_CHAR_LEN_LIMIT] # Hard cut-off + post_processed_text = _prepare_text_with_pdf(text, pdfs) + for i in range(num_sides): post_processed_text = _prepare_text_with_image( - states[i], text, images, csam_flag=csam_flag + states[i], post_processed_text, images, csam_flag=csam_flag ) - post_processed_text = _prepare_text_with_pdf(states[i], text, pdfs) states[i].conv.append_message(states[i].conv.roles[0], post_processed_text) states[i].conv.append_message(states[i].conv.roles[1], None) From b59cea84d8b34c1654306ac7876929a31a62aaa6 Mon Sep 17 00:00:00 2001 From: CodingWithTim Date: Mon, 30 Dec 2024 05:33:40 +0000 Subject: [PATCH 09/40] fix formatting --- fastchat/serve/gradio_block_arena_vision.py | 2 +- fastchat/serve/gradio_block_arena_vision_anony.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fastchat/serve/gradio_block_arena_vision.py b/fastchat/serve/gradio_block_arena_vision.py index 09139a2ec..6abc4aef9 100644 --- a/fastchat/serve/gradio_block_arena_vision.py +++ b/fastchat/serve/gradio_block_arena_vision.py @@ -240,7 +240,7 @@ def parse_pdf(file_path): result_type="markdown", verbose=True, ).load_data(file_path) - + assert len(document) > 0 output = document[0].text diff --git a/fastchat/serve/gradio_block_arena_vision_anony.py b/fastchat/serve/gradio_block_arena_vision_anony.py index 18361510b..28ec9d645 100644 --- a/fastchat/serve/gradio_block_arena_vision_anony.py +++ b/fastchat/serve/gradio_block_arena_vision_anony.py @@ -376,7 +376,7 @@ def add_text( text = text[:BLIND_MODE_INPUT_CHAR_LEN_LIMIT] # Hard cut-off post_processed_text = _prepare_text_with_pdf(text, pdfs) - + for i in range(num_sides): post_processed_text = _prepare_text_with_image( states[i], post_processed_text, images, csam_flag=csam_flag From d5efd2c13f1646ec40fe711c057c8d7401f70a2f Mon Sep 17 00:00:00 2001 From: Tim Li Date: Mon, 30 Dec 2024 00:50:39 -0800 Subject: [PATCH 10/40] fix first page only parsing issue --- fastchat/serve/gradio_block_arena_vision.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/fastchat/serve/gradio_block_arena_vision.py b/fastchat/serve/gradio_block_arena_vision.py index 09139a2ec..c6c6c0393 100644 --- a/fastchat/serve/gradio_block_arena_vision.py +++ b/fastchat/serve/gradio_block_arena_vision.py @@ -215,8 +215,7 @@ def wrap_pdfchat_query(query, document): document_context = f""" The following is the content of a document: {document} - Based on this document, answer the following question: - {query} + Based on this document, answer the user query. """ reformatted_query_context = ( @@ -236,13 +235,13 @@ def parse_pdf(file_path): assert ( "LLAMA_CLOUD_API_KEY" in os.environ ), "Make sure to specify LlamaParse API key." - document = LlamaParse( + documents = LlamaParse( result_type="markdown", verbose=True, ).load_data(file_path) - - assert len(document) > 0 - output = document[0].text + + assert len(documents) > 0 + output = "\n".join([doc.text for doc in documents]) return output From e7ab73f2752057563d14517a20e5a90a4e23577e Mon Sep 17 00:00:00 2001 From: Tim Li Date: Mon, 30 Dec 2024 01:29:14 -0800 Subject: [PATCH 11/40] additional improvements --- fastchat/serve/gradio_block_arena_vision.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/fastchat/serve/gradio_block_arena_vision.py b/fastchat/serve/gradio_block_arena_vision.py index c6c6c0393..2419cd91c 100644 --- a/fastchat/serve/gradio_block_arena_vision.py +++ b/fastchat/serve/gradio_block_arena_vision.py @@ -212,21 +212,24 @@ def report_csam_image(state, image): def wrap_pdfchat_query(query, document): # TODO: Considering redesign the context format. - document_context = f""" - The following is the content of a document: - {document} - Based on this document, answer the user query. - """ + # document_context = f""" + # The following is the content of a document: + # {document} + # Based on this document, answer the following question: + # {query} + # """ reformatted_query_context = ( + f"Answer the user query given the context.\n" f"[QUERY CONTEXT]\n" f"
\n" f"Expand context details\n\n" - f"{document_context}\n\n" + f"{document}\n\n" f"
" + f"\n\n[USER QUERY]\n\n{query}" ) - return reformatted_query_context + f"\n\n[USER QUERY]\n\n{query}" + return reformatted_query_context def parse_pdf(file_path): @@ -241,7 +244,9 @@ def parse_pdf(file_path): ).load_data(file_path) assert len(documents) > 0 - output = "\n".join([doc.text for doc in documents]) + output = "\n".join( + [f"Page {i+1}:\n{doc.text}\n" for i, doc in enumerate(documents)] + ) return output From f1c6185f20725bf298bc40c58b29fef4972a2bd9 Mon Sep 17 00:00:00 2001 From: CodingWithTim Date: Thu, 2 Jan 2025 08:24:44 +0000 Subject: [PATCH 12/40] add multilingual support --- fastchat/serve/gradio_block_arena_vision.py | 72 +++++++++++++++++++-- 1 file changed, 67 insertions(+), 5 deletions(-) diff --git a/fastchat/serve/gradio_block_arena_vision.py b/fastchat/serve/gradio_block_arena_vision.py index 2419cd91c..a7192f0d9 100644 --- a/fastchat/serve/gradio_block_arena_vision.py +++ b/fastchat/serve/gradio_block_arena_vision.py @@ -232,18 +232,80 @@ def wrap_pdfchat_query(query, document): return reformatted_query_context +LLAMA_PARSE_MAX_RETRY = 2 +TESSERACT_SUPPORTED_LANGS = "+".join( + [ + "en", + "chi_tra", + "chi_sim", + "rus", + "spa", + "jpn", + "kor", + "fra", + "deu", # German + "vie", + ] +) +LLAMAPARSE_SUPPORTED_LANGS = { + "English": "en", + "Chinese": "ch_sim", + "Russian": "ru", + "Spanish": "es", + "Japanese": "ja", + "Korean": "ko", + "French": "fr", + "German": "de", + "Vietnamese": "vi", +} + + +def detect_language_from_doc(pdf_file_path): + from pdf2image import convert_from_path + from polyglot.detect import Detector + + import pytesseract # Google's open-source OCR tool + + assert os.environ[ + "TESSDATA_PREFIX" + ], "Make sure to specify location of train data for Tesseract." + + # Convert pdf into image (first page only for efficiency) + images = convert_from_path(pdf_file_path) + extracted_text = pytesseract.image_to_string( + images[0], lang=TESSERACT_SUPPORTED_LANGS + ) + + languages = Detector(extracted_text, quiet=True) + # return languages + return [lang.name for lang in languages.languages if lang.name != "un"] + + def parse_pdf(file_path): from llama_parse import LlamaParse assert ( "LLAMA_CLOUD_API_KEY" in os.environ ), "Make sure to specify LlamaParse API key." - documents = LlamaParse( - result_type="markdown", - verbose=True, - ).load_data(file_path) - assert len(documents) > 0 + doc_lang = detect_language_from_doc(file_path) + doc_lang = LLAMAPARSE_SUPPORTED_LANGS[doc_lang[0]] + + print(doc_lang) + + for _ in range(LLAMA_PARSE_MAX_RETRY): + try: + documents = LlamaParse( + result_type="markdown", + verbose=True, + language=doc_lang, + accurate_mode=True, + ).load_data(file_path) + assert len(documents) > 0 + break + except AssertionError as e: + continue + output = "\n".join( [f"Page {i+1}:\n{doc.text}\n" for i, doc in enumerate(documents)] ) From f7e92e141b9f2a82ed63ec614fe7e56441b8d553 Mon Sep 17 00:00:00 2001 From: CodingWithTim Date: Sat, 4 Jan 2025 20:55:39 +0000 Subject: [PATCH 13/40] support google cloud storage --- fastchat/constants.py | 3 ++ fastchat/serve/gradio_block_arena_vision.py | 41 ++++++++++++++++++- .../serve/gradio_block_arena_vision_anony.py | 27 +++++++++--- fastchat/serve/gradio_web_server.py | 4 +- fastchat/serve/setup_pdfchat.sh | 19 +++++++++ fastchat/utils.py | 33 +++++++++++++++ 6 files changed, 119 insertions(+), 8 deletions(-) create mode 100644 fastchat/serve/setup_pdfchat.sh diff --git a/fastchat/constants.py b/fastchat/constants.py index a509a8346..6dbf5d152 100644 --- a/fastchat/constants.py +++ b/fastchat/constants.py @@ -25,6 +25,9 @@ IMAGE_MODERATION_MSG = ( "$MODERATION$ YOUR IMAGE VIOLATES OUR CONTENT MODERATION GUIDELINES." ) +PDF_MODERATION_MSG = ( + "$MODERATION$ YOUR PDF VIOLATES OUR CONTENT MODERATION GUIDELINES." +) MODERATION_MSG = "$MODERATION$ YOUR INPUT VIOLATES OUR CONTENT MODERATION GUIDELINES." CONVERSATION_LIMIT_MSG = "YOU HAVE REACHED THE CONVERSATION LENGTH LIMIT. PLEASE CLEAR HISTORY AND START A NEW CONVERSATION." INACTIVE_MSG = "THIS SESSION HAS BEEN INACTIVE FOR TOO LONG. PLEASE REFRESH THIS PAGE." diff --git a/fastchat/serve/gradio_block_arena_vision.py b/fastchat/serve/gradio_block_arena_vision.py index a7192f0d9..5e90bfad0 100644 --- a/fastchat/serve/gradio_block_arena_vision.py +++ b/fastchat/serve/gradio_block_arena_vision.py @@ -260,6 +260,44 @@ def wrap_pdfchat_query(query, document): } +# TODO: P1: Integrate this. +def pdf_moderator(images): + import base64 + from openai import OpenAI + from io import BytesIO + + base64_urls = [] + for image in images: + buffer = BytesIO() + image.save(buffer, format="JPEG") + + image_bytes = buffer.getvalue() + image_b64 = base64.b64encode(image_bytes).decode("utf-8") + + # convert to openai format + base64_urls.append({ + "type": "image_url", + "image_url": { + "url": f"data:image/jpeg;base64,{image_b64}", + } + }) + + # OpenAI's maximum number of images is 1 at the moment. + client = OpenAI() + moderations = [] + for url in base64_urls: + try: + response = client.moderations.create( + model="omni-moderation-latest", + input=url, + ) + moderations.append(response[0].results.flagged) + except Exception as e: + print(e) + + return all(moderations) + + def detect_language_from_doc(pdf_file_path): from pdf2image import convert_from_path from polyglot.detect import Detector @@ -272,6 +310,7 @@ def detect_language_from_doc(pdf_file_path): # Convert pdf into image (first page only for efficiency) images = convert_from_path(pdf_file_path) + extracted_text = pytesseract.image_to_string( images[0], lang=TESSERACT_SUPPORTED_LANGS ) @@ -291,8 +330,6 @@ def parse_pdf(file_path): doc_lang = detect_language_from_doc(file_path) doc_lang = LLAMAPARSE_SUPPORTED_LANGS[doc_lang[0]] - print(doc_lang) - for _ in range(LLAMA_PARSE_MAX_RETRY): try: documents = LlamaParse( diff --git a/fastchat/serve/gradio_block_arena_vision_anony.py b/fastchat/serve/gradio_block_arena_vision_anony.py index 28ec9d645..d26aed1e5 100644 --- a/fastchat/serve/gradio_block_arena_vision_anony.py +++ b/fastchat/serve/gradio_block_arena_vision_anony.py @@ -13,6 +13,7 @@ from fastchat.constants import ( TEXT_MODERATION_MSG, IMAGE_MODERATION_MSG, + PDF_MODERATION_MSG, MODERATION_MSG, CONVERSATION_LIMIT_MSG, SLOW_MODEL_MSG, @@ -77,6 +78,8 @@ build_logger, moderation_filter, image_moderation_filter, + upload_pdf_file_to_gcs, + hash_pdf, ) logger = build_logger("gradio_web_server_multi", "gradio_web_server_multi.log") @@ -297,10 +300,26 @@ def add_text( PDFCHAT_SAMPLING_WEIGHTS, PDFCHAT_SAMPLING_BOOST_MODELS, ) + + # Save an unique id for mapping conversation back to the file on google cloud. + unique_id = hash_pdf(pdfs[0]) + states = [ - State(model_left, is_vision=False), - State(model_right, is_vision=False), + State( + model_left, + is_vision=False, + pdf_id=unique_id + ), + State( + model_right, + is_vision=False, + pdf_id=unique_id + ), ] + upload_pdf_file_to_gcs( + pdf_file_path=pdfs[0], + filename=unique_id, + ) else: model_left, model_right = get_battle_pair( context.all_text_models, @@ -309,7 +328,6 @@ def add_text( SAMPLING_WEIGHTS, SAMPLING_BOOST_MODELS, ) - states = [ State(model_left, is_vision=False), State(model_right, is_vision=False), @@ -333,12 +351,11 @@ def add_text( images = convert_images_to_conversation_format(images) + # TODO: add PDF moderator text, image_flagged, csam_flag = moderate_input( state0, text, text, model_list, images, ip ) - # TODO: add PDF moderator - conv = states[0].conv if (len(conv.messages) - conv.offset) // 2 >= CONVERSATION_TURN_LIMIT: logger.info(f"conversation turn limit. ip: {get_ip(request)}. text: {text}") diff --git a/fastchat/serve/gradio_web_server.py b/fastchat/serve/gradio_web_server.py index 1cc4a4982..a5d6db2a1 100644 --- a/fastchat/serve/gradio_web_server.py +++ b/fastchat/serve/gradio_web_server.py @@ -112,13 +112,14 @@ class State: - def __init__(self, model_name, is_vision=False): + def __init__(self, model_name, is_vision=False, pdf_id=None): self.conv = get_conversation_template(model_name) self.conv_id = uuid.uuid4().hex self.skip_next = False self.model_name = model_name self.oai_thread_id = None self.is_vision = is_vision + self.pdf_id = pdf_id # NOTE(Tim): Version 1 PDFChat Architecture, could be revised later. # NOTE(chris): This could be sort of a hack since it assumes the user only uploads one image. If they can upload multiple, we should store a list of image hashes. self.has_csam_image = False @@ -151,6 +152,7 @@ def dict(self): { "conv_id": self.conv_id, "model_name": self.model_name, + "pdf_id": self.pdf_id, } ) diff --git a/fastchat/serve/setup_pdfchat.sh b/fastchat/serve/setup_pdfchat.sh new file mode 100644 index 000000000..e2d3fbc15 --- /dev/null +++ b/fastchat/serve/setup_pdfchat.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +# Install Python packages +pip install llama-index-core llama-parse llama-index-readers-file python-dotenv +pip install polyglot +pip install PyICU +pip install pycld2 +pip install pytesseract + +pip install pdf2image + +# Clone the Tesseract tessdata repository +git clone https://github.com/tesseract-ocr/tessdata + +# cd into tessdata and set TESSDATA_PREFIX to the current directory +cd tessdata +export TESSDATA_PREFIX="$(pwd)" + +echo "TESSDATA_PREFIX is set to: $TESSDATA_PREFIX" \ No newline at end of file diff --git a/fastchat/utils.py b/fastchat/utils.py index d3531928f..6f1d1bb2b 100644 --- a/fastchat/utils.py +++ b/fastchat/utils.py @@ -430,6 +430,20 @@ def upload_image_file_to_gcs(image, filename): return blob.public_url +def upload_pdf_file_to_gcs(pdf_file_path, filename): + from google.cloud import storage + + storage_client = storage.Client() + # upload file to GCS + bucket = storage_client.get_bucket("arena-pdf-dev") + + blob = bucket.blob(filename) + # Automatically opens the file in binary read mode + blob.upload_from_filename(pdf_file_path, content_type="application/pdf") + + return blob.public_url + + def get_image_file_from_gcs(filename): from google.cloud import storage @@ -441,6 +455,25 @@ def get_image_file_from_gcs(filename): return contents +def get_pdf_file_from_gcs(filename): + from google.cloud import storage + + storage_client = storage.Client() + bucket = storage_client.get_bucket("arena-pdf-dev") + blob = bucket.blob(f"{filename}") + contents = blob.download_as_bytes() + + return contents + + +def hash_pdf(file_path): + import hashlib + + with open(file_path, 'rb') as f: + file_content = f.read() + return hashlib.md5(file_content).hexdigest() + + def image_moderation_request(image_bytes, endpoint, api_key): headers = {"Content-Type": "image/jpeg", "Ocp-Apim-Subscription-Key": api_key} From 0daef323952ebdff83c9acde5ebe6d5cb778a382 Mon Sep 17 00:00:00 2001 From: CodingWithTim Date: Sat, 4 Jan 2025 21:03:46 +0000 Subject: [PATCH 14/40] fix format --- fastchat/constants.py | 4 +--- fastchat/serve/gradio_block_arena_vision.py | 22 ++++++++++--------- .../serve/gradio_block_arena_vision_anony.py | 16 ++++---------- fastchat/serve/gradio_web_server.py | 4 +++- fastchat/utils.py | 4 ++-- 5 files changed, 22 insertions(+), 28 deletions(-) diff --git a/fastchat/constants.py b/fastchat/constants.py index 6dbf5d152..a5ba0752a 100644 --- a/fastchat/constants.py +++ b/fastchat/constants.py @@ -25,9 +25,7 @@ IMAGE_MODERATION_MSG = ( "$MODERATION$ YOUR IMAGE VIOLATES OUR CONTENT MODERATION GUIDELINES." ) -PDF_MODERATION_MSG = ( - "$MODERATION$ YOUR PDF VIOLATES OUR CONTENT MODERATION GUIDELINES." -) +PDF_MODERATION_MSG = "$MODERATION$ YOUR PDF VIOLATES OUR CONTENT MODERATION GUIDELINES." MODERATION_MSG = "$MODERATION$ YOUR INPUT VIOLATES OUR CONTENT MODERATION GUIDELINES." CONVERSATION_LIMIT_MSG = "YOU HAVE REACHED THE CONVERSATION LENGTH LIMIT. PLEASE CLEAR HISTORY AND START A NEW CONVERSATION." INACTIVE_MSG = "THIS SESSION HAS BEEN INACTIVE FOR TOO LONG. PLEASE REFRESH THIS PAGE." diff --git a/fastchat/serve/gradio_block_arena_vision.py b/fastchat/serve/gradio_block_arena_vision.py index 5e90bfad0..b78da167e 100644 --- a/fastchat/serve/gradio_block_arena_vision.py +++ b/fastchat/serve/gradio_block_arena_vision.py @@ -265,23 +265,25 @@ def pdf_moderator(images): import base64 from openai import OpenAI from io import BytesIO - + base64_urls = [] for image in images: buffer = BytesIO() image.save(buffer, format="JPEG") - + image_bytes = buffer.getvalue() image_b64 = base64.b64encode(image_bytes).decode("utf-8") # convert to openai format - base64_urls.append({ - "type": "image_url", - "image_url": { - "url": f"data:image/jpeg;base64,{image_b64}", + base64_urls.append( + { + "type": "image_url", + "image_url": { + "url": f"data:image/jpeg;base64,{image_b64}", + }, } - }) - + ) + # OpenAI's maximum number of images is 1 at the moment. client = OpenAI() moderations = [] @@ -294,7 +296,7 @@ def pdf_moderator(images): moderations.append(response[0].results.flagged) except Exception as e: print(e) - + return all(moderations) @@ -310,7 +312,7 @@ def detect_language_from_doc(pdf_file_path): # Convert pdf into image (first page only for efficiency) images = convert_from_path(pdf_file_path) - + extracted_text = pytesseract.image_to_string( images[0], lang=TESSERACT_SUPPORTED_LANGS ) diff --git a/fastchat/serve/gradio_block_arena_vision_anony.py b/fastchat/serve/gradio_block_arena_vision_anony.py index d26aed1e5..52a244a2a 100644 --- a/fastchat/serve/gradio_block_arena_vision_anony.py +++ b/fastchat/serve/gradio_block_arena_vision_anony.py @@ -300,21 +300,13 @@ def add_text( PDFCHAT_SAMPLING_WEIGHTS, PDFCHAT_SAMPLING_BOOST_MODELS, ) - + # Save an unique id for mapping conversation back to the file on google cloud. unique_id = hash_pdf(pdfs[0]) - + states = [ - State( - model_left, - is_vision=False, - pdf_id=unique_id - ), - State( - model_right, - is_vision=False, - pdf_id=unique_id - ), + State(model_left, is_vision=False, pdf_id=unique_id), + State(model_right, is_vision=False, pdf_id=unique_id), ] upload_pdf_file_to_gcs( pdf_file_path=pdfs[0], diff --git a/fastchat/serve/gradio_web_server.py b/fastchat/serve/gradio_web_server.py index a5d6db2a1..ff10e4580 100644 --- a/fastchat/serve/gradio_web_server.py +++ b/fastchat/serve/gradio_web_server.py @@ -119,7 +119,9 @@ def __init__(self, model_name, is_vision=False, pdf_id=None): self.model_name = model_name self.oai_thread_id = None self.is_vision = is_vision - self.pdf_id = pdf_id # NOTE(Tim): Version 1 PDFChat Architecture, could be revised later. + self.pdf_id = ( + pdf_id # NOTE(Tim): Version 1 PDFChat Architecture, could be revised later. + ) # NOTE(chris): This could be sort of a hack since it assumes the user only uploads one image. If they can upload multiple, we should store a list of image hashes. self.has_csam_image = False diff --git a/fastchat/utils.py b/fastchat/utils.py index 6f1d1bb2b..2f3237c06 100644 --- a/fastchat/utils.py +++ b/fastchat/utils.py @@ -468,8 +468,8 @@ def get_pdf_file_from_gcs(filename): def hash_pdf(file_path): import hashlib - - with open(file_path, 'rb') as f: + + with open(file_path, "rb") as f: file_content = f.read() return hashlib.md5(file_content).hexdigest() From 32c6724c41881f487ee2905b661b1252094af1bc Mon Sep 17 00:00:00 2001 From: CodingWithTim Date: Sat, 4 Jan 2025 22:36:14 +0000 Subject: [PATCH 15/40] add pdf maximum page limit --- fastchat/constants.py | 3 ++ fastchat/serve/gradio_block_arena_vision.py | 40 ------------------- .../serve/gradio_block_arena_vision_anony.py | 27 ++++++++++++- fastchat/utils.py | 8 ++++ 4 files changed, 36 insertions(+), 42 deletions(-) diff --git a/fastchat/constants.py b/fastchat/constants.py index a5ba0752a..1a98f9a1b 100644 --- a/fastchat/constants.py +++ b/fastchat/constants.py @@ -40,6 +40,9 @@ ) # Maximum conversation turns CONVERSATION_TURN_LIMIT = 50 +# Maximum PDF Page Limit +PDF_PAGE_LIMIT = 50 +PDF_LIMIT_MSG = f"YOU HAVE REACHED THE MAXIMUM PDF PAGE LIMIT ({PDF_PAGE_LIMIT} PAGES). PLEASE UPLOAD A SMALLER DOCUMENT." # Session expiration time SESSION_EXPIRATION_TIME = 3600 # The output dir of log files diff --git a/fastchat/serve/gradio_block_arena_vision.py b/fastchat/serve/gradio_block_arena_vision.py index b78da167e..e5b529bff 100644 --- a/fastchat/serve/gradio_block_arena_vision.py +++ b/fastchat/serve/gradio_block_arena_vision.py @@ -260,46 +260,6 @@ def wrap_pdfchat_query(query, document): } -# TODO: P1: Integrate this. -def pdf_moderator(images): - import base64 - from openai import OpenAI - from io import BytesIO - - base64_urls = [] - for image in images: - buffer = BytesIO() - image.save(buffer, format="JPEG") - - image_bytes = buffer.getvalue() - image_b64 = base64.b64encode(image_bytes).decode("utf-8") - - # convert to openai format - base64_urls.append( - { - "type": "image_url", - "image_url": { - "url": f"data:image/jpeg;base64,{image_b64}", - }, - } - ) - - # OpenAI's maximum number of images is 1 at the moment. - client = OpenAI() - moderations = [] - for url in base64_urls: - try: - response = client.moderations.create( - model="omni-moderation-latest", - input=url, - ) - moderations.append(response[0].results.flagged) - except Exception as e: - print(e) - - return all(moderations) - - def detect_language_from_doc(pdf_file_path): from pdf2image import convert_from_path from polyglot.detect import Detector diff --git a/fastchat/serve/gradio_block_arena_vision_anony.py b/fastchat/serve/gradio_block_arena_vision_anony.py index 52a244a2a..ca74303ab 100644 --- a/fastchat/serve/gradio_block_arena_vision_anony.py +++ b/fastchat/serve/gradio_block_arena_vision_anony.py @@ -14,6 +14,8 @@ TEXT_MODERATION_MSG, IMAGE_MODERATION_MSG, PDF_MODERATION_MSG, + PDF_LIMIT_MSG, + PDF_PAGE_LIMIT, MODERATION_MSG, CONVERSATION_LIMIT_MSG, SLOW_MODEL_MSG, @@ -78,6 +80,7 @@ build_logger, moderation_filter, image_moderation_filter, + get_pdf_num_page, upload_pdf_file_to_gcs, hash_pdf, ) @@ -339,6 +342,25 @@ def add_text( + [""] ) + if len(pdfs) > 0 and get_pdf_num_page(pdfs[0]) > PDF_PAGE_LIMIT: + logger.info(f"pdf page limit exceeded. ip: {ip}. text: {text}") + for i in range(num_sides): + states[i].skip_next = True + return ( + states + + [x.to_gradio_chatbot() for x in states] + + [ + { + "text": PDF_LIMIT_MSG + + " PLEASE CLICK 🎲 NEW ROUND TO START A NEW CONVERSATION." + }, + "", + no_change_btn, + ] + + [no_change_btn] * 7 + + [""] + ) + model_list = [states[i].model_name for i in range(num_sides)] images = convert_images_to_conversation_format(images) @@ -356,11 +378,12 @@ def add_text( return ( states + [x.to_gradio_chatbot() for x in states] - + [{"text": CONVERSATION_LIMIT_MSG}, "", no_change_btn] + [ + {"text": CONVERSATION_LIMIT_MSG}, + "", no_change_btn, ] - * 7 + + [no_change_btn] * 7 + [""] ) diff --git a/fastchat/utils.py b/fastchat/utils.py index 2f3237c06..88ae37c7e 100644 --- a/fastchat/utils.py +++ b/fastchat/utils.py @@ -474,6 +474,14 @@ def hash_pdf(file_path): return hashlib.md5(file_content).hexdigest() +def get_pdf_num_page(file_path): + from pypdf import PdfReader + + reader = PdfReader(file_path) + + return len(reader.pages) + + def image_moderation_request(image_bytes, endpoint, api_key): headers = {"Content-Type": "image/jpeg", "Ocp-Apim-Subscription-Key": api_key} From 2cb0937ebfa5842f6e11cac335381d852e3ca0b7 Mon Sep 17 00:00:00 2001 From: CodingWithTim Date: Sun, 5 Jan 2025 01:46:18 +0000 Subject: [PATCH 16/40] remove language detection --- fastchat/serve/gradio_block_arena_vision.py | 43 ++------------------- fastchat/serve/setup_pdfchat.sh | 16 +------- 2 files changed, 4 insertions(+), 55 deletions(-) diff --git a/fastchat/serve/gradio_block_arena_vision.py b/fastchat/serve/gradio_block_arena_vision.py index e5b529bff..bd745791f 100644 --- a/fastchat/serve/gradio_block_arena_vision.py +++ b/fastchat/serve/gradio_block_arena_vision.py @@ -233,20 +233,6 @@ def wrap_pdfchat_query(query, document): LLAMA_PARSE_MAX_RETRY = 2 -TESSERACT_SUPPORTED_LANGS = "+".join( - [ - "en", - "chi_tra", - "chi_sim", - "rus", - "spa", - "jpn", - "kor", - "fra", - "deu", # German - "vie", - ] -) LLAMAPARSE_SUPPORTED_LANGS = { "English": "en", "Chinese": "ch_sim", @@ -260,28 +246,6 @@ def wrap_pdfchat_query(query, document): } -def detect_language_from_doc(pdf_file_path): - from pdf2image import convert_from_path - from polyglot.detect import Detector - - import pytesseract # Google's open-source OCR tool - - assert os.environ[ - "TESSDATA_PREFIX" - ], "Make sure to specify location of train data for Tesseract." - - # Convert pdf into image (first page only for efficiency) - images = convert_from_path(pdf_file_path) - - extracted_text = pytesseract.image_to_string( - images[0], lang=TESSERACT_SUPPORTED_LANGS - ) - - languages = Detector(extracted_text, quiet=True) - # return languages - return [lang.name for lang in languages.languages if lang.name != "un"] - - def parse_pdf(file_path): from llama_parse import LlamaParse @@ -289,15 +253,14 @@ def parse_pdf(file_path): "LLAMA_CLOUD_API_KEY" in os.environ ), "Make sure to specify LlamaParse API key." - doc_lang = detect_language_from_doc(file_path) - doc_lang = LLAMAPARSE_SUPPORTED_LANGS[doc_lang[0]] - for _ in range(LLAMA_PARSE_MAX_RETRY): try: documents = LlamaParse( result_type="markdown", verbose=True, - language=doc_lang, + languages=list( + LLAMAPARSE_SUPPORTED_LANGS.values() + ), accurate_mode=True, ).load_data(file_path) assert len(documents) > 0 diff --git a/fastchat/serve/setup_pdfchat.sh b/fastchat/serve/setup_pdfchat.sh index e2d3fbc15..2511b939d 100644 --- a/fastchat/serve/setup_pdfchat.sh +++ b/fastchat/serve/setup_pdfchat.sh @@ -2,18 +2,4 @@ # Install Python packages pip install llama-index-core llama-parse llama-index-readers-file python-dotenv -pip install polyglot -pip install PyICU -pip install pycld2 -pip install pytesseract - -pip install pdf2image - -# Clone the Tesseract tessdata repository -git clone https://github.com/tesseract-ocr/tessdata - -# cd into tessdata and set TESSDATA_PREFIX to the current directory -cd tessdata -export TESSDATA_PREFIX="$(pwd)" - -echo "TESSDATA_PREFIX is set to: $TESSDATA_PREFIX" \ No newline at end of file +pip install pdf2image \ No newline at end of file From e4c0f3b2a371b97247f950feb56b539aa5dea3b3 Mon Sep 17 00:00:00 2001 From: CodingWithTim Date: Sun, 5 Jan 2025 01:47:46 +0000 Subject: [PATCH 17/40] fix format --- fastchat/serve/gradio_block_arena_vision.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/fastchat/serve/gradio_block_arena_vision.py b/fastchat/serve/gradio_block_arena_vision.py index bd745791f..40ba51bc9 100644 --- a/fastchat/serve/gradio_block_arena_vision.py +++ b/fastchat/serve/gradio_block_arena_vision.py @@ -258,9 +258,7 @@ def parse_pdf(file_path): documents = LlamaParse( result_type="markdown", verbose=True, - languages=list( - LLAMAPARSE_SUPPORTED_LANGS.values() - ), + languages=list(LLAMAPARSE_SUPPORTED_LANGS.values()), accurate_mode=True, ).load_data(file_path) assert len(documents) > 0 From 5c52665cb21b3a4f2d831d71d70680b1ce1e438e Mon Sep 17 00:00:00 2001 From: CodingWithTim Date: Mon, 6 Jan 2025 01:14:11 +0000 Subject: [PATCH 18/40] support multimodal pdfchat and switch to marker pdf --- fastchat/conversation.py | 33 +++++- fastchat/serve/gradio_block_arena_vision.py | 118 ++++++++++++++++---- fastchat/serve/vision/image.py | 21 +++- 3 files changed, 139 insertions(+), 33 deletions(-) diff --git a/fastchat/conversation.py b/fastchat/conversation.py index 4a46103ec..d81477b22 100644 --- a/fastchat/conversation.py +++ b/fastchat/conversation.py @@ -362,18 +362,39 @@ def update_last_message(self, message: str): def to_gradio_chatbot(self): """Convert the conversation to gradio chatbot format.""" from fastchat.serve.vision.image import ImageFormat + import re ret = [] for i, (role, msg) in enumerate(self.messages[self.offset :]): if i % 2 == 0: if type(msg) is tuple: msg, images = msg - image = images[0] # Only one image on gradio at one time - if image.image_format == ImageFormat.URL: - img_str = f'user upload image' - elif image.image_format == ImageFormat.BYTES: - img_str = f'user upload image' - msg = img_str + msg.replace("\n", "").strip() + + pattern = re.compile("!\[\]\(_page_\d_Figure_\d\.jpeg\)") + embed_locations = pattern.findall(msg) + + pdfchat = False + for i, embed_str in enumerate(embed_locations): + if i >= len(images): + break + + image = images[i] + msg = msg.replace( + embed_str, + f'document image', + ) + pdfchat = True + + if not pdfchat: + # vision arena only supports one image on gradio at one time + image = images[0] + if image.image_format == ImageFormat.URL: + img_str = ( + f'user upload image' + ) + elif image.image_format == ImageFormat.BYTES: + img_str = f'user upload image' + msg = img_str + msg.replace("\n", "").strip() ret.append([msg, None]) else: diff --git a/fastchat/serve/gradio_block_arena_vision.py b/fastchat/serve/gradio_block_arena_vision.py index 40ba51bc9..0664a9b06 100644 --- a/fastchat/serve/gradio_block_arena_vision.py +++ b/fastchat/serve/gradio_block_arena_vision.py @@ -232,10 +232,51 @@ def wrap_pdfchat_query(query, document): return reformatted_query_context -LLAMA_PARSE_MAX_RETRY = 2 -LLAMAPARSE_SUPPORTED_LANGS = { +# LLAMA_PARSE_MAX_RETRY = 2 +# LLAMAPARSE_SUPPORTED_LANGS = { +# "English": "en", +# "Chinese": "ch_sim", +# "Russian": "ru", +# "Spanish": "es", +# "Japanese": "ja", +# "Korean": "ko", +# "French": "fr", +# "German": "de", +# "Vietnamese": "vi", +# } + + +# def parse_pdf(file_path): +# from llama_parse import LlamaParse + +# assert ( +# "LLAMA_CLOUD_API_KEY" in os.environ +# ), "Make sure to specify LlamaParse API key." + +# for _ in range(LLAMA_PARSE_MAX_RETRY): +# try: +# documents = LlamaParse( +# result_type="markdown", +# verbose=True, +# languages=list(LLAMAPARSE_SUPPORTED_LANGS.values()), +# accurate_mode=True, +# ).load_data(file_path) +# assert len(documents) > 0 +# break +# except AssertionError as e: +# continue + +# output = "\n".join( +# [f"Page {i+1}:\n{doc.text}\n" for i, doc in enumerate(documents)] +# ) + +# return output + + +PDFPARSE_MAX_RETRY = 2 +PDFPARSE_SUPPORTED_LANGS = { "English": "en", - "Chinese": "ch_sim", + "Chinese": "zh", "Russian": "ru", "Spanish": "es", "Japanese": "ja", @@ -244,33 +285,36 @@ def wrap_pdfchat_query(query, document): "German": "de", "Vietnamese": "vi", } +MARKER_PDFPARSE_CONFIG = { + "output_format": "markdown", + "languages": ",".join(PDFPARSE_SUPPORTED_LANGS.values()), +} def parse_pdf(file_path): - from llama_parse import LlamaParse - - assert ( - "LLAMA_CLOUD_API_KEY" in os.environ - ), "Make sure to specify LlamaParse API key." + from marker.config.parser import ConfigParser + from marker.models import create_model_dict + from marker.converters.pdf import PdfConverter - for _ in range(LLAMA_PARSE_MAX_RETRY): + output_md, output_images = None, None + for _ in range(PDFPARSE_MAX_RETRY): try: - documents = LlamaParse( - result_type="markdown", - verbose=True, - languages=list(LLAMAPARSE_SUPPORTED_LANGS.values()), - accurate_mode=True, - ).load_data(file_path) - assert len(documents) > 0 + config_parser = ConfigParser(MARKER_PDFPARSE_CONFIG) + + converter = PdfConverter( + config=config_parser.generate_config_dict(), + artifact_dict=create_model_dict(), + processor_list=config_parser.get_processors(), + renderer=config_parser.get_renderer(), + ) + rendered = converter(file_path) + output_md = rendered.markdown + output_images = list(rendered.images.values()) break except AssertionError as e: continue - output = "\n".join( - [f"Page {i+1}:\n{doc.text}\n" for i, doc in enumerate(documents)] - ) - - return output + return output_md, output_images def _prepare_text_with_image(state, text, images, csam_flag): @@ -284,12 +328,26 @@ def _prepare_text_with_image(state, text, images, csam_flag): return text +# def _prepare_text_with_pdf(text, pdfs): +# if len(pdfs) > 0: +# document_content = parse_pdf(pdfs[0]) +# print("Document processed") +# text = wrap_pdfchat_query(text, document_content) + +# return text + + def _prepare_text_with_pdf(text, pdfs): if len(pdfs) > 0: - document_content = parse_pdf(pdfs[0]) + parsed_text, imgs = parse_pdf(pdfs[0]) print("Document processed") - text = wrap_pdfchat_query(text, document_content) + wrapped_text = wrap_pdfchat_query(text, parsed_text) + imgs = convert_pdf_images_to_conversation_format(imgs) + + if len(imgs) > 0: + return wrapped_text, imgs + return wrapped_text return text @@ -307,6 +365,20 @@ def convert_images_to_conversation_format(images): return conv_images +def convert_pdf_images_to_conversation_format(images): + MAX_NSFW_ENDPOINT_IMAGE_SIZE_IN_MB = 5 / 1.5 + conv_images = [] + if len(images) > 0: + for img in images: + # pdf parser returns a PIL image object instead of path + conv_images.append( + Image(url="").to_conversation_format( + MAX_NSFW_ENDPOINT_IMAGE_SIZE_IN_MB, pil_img=img + ) + ) + return conv_images + + def moderate_input(state, text, all_conv_text, model_list, images, ip): text_flagged = moderation_filter(all_conv_text, model_list) # flagged = moderation_filter(text, [state.model_name]) diff --git a/fastchat/serve/vision/image.py b/fastchat/serve/vision/image.py index 58bedf8f0..b4d740b66 100644 --- a/fastchat/serve/vision/image.py +++ b/fastchat/serve/vision/image.py @@ -1,7 +1,6 @@ import base64 from enum import auto, IntEnum from io import BytesIO - from pydantic import BaseModel @@ -115,11 +114,25 @@ def convert_url_to_image_bytes(self, max_image_size_mb): return image_format, img_base64_str - def to_conversation_format(self, max_image_size_mb): - image_format, image_bytes = self.convert_url_to_image_bytes( - max_image_size_mb=max_image_size_mb + def convert_pil_image_to_image_bytes(self, pil_img, max_image_size_mb): + image_format, image_bytes = self.resize_image_and_return_image_in_bytes( + pil_img, max_image_size_mb ) + img_base64_str = base64.b64encode(image_bytes.getvalue()).decode() + + return image_format, img_base64_str + + def to_conversation_format(self, max_image_size_mb, pil_img=None): + if pil_img: + image_format, image_bytes = self.convert_pil_image_to_image_bytes( + pil_img=pil_img, max_image_size_mb=max_image_size_mb + ) + else: + image_format, image_bytes = self.convert_url_to_image_bytes( + max_image_size_mb=max_image_size_mb + ) + self.filetype = image_format self.image_format = ImageFormat.BYTES self.base64_str = image_bytes From 61284e0b103dcc2b75ab5bcb32eef39c62d9f329 Mon Sep 17 00:00:00 2001 From: CodingWithTim Date: Mon, 6 Jan 2025 01:24:36 +0000 Subject: [PATCH 19/40] switch to package implementation of 'is_image' --- fastchat/serve/gradio_block_arena_vision.py | 33 ------------------- .../serve/gradio_block_arena_vision_anony.py | 4 +-- 2 files changed, 2 insertions(+), 35 deletions(-) diff --git a/fastchat/serve/gradio_block_arena_vision.py b/fastchat/serve/gradio_block_arena_vision.py index 0664a9b06..6f209bcb2 100644 --- a/fastchat/serve/gradio_block_arena_vision.py +++ b/fastchat/serve/gradio_block_arena_vision.py @@ -74,39 +74,6 @@ def get_vqa_sample(): return (res, path) -def is_image(file_path): - magic_numbers = { - b"\xff\xd8\xff": "JPEG", - b"\x89PNG\r\n\x1a\n": "PNG", - b"GIF87a": "GIF", - b"GIF89a": "GIF", - b"BM": "BMP", - b"\x00\x00\x01\x00": "ICO", - b"\x49\x49\x2a\x00": "TIFF", - b"\x4d\x4d\x00\x2a": "TIFF", - # For WebP, the first four bytes are "RIFF", but we also check for "WEBP" - # in bytes 8–12. - } - - try: - with open(file_path, "rb") as f: - header = f.read(16) # Read a bit more to handle WebP safely - - # Check for WebP (RIFF + WEBP) - if header.startswith(b"RIFF") and header[8:12] == b"WEBP": - return True - - # Check other formats - for magic in magic_numbers: - if header.startswith(magic): - return True - - return False - except Exception as e: - print(f"Error reading file: {e}") - return False - - def is_pdf(file_path): try: with open(file_path, "rb") as file: diff --git a/fastchat/serve/gradio_block_arena_vision_anony.py b/fastchat/serve/gradio_block_arena_vision_anony.py index ca74303ab..f075e7fb1 100644 --- a/fastchat/serve/gradio_block_arena_vision_anony.py +++ b/fastchat/serve/gradio_block_arena_vision_anony.py @@ -5,6 +5,7 @@ import json import time +import filetype import gradio as gr import numpy as np @@ -71,7 +72,6 @@ invisible_text, visible_text, disable_multimodal, - is_image, is_pdf, ) from fastchat.serve.gradio_global_state import Context @@ -276,7 +276,7 @@ def add_text( states = [state0, state1] model_selectors = [model_selector0, model_selector1] - images = [file for file in files if is_image(file)] + images = [file for file in files if filetype.is_image(file)] pdfs = [file for file in files if is_pdf(file)] # Init states if necessary From 586a2f6566a26473c6109b0bdc7d065ae436fe92 Mon Sep 17 00:00:00 2001 From: CodingWithTim Date: Mon, 6 Jan 2025 01:27:17 +0000 Subject: [PATCH 20/40] flexible state variable for pdf_id --- fastchat/serve/gradio_web_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastchat/serve/gradio_web_server.py b/fastchat/serve/gradio_web_server.py index ff10e4580..14feaddd7 100644 --- a/fastchat/serve/gradio_web_server.py +++ b/fastchat/serve/gradio_web_server.py @@ -154,7 +154,7 @@ def dict(self): { "conv_id": self.conv_id, "model_name": self.model_name, - "pdf_id": self.pdf_id, + "pdf_id": [self.pdf_id], } ) From 2ea729cf4258ff765278bea1a6178367dec85192 Mon Sep 17 00:00:00 2001 From: CodingWithTim Date: Mon, 6 Jan 2025 01:34:59 +0000 Subject: [PATCH 21/40] fix error --- fastchat/serve/gradio_block_arena_vision.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fastchat/serve/gradio_block_arena_vision.py b/fastchat/serve/gradio_block_arena_vision.py index 6f209bcb2..981632e98 100644 --- a/fastchat/serve/gradio_block_arena_vision.py +++ b/fastchat/serve/gradio_block_arena_vision.py @@ -85,14 +85,16 @@ def is_pdf(file_path): def set_visible_image(textbox): + import filetype + files = textbox["files"] if len(files) == 0: return invisible_image_column elif len(files) > 1: gr.Warning( - "We only support single image conversations. Please start a new round if you would like to chat using this image." + "We only support single image or document conversations. Please start a new round if you would like to chat using this image or document." ) - elif is_image(files[0]): + elif filetype.is_image(files[0]): return visible_image_column return invisible_image_column From f2c4d6422797591ac06a10dc1a872cfeaae57e89 Mon Sep 17 00:00:00 2001 From: PranavB-11 Date: Fri, 31 Jan 2025 07:59:33 +0000 Subject: [PATCH 22/40] Marker API Implemented + Updated Llama code if ever needed --- fastchat/serve/gradio_block_arena_vision.py | 111 +++++++++++------- .../serve/gradio_block_arena_vision_anony.py | 3 +- fastchat/serve/test.ipynb | 104 ++++++++++++++++ 3 files changed, 177 insertions(+), 41 deletions(-) create mode 100644 fastchat/serve/test.ipynb diff --git a/fastchat/serve/gradio_block_arena_vision.py b/fastchat/serve/gradio_block_arena_vision.py index 981632e98..a7273f57f 100644 --- a/fastchat/serve/gradio_block_arena_vision.py +++ b/fastchat/serve/gradio_block_arena_vision.py @@ -16,6 +16,9 @@ from gradio.data_classes import FileData import numpy as np +from io import BytesIO +import base64 + from fastchat.constants import ( TEXT_MODERATION_MSG, IMAGE_MODERATION_MSG, @@ -217,29 +220,38 @@ def wrap_pdfchat_query(query, document): # def parse_pdf(file_path): # from llama_parse import LlamaParse +# from llama_index.core.schema import ImageDocument, TextNode + +# from PIL import Image -# assert ( -# "LLAMA_CLOUD_API_KEY" in os.environ -# ), "Make sure to specify LlamaParse API key." - -# for _ in range(LLAMA_PARSE_MAX_RETRY): -# try: -# documents = LlamaParse( -# result_type="markdown", -# verbose=True, -# languages=list(LLAMAPARSE_SUPPORTED_LANGS.values()), -# accurate_mode=True, -# ).load_data(file_path) -# assert len(documents) > 0 -# break -# except AssertionError as e: -# continue - -# output = "\n".join( -# [f"Page {i+1}:\n{doc.text}\n" for i, doc in enumerate(documents)] +# parser = LlamaParse( +# api_key=os.getenv("LLAMA_CLOUD_API_KEY"), +# result_type="markdown", # ) -# return output +# def get_image_nodes(json_objs: List[dict], download_path: str): +# image_dicts = parser.get_images(json_objs, download_path=download_path) +# return [ImageDocument(image_path=image_dict["path"]) for image_dict in image_dicts] + +# json_objs = parser.get_json_result(file_path) +# json_list = json_objs[0]["pages"] + +# text = "" +# for page in json_list: +# text += f"Page {page['page']}:\n{page['md']}\n" +# if (page['images']): +# for i, image in enumerate(page['images']): +# text += f"page{page['page']}_figure{i + 1}\n" + +# image_documents = get_image_nodes(json_objs, ".") +# images = [] + +# for image_doc in image_documents: +# image_path = image_doc.image_path +# image = Image.open(image_path) +# images.append(image) + +# return text, images PDFPARSE_MAX_RETRY = 2 @@ -259,29 +271,48 @@ def wrap_pdfchat_query(query, document): "languages": ",".join(PDFPARSE_SUPPORTED_LANGS.values()), } +def convert_base64_to_pil_image(b64_string): + from PIL import Image + + image_data = base64.b64decode(b64_string) + image_bytes = BytesIO(image_data) + image = Image.open(image_bytes) + + return image def parse_pdf(file_path): - from marker.config.parser import ConfigParser - from marker.models import create_model_dict - from marker.converters.pdf import PdfConverter - - output_md, output_images = None, None - for _ in range(PDFPARSE_MAX_RETRY): - try: - config_parser = ConfigParser(MARKER_PDFPARSE_CONFIG) - - converter = PdfConverter( - config=config_parser.generate_config_dict(), - artifact_dict=create_model_dict(), - processor_list=config_parser.get_processors(), - renderer=config_parser.get_renderer(), - ) - rendered = converter(file_path) - output_md = rendered.markdown - output_images = list(rendered.images.values()) + import requests + + url = "https://www.datalab.to/api/v1/marker" + + form_data = { + 'file': ('test.pdf', open(file_path, 'rb'), 'application/pdf'), + 'langs': (None, "English"), + "force_ocr": (None, False), + "paginate": (None, False), + 'output_format': (None, 'markdown'), + "use_llm": (None, True), + "strip_existing_ocr": (None, False), + "disable_image_extraction": (None, False) + } + + headers = {"X-Api-Key": os.getenv("X-Api-Key")} + response = requests.post(url, files=form_data, headers=headers) + data = response.json() + + max_polls = 300 + check_url = data["request_check_url"] + + for i in range(max_polls): + time.sleep(2) + response = requests.get(check_url, headers=headers) + data = response.json() + + if data["status"] == "complete": break - except AssertionError as e: - continue + + output_md = data["markdown"] + output_images = [convert_base64_to_pil_image(b64_image) for b64_image in data["images"].values()] return output_md, output_images diff --git a/fastchat/serve/gradio_block_arena_vision_anony.py b/fastchat/serve/gradio_block_arena_vision_anony.py index f075e7fb1..2227c07c7 100644 --- a/fastchat/serve/gradio_block_arena_vision_anony.py +++ b/fastchat/serve/gradio_block_arena_vision_anony.py @@ -95,7 +95,8 @@ # TODO(chris): fix sampling weights VISION_SAMPLING_WEIGHTS = {} -PDFCHAT_SAMPLING_WEIGHTS = {} +PDFCHAT_SAMPLING_WEIGHTS = { + "gpt-4o-2024-05-13": 1, "gpt-4o-mini-2024-07-18": 1} # TODO(chris): Find battle targets that make sense VISION_BATTLE_TARGETS = {} diff --git a/fastchat/serve/test.ipynb b/fastchat/serve/test.ipynb new file mode 100644 index 000000000..41ba70d69 --- /dev/null +++ b/fastchat/serve/test.ipynb @@ -0,0 +1,104 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "import requests\n", + "\n", + "url = \"https://www.datalab.to/api/v1/marker\"\n", + "import os\n", + "form_data = {\n", + " 'file': ('test.pdf', open(\"ddsppaper (1).pdf\", 'rb'), 'application/pdf'),\n", + " 'langs': (None, \"English\"),\n", + " \"force_ocr\": (None, False),\n", + " \"paginate\": (None, False),\n", + " 'output_format': (None, 'markdown'),\n", + " \"use_llm\": (None, True),\n", + " \"strip_existing_ocr\": (None, False),\n", + " \"disable_image_extraction\": (None, False)\n", + "}\n", + "\n", + "headers = {\"X-Api-Key\": \"wAdzo2tLEsd5PzQTtQT4RNZSBM6rJy_LWFTtj8hjbZ0\"} \n", + "response = requests.post(url, files=form_data, headers=headers)\n", + "data = response.json()" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'detail': 'Not authenticated'}\n" + ] + } + ], + "source": [ + "print(data)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "ename": "KeyError", + "evalue": "'request_check_url'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[4], line 2\u001b[0m\n\u001b[1;32m 1\u001b[0m max_polls \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m300\u001b[39m\n\u001b[0;32m----> 2\u001b[0m check_url \u001b[38;5;241m=\u001b[39m data[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mrequest_check_url\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[1;32m 3\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mtime\u001b[39;00m\n\u001b[1;32m 4\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m i \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(max_polls):\n", + "\u001b[0;31mKeyError\u001b[0m: 'request_check_url'" + ] + } + ], + "source": [ + "max_polls = 300\n", + "check_url = data[\"request_check_url\"]\n", + "import time\n", + "for i in range(max_polls):\n", + " time.sleep(2)\n", + " response = requests.get(check_url, headers=headers)\n", + " data = response.json()\n", + "\n", + " if data[\"status\"] == \"complete\":\n", + " break" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.12.8 ('myenv')", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.8" + }, + "orig_nbformat": 4, + "vscode": { + "interpreter": { + "hash": "03c457903a5d26c69a3bb8be9c56ac1ee96fb7ba834b2e69a22fb0607b146481" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From cf9b4084c63c9ff12cacf4aefb5eee13bc480951 Mon Sep 17 00:00:00 2001 From: PranavB-11 Date: Tue, 11 Feb 2025 06:08:35 +0000 Subject: [PATCH 23/40] Content Moderation implemented + couple of latency improvements --- fastchat/serve/gradio_block_arena_vision.py | 13 +++++++--- .../serve/gradio_block_arena_vision_anony.py | 25 ++++++++++++------- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/fastchat/serve/gradio_block_arena_vision.py b/fastchat/serve/gradio_block_arena_vision.py index a7273f57f..5bb86826f 100644 --- a/fastchat/serve/gradio_block_arena_vision.py +++ b/fastchat/serve/gradio_block_arena_vision.py @@ -273,13 +273,19 @@ def wrap_pdfchat_query(query, document): def convert_base64_to_pil_image(b64_string): from PIL import Image + import numpy as np - image_data = base64.b64decode(b64_string) + image_data = np.frombuffer(base64.b64decode(b64_string), dtype=np.uint8) image_bytes = BytesIO(image_data) image = Image.open(image_bytes) return image +def batch_convert_base64_to_images(base64_dict): + import concurrent.futures + with concurrent.futures.ThreadPoolExecutor() as executor: + return list(executor.map(convert_base64_to_pil_image, base64_dict.values())) + def parse_pdf(file_path): import requests @@ -287,7 +293,6 @@ def parse_pdf(file_path): form_data = { 'file': ('test.pdf', open(file_path, 'rb'), 'application/pdf'), - 'langs': (None, "English"), "force_ocr": (None, False), "paginate": (None, False), 'output_format': (None, 'markdown'), @@ -296,7 +301,7 @@ def parse_pdf(file_path): "disable_image_extraction": (None, False) } - headers = {"X-Api-Key": os.getenv("X-Api-Key")} + headers = {"X-Api-Key": str(os.getenv("MARKER_API_KEY"))} response = requests.post(url, files=form_data, headers=headers) data = response.json() @@ -312,7 +317,7 @@ def parse_pdf(file_path): break output_md = data["markdown"] - output_images = [convert_base64_to_pil_image(b64_image) for b64_image in data["images"].values()] + output_images = batch_convert_base64_to_images(data["images"]) return output_md, output_images diff --git a/fastchat/serve/gradio_block_arena_vision_anony.py b/fastchat/serve/gradio_block_arena_vision_anony.py index 2227c07c7..3efbba99d 100644 --- a/fastchat/serve/gradio_block_arena_vision_anony.py +++ b/fastchat/serve/gradio_block_arena_vision_anony.py @@ -312,10 +312,6 @@ def add_text( State(model_left, is_vision=False, pdf_id=unique_id), State(model_right, is_vision=False, pdf_id=unique_id), ] - upload_pdf_file_to_gcs( - pdf_file_path=pdfs[0], - filename=unique_id, - ) else: model_left, model_right = get_battle_pair( context.all_text_models, @@ -366,10 +362,17 @@ def add_text( images = convert_images_to_conversation_format(images) - # TODO: add PDF moderator - text, image_flagged, csam_flag = moderate_input( - state0, text, text, model_list, images, ip - ) + post_processed_text = _prepare_text_with_pdf(text[:BLIND_MODE_INPUT_CHAR_LEN_LIMIT], pdfs) + if type(post_processed_text) is tuple: + text += post_processed_text[0] + text, image_flagged, csam_flag = moderate_input( + state0, text, text, model_list, images + post_processed_text[1], ip + ) + else: + text += post_processed_text + text, image_flagged, csam_flag = moderate_input( + state0, text, text, model_list, images, ip + ) conv = states[0].conv if (len(conv.messages) - conv.offset) // 2 >= CONVERSATION_TURN_LIMIT: @@ -408,7 +411,11 @@ def add_text( ) text = text[:BLIND_MODE_INPUT_CHAR_LEN_LIMIT] # Hard cut-off - post_processed_text = _prepare_text_with_pdf(text, pdfs) + + upload_pdf_file_to_gcs( + pdf_file_path=pdfs[0], + filename=unique_id, + ) for i in range(num_sides): post_processed_text = _prepare_text_with_image( From 2bf158ceb6454480eb28a52a21d6e3395d0b8c99 Mon Sep 17 00:00:00 2001 From: Andrew Wan Date: Wed, 12 Feb 2025 10:10:40 -0800 Subject: [PATCH 24/40] fixed bug where text moderation wasn't being flagged --- .../serve/gradio_block_arena_vision_anony.py | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/fastchat/serve/gradio_block_arena_vision_anony.py b/fastchat/serve/gradio_block_arena_vision_anony.py index 3efbba99d..8093f206c 100644 --- a/fastchat/serve/gradio_block_arena_vision_anony.py +++ b/fastchat/serve/gradio_block_arena_vision_anony.py @@ -365,12 +365,12 @@ def add_text( post_processed_text = _prepare_text_with_pdf(text[:BLIND_MODE_INPUT_CHAR_LEN_LIMIT], pdfs) if type(post_processed_text) is tuple: text += post_processed_text[0] - text, image_flagged, csam_flag = moderate_input( + text, text_flagged, image_flagged, csam_flag = moderate_input( state0, text, text, model_list, images + post_processed_text[1], ip ) else: text += post_processed_text - text, image_flagged, csam_flag = moderate_input( + text, text_flagged, image_flagged, csam_flag = moderate_input( state0, text, text, model_list, images, ip ) @@ -409,6 +409,25 @@ def add_text( + [no_change_btn] * 7 + [""] ) + + if text_flagged: + logger.info(f"image flagged. ip: {ip}. text: {text}") + for i in range(num_sides): + states[i].skip_next = True + return ( + states + + [x.to_gradio_chatbot() for x in states] + + [ + { + "text": TEXT_MODERATION_MSG + + " PLEASE CLICK 🎲 NEW ROUND TO START A NEW CONVERSATION." + }, + "", + no_change_btn, + ] + + [no_change_btn] * 7 + + [""] + ) text = text[:BLIND_MODE_INPUT_CHAR_LEN_LIMIT] # Hard cut-off From 06110d2395231f1389ddeded3c68c83de21434fe Mon Sep 17 00:00:00 2001 From: Andrew Wan Date: Wed, 12 Feb 2025 10:10:59 -0800 Subject: [PATCH 25/40] fixed bug where text moderation wasn't being flagged --- fastchat/serve/gradio_block_arena_vision.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastchat/serve/gradio_block_arena_vision.py b/fastchat/serve/gradio_block_arena_vision.py index 5bb86826f..e94c6302e 100644 --- a/fastchat/serve/gradio_block_arena_vision.py +++ b/fastchat/serve/gradio_block_arena_vision.py @@ -406,7 +406,7 @@ def moderate_input(state, text, all_conv_text, model_list, images, ip): state.has_csam_image = True report_csam_image(state, images[0]) - return text, image_flagged, csam_flagged + return text, text_flagged, image_flagged, csam_flagged def add_text( From 34c7a8ed53043e3bdd3a23064b9d6802bf8a8265 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cyixin-huang1=E2=80=9D?= Date: Wed, 12 Feb 2025 17:35:40 -0800 Subject: [PATCH 26/40] added image_resize functionality for image moderation --- fastchat/utils.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/fastchat/utils.py b/fastchat/utils.py index 88ae37c7e..60527099d 100644 --- a/fastchat/utils.py +++ b/fastchat/utils.py @@ -18,6 +18,8 @@ from fastchat.constants import LOGDIR +from PIL import Image + handler = None visited_loggers = set() @@ -392,7 +394,6 @@ def str_to_torch_dtype(dtype: str): def load_image(image_file): - from PIL import Image import requests image = None @@ -481,6 +482,18 @@ def get_pdf_num_page(file_path): return len(reader.pages) +def resize_image_if_needed(image_bytes): + image = Image.open(BytesIO(image_bytes)) + if min(image.size) < 128: + print(f"⚠️ Resizing image from {image.size} to minimum required size") + new_size = (max(128, image.size[0]), max(128, image.size[1])) + image = image.resize(new_size, Image.LANCZOS) + + img_byte_arr = BytesIO() + image.save(img_byte_arr, format="PNG") + return img_byte_arr.getvalue() + + return image_bytes def image_moderation_request(image_bytes, endpoint, api_key): headers = {"Content-Type": "image/jpeg", "Ocp-Apim-Subscription-Key": api_key} @@ -516,6 +529,7 @@ def image_moderation_filter(image): print(f"moderating image") image_bytes = base64.b64decode(image.base64_str) + image_bytes = resize_image_if_needed(image_bytes) nsfw_flagged = image_moderation_provider(image_bytes, "nsfw") csam_flagged = False From 0955a76ea3546229a67ea20d31d21fcc77dae311 Mon Sep 17 00:00:00 2001 From: Andrew Wan Date: Thu, 13 Feb 2025 22:45:02 +0000 Subject: [PATCH 27/40] minor text fix --- fastchat/serve/gradio_block_arena_vision_anony.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastchat/serve/gradio_block_arena_vision_anony.py b/fastchat/serve/gradio_block_arena_vision_anony.py index 8093f206c..d885264ab 100644 --- a/fastchat/serve/gradio_block_arena_vision_anony.py +++ b/fastchat/serve/gradio_block_arena_vision_anony.py @@ -411,7 +411,7 @@ def add_text( ) if text_flagged: - logger.info(f"image flagged. ip: {ip}. text: {text}") + logger.info(f"text flagged. ip: {ip}. text: {text}") for i in range(num_sides): states[i].skip_next = True return ( From 701f7c5da8ce760cd8da7e9ee32d9a065df25a02 Mon Sep 17 00:00:00 2001 From: Andrew Wan Date: Fri, 14 Feb 2025 00:26:43 +0000 Subject: [PATCH 28/40] applied black formatting py3.10 --- fastchat/model/apply_delta.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/fastchat/model/apply_delta.py b/fastchat/model/apply_delta.py index 4bfeb0742..776853f00 100644 --- a/fastchat/model/apply_delta.py +++ b/fastchat/model/apply_delta.py @@ -78,7 +78,10 @@ def apply_delta_low_cpu_mem(base_model_path, target_model_path, delta_path): split_size = 4 * GB - with tempfile.TemporaryDirectory() as tmp_base_path, tempfile.TemporaryDirectory() as tmp_delta_path: + with ( + tempfile.TemporaryDirectory() as tmp_base_path, + tempfile.TemporaryDirectory() as tmp_delta_path, + ): print(f"Split files for the base model to {tmp_base_path}") split_files(base_model_path, tmp_base_path, split_size) print(f"Split files for the delta weights to {tmp_delta_path}") From ab2443cc82ed627485fbdf8712aaf6dca000a1e0 Mon Sep 17 00:00:00 2001 From: Andrew Wan Date: Fri, 14 Feb 2025 02:19:12 +0000 Subject: [PATCH 29/40] fixed black version --- fastchat/llm_judge/qa_browser.py | 12 ++++++------ fastchat/model/model_adapter.py | 6 +++--- fastchat/serve/monitor/monitor.py | 4 +++- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/fastchat/llm_judge/qa_browser.py b/fastchat/llm_judge/qa_browser.py index 9d9d2125a..1107756db 100644 --- a/fastchat/llm_judge/qa_browser.py +++ b/fastchat/llm_judge/qa_browser.py @@ -405,12 +405,12 @@ def build_demo(): model_answers = load_model_answers(answer_dir) # Load model judgments - model_judgments_normal_single = model_judgments_math_single = ( - load_single_model_judgments(single_model_judgment_file) - ) - model_judgments_normal_pairwise = model_judgments_math_pairwise = ( - load_pairwise_model_judgments(pairwise_model_judgment_file) - ) + model_judgments_normal_single = ( + model_judgments_math_single + ) = load_single_model_judgments(single_model_judgment_file) + model_judgments_normal_pairwise = ( + model_judgments_math_pairwise + ) = load_pairwise_model_judgments(pairwise_model_judgment_file) demo = build_demo() demo.queue( diff --git a/fastchat/model/model_adapter.py b/fastchat/model/model_adapter.py index 6353277b9..16cf5d2b6 100644 --- a/fastchat/model/model_adapter.py +++ b/fastchat/model/model_adapter.py @@ -239,9 +239,9 @@ def load_model( if num_gpus != 1: kwargs["device_map"] = "auto" if max_gpu_memory is None: - kwargs["device_map"] = ( - "sequential" # This is important for not the same VRAM sizes - ) + kwargs[ + "device_map" + ] = "sequential" # This is important for not the same VRAM sizes available_gpu_memory = get_gpu_memory(num_gpus) kwargs["max_memory"] = { i: str(int(available_gpu_memory[i] * 0.85)) + "GiB" diff --git a/fastchat/serve/monitor/monitor.py b/fastchat/serve/monitor/monitor.py index 8210d13b9..f36fbf278 100644 --- a/fastchat/serve/monitor/monitor.py +++ b/fastchat/serve/monitor/monitor.py @@ -369,7 +369,9 @@ def highlight_rank_max(s): ( "color: green; font-weight: bold" if v > 0 - else "color: red; font-weight: bold" if v < 0 else "" + else "color: red; font-weight: bold" + if v < 0 + else "" ) for v in s ] From 89743ad190f43b34fe7b6a396b6515736815f11c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cyixin-huang1=E2=80=9D?= Date: Sat, 15 Feb 2025 22:23:30 -0800 Subject: [PATCH 30/40] Revert "minor text fix" This reverts commit 0955a76ea3546229a67ea20d31d21fcc77dae311. --- fastchat/serve/gradio_block_arena_vision_anony.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastchat/serve/gradio_block_arena_vision_anony.py b/fastchat/serve/gradio_block_arena_vision_anony.py index 543523cda..4d84f2025 100644 --- a/fastchat/serve/gradio_block_arena_vision_anony.py +++ b/fastchat/serve/gradio_block_arena_vision_anony.py @@ -412,7 +412,7 @@ def add_text( ) if text_flagged: - logger.info(f"text flagged. ip: {ip}. text: {text}") + logger.info(f"image flagged. ip: {ip}. text: {text}") for i in range(num_sides): states[i].skip_next = True return ( From 63cb5554f37c09fe67764e93873f2dfe61a713bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cyixin-huang1=E2=80=9D?= Date: Sat, 15 Feb 2025 22:32:52 -0800 Subject: [PATCH 31/40] revert + fix some PR issues --- fastchat/serve/gradio_block_arena_vision.py | 71 ---- .../serve/gradio_block_arena_vision_anony.py | 5 +- fastchat/serve/test.ipynb | 106 ----- playground/FastChat_API_GoogleColab.ipynb | 374 ------------------ 4 files changed, 1 insertion(+), 555 deletions(-) delete mode 100644 fastchat/serve/test.ipynb delete mode 100644 playground/FastChat_API_GoogleColab.ipynb diff --git a/fastchat/serve/gradio_block_arena_vision.py b/fastchat/serve/gradio_block_arena_vision.py index 63ca999e1..ee7964c15 100644 --- a/fastchat/serve/gradio_block_arena_vision.py +++ b/fastchat/serve/gradio_block_arena_vision.py @@ -177,20 +177,11 @@ def clear_history_example(request: gr.Request): ) * 5 -# TODO(Chris): At some point, we would like this to be a live-reporting feature. def report_csam_image(state, image): pass def wrap_pdfchat_query(query, document): - # TODO: Considering redesign the context format. - # document_context = f""" - # The following is the content of a document: - # {document} - # Based on this document, answer the following question: - # {query} - # """ - reformatted_query_context = ( f"Answer the user query given the context.\n" f"[QUERY CONTEXT]\n" @@ -203,57 +194,6 @@ def wrap_pdfchat_query(query, document): return reformatted_query_context - -# LLAMA_PARSE_MAX_RETRY = 2 -# LLAMAPARSE_SUPPORTED_LANGS = { -# "English": "en", -# "Chinese": "ch_sim", -# "Russian": "ru", -# "Spanish": "es", -# "Japanese": "ja", -# "Korean": "ko", -# "French": "fr", -# "German": "de", -# "Vietnamese": "vi", -# } - - -# def parse_pdf(file_path): -# from llama_parse import LlamaParse -# from llama_index.core.schema import ImageDocument, TextNode - -# from PIL import Image - -# parser = LlamaParse( -# api_key=os.getenv("LLAMA_CLOUD_API_KEY"), -# result_type="markdown", -# ) - -# def get_image_nodes(json_objs: List[dict], download_path: str): -# image_dicts = parser.get_images(json_objs, download_path=download_path) -# return [ImageDocument(image_path=image_dict["path"]) for image_dict in image_dicts] - -# json_objs = parser.get_json_result(file_path) -# json_list = json_objs[0]["pages"] - -# text = "" -# for page in json_list: -# text += f"Page {page['page']}:\n{page['md']}\n" -# if (page['images']): -# for i, image in enumerate(page['images']): -# text += f"page{page['page']}_figure{i + 1}\n" - -# image_documents = get_image_nodes(json_objs, ".") -# images = [] - -# for image_doc in image_documents: -# image_path = image_doc.image_path -# image = Image.open(image_path) -# images.append(image) - -# return text, images - - PDFPARSE_MAX_RETRY = 2 PDFPARSE_SUPPORTED_LANGS = { "English": "en", @@ -336,16 +276,6 @@ def _prepare_text_with_image(state, text, images, csam_flag): return text - -# def _prepare_text_with_pdf(text, pdfs): -# if len(pdfs) > 0: -# document_content = parse_pdf(pdfs[0]) -# print("Document processed") -# text = wrap_pdfchat_query(text, document_content) - -# return text - - def _prepare_text_with_pdf(text, pdfs): if len(pdfs) > 0: parsed_text, imgs = parse_pdf(pdfs[0]) @@ -360,7 +290,6 @@ def _prepare_text_with_pdf(text, pdfs): return text -# NOTE(chris): take multiple images later on def convert_images_to_conversation_format(images): import base64 diff --git a/fastchat/serve/gradio_block_arena_vision_anony.py b/fastchat/serve/gradio_block_arena_vision_anony.py index 4d84f2025..304105502 100644 --- a/fastchat/serve/gradio_block_arena_vision_anony.py +++ b/fastchat/serve/gradio_block_arena_vision_anony.py @@ -93,15 +93,12 @@ text_models = [] vl_models = [] -# TODO(chris): fix sampling weights VISION_SAMPLING_WEIGHTS = {} -PDFCHAT_SAMPLING_WEIGHTS = {"gpt-4o-2024-05-13": 1, "gpt-4o-mini-2024-07-18": 1} +PDFCHAT_SAMPLING_WEIGHTS = {} -# TODO(chris): Find battle targets that make sense VISION_BATTLE_TARGETS = {} PDFCHAT_BATTLE_TARGETS = {} -# TODO(chris): Fill out models that require sampling boost VISION_SAMPLING_BOOST_MODELS = [] PDFCHAT_SAMPLING_BOOST_MODELS = [] diff --git a/fastchat/serve/test.ipynb b/fastchat/serve/test.ipynb deleted file mode 100644 index 8a6021c30..000000000 --- a/fastchat/serve/test.ipynb +++ /dev/null @@ -1,106 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "import requests\n", - "\n", - "url = \"https://www.datalab.to/api/v1/marker\"\n", - "import os\n", - "\n", - "form_data = {\n", - " \"file\": (\"test.pdf\", open(\"ddsppaper (1).pdf\", \"rb\"), \"application/pdf\"),\n", - " \"langs\": (None, \"English\"),\n", - " \"force_ocr\": (None, False),\n", - " \"paginate\": (None, False),\n", - " \"output_format\": (None, \"markdown\"),\n", - " \"use_llm\": (None, True),\n", - " \"strip_existing_ocr\": (None, False),\n", - " \"disable_image_extraction\": (None, False),\n", - "}\n", - "\n", - "headers = {\"X-Api-Key\": \"wAdzo2tLEsd5PzQTtQT4RNZSBM6rJy_LWFTtj8hjbZ0\"}\n", - "response = requests.post(url, files=form_data, headers=headers)\n", - "data = response.json()" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'detail': 'Not authenticated'}\n" - ] - } - ], - "source": [ - "print(data)" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "ename": "KeyError", - "evalue": "'request_check_url'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[4], line 2\u001b[0m\n\u001b[1;32m 1\u001b[0m max_polls \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m300\u001b[39m\n\u001b[0;32m----> 2\u001b[0m check_url \u001b[38;5;241m=\u001b[39m data[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mrequest_check_url\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[1;32m 3\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mtime\u001b[39;00m\n\u001b[1;32m 4\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m i \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(max_polls):\n", - "\u001b[0;31mKeyError\u001b[0m: 'request_check_url'" - ] - } - ], - "source": [ - "max_polls = 300\n", - "check_url = data[\"request_check_url\"]\n", - "import time\n", - "\n", - "for i in range(max_polls):\n", - " time.sleep(2)\n", - " response = requests.get(check_url, headers=headers)\n", - " data = response.json()\n", - "\n", - " if data[\"status\"] == \"complete\":\n", - " break" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3.12.8 ('myenv')", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.8" - }, - "orig_nbformat": 4, - "vscode": { - "interpreter": { - "hash": "03c457903a5d26c69a3bb8be9c56ac1ee96fb7ba834b2e69a22fb0607b146481" - } - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/playground/FastChat_API_GoogleColab.ipynb b/playground/FastChat_API_GoogleColab.ipynb deleted file mode 100644 index c158163ad..000000000 --- a/playground/FastChat_API_GoogleColab.ipynb +++ /dev/null @@ -1,374 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "1UDur96B5C7T" - }, - "source": [ - "# FastChat API using Google Colab\n", - "\n", - "[ggcr](https://github.com/ggcr)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "NQWpzwse8PrC" - }, - "outputs": [], - "source": [ - "%cd /content/\n", - "\n", - "# clone FastChat\n", - "!git clone https://github.com/lm-sys/FastChat.git\n", - "\n", - "# install dependencies\n", - "%cd FastChat\n", - "!python3 -m pip install -e \".[model_worker,webui]\" --quiet" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "97181RzwSjha" - }, - "source": [ - "See [openai_api.md](https://github.com/lm-sys/FastChat/blob/main/docs/openai_api.md) from FastChat docs.\n", - "\n", - "Because in Google Colab we are limited in resources and executing things in the background is not stable, we will run each API process in a thread and communicate them via explicit addresses:" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "BrhPP9ZggVL0", - "outputId": "be510360-21ba-4f6f-d6b6-24c710bdff4d" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "/content\n" - ] - } - ], - "source": [ - "import subprocess\n", - "import threading\n", - "\n", - "%cd /content/\n", - "\n", - "# Using 127.0.0.1 because localhost does not work properly in Colab\n", - "\n", - "\n", - "def run_controller():\n", - " subprocess.run(\n", - " [\"python3\", \"-m\", \"fastchat.serve.controller\", \"--host\", \"127.0.0.1\"]\n", - " )\n", - "\n", - "\n", - "def run_model_worker():\n", - " subprocess.run(\n", - " [\n", - " \"python3\",\n", - " \"-m\",\n", - " \"fastchat.serve.model_worker\",\n", - " \"--host\",\n", - " \"127.0.0.1\",\n", - " \"--controller-address\",\n", - " \"http://127.0.0.1:21001\",\n", - " \"--model-path\",\n", - " \"lmsys/vicuna-7b-v1.5\",\n", - " \"--load-8bit\",\n", - " ]\n", - " )\n", - "\n", - "\n", - "def run_api_server():\n", - " subprocess.run(\n", - " [\n", - " \"python3\",\n", - " \"-m\",\n", - " \"fastchat.serve.openai_api_server\",\n", - " \"--host\",\n", - " \"127.0.0.1\",\n", - " \"--controller-address\",\n", - " \"http://127.0.0.1:21001\",\n", - " \"--port\",\n", - " \"8000\",\n", - " ]\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "id": "3S8vDHy3gWUv" - }, - "outputs": [], - "source": [ - "# Start controller thread\n", - "# see `controller.log` on the local storage provided by Colab\n", - "controller_thread = threading.Thread(target=run_controller)\n", - "controller_thread.start()" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "id": "UAU097ymgbNf" - }, - "outputs": [], - "source": [ - "# Start model worker thread\n", - "\n", - "# see `controller.log` on the local storage provided by Colab\n", - "# important to wait until the checkpoint shards are fully downloaded\n", - "model_worker_thread = threading.Thread(target=run_model_worker)\n", - "model_worker_thread.start()" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": { - "id": "bTqHMMr1gcQJ" - }, - "outputs": [], - "source": [ - "# Start API server thread\n", - "api_server_thread = threading.Thread(target=run_api_server)\n", - "api_server_thread.start()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "iBdjt9I6fuSn" - }, - "source": [ - "We now have the API running at http://127.0.0.1:8000/v1/ locally from Google Colab.\n", - "\n", - "We can run the examples from FastChat with curl." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "KtaxADXqhazs" - }, - "source": [ - "Try chat completion with" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "MZGd4y2SfBTT", - "outputId": "066835bb-f7f0-4e16-f54a-2f74b0e2f9d9" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\"id\":\"chatcmpl-3RViU5mrsEBNu8oSxexAEb\",\"object\":\"chat.completion\",\"created\":1705781842,\"model\":\"vicuna-7b-v1.5\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"Sure thing! Here's one for you:\\n\\nWhy did the tomato turn red?\\n\\nBecause it saw the salad dressing!\"},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":50,\"total_tokens\":82,\"completion_tokens\":32}}" - ] - } - ], - "source": [ - "!curl http://127.0.0.1:8000/v1/chat/completions \\\n", - " -H \"Content-Type: application/json\" \\\n", - " -d '{ \\\n", - " \"model\": \"vicuna-7b-v1.5\", \\\n", - " \"messages\": [{\"role\": \"user\", \"content\": \"Hello, can you tell me a joke for me?\"}], \\\n", - " \"temperature\": 0.5 \\\n", - " }'" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "umgVIilThc6a" - }, - "source": [ - "Try embeddings with" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "VraqDkMahfAQ", - "outputId": "18710c2c-1994-4f36-eff1-6aff5a2a83a4" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\"object\":\"list\",\"data\":[{\"object\":\"embedding\",\"embedding\":[0.0229715034365654,-0.020740192383527756,0.01663232035934925,0.013713006861507893,-0.01602417416870594,-0.006382038351148367,0.011642662808299065,-0.021167458966374397,0.004879815969616175,-0.005442662630230188,0.0034834356047213078,-0.010336925275623798,-0.009551243856549263,0.0005828586872667074,-0.0089940270408988,-0.0018360239919275045,-0.021827373653650284,0.007349758874624968,-0.0011765437666326761,-0.01432803925126791,0.012239773757755756,-0.018455859273672104,0.016475312411785126,-0.006144467741250992,-0.013893244788050652,-0.00961716752499342,0.00827623251825571,0.0013034207513555884,0.006355977617204189,0.007773293182253838,0.0029199880082160234,-0.014487813226878643,-0.01615595631301403,0.007242684718221426,-0.004686516709625721,-0.0034376305993646383,-0.0046915397979319096,0.0007899928605183959,-0.003679676679894328,-0.022176748141646385,-0.005467468872666359,-0.02236158587038517,0.02086811512708664,0.0029669292271137238,-0.0168694406747818,0.025603512302041054,0.009139388799667358,0.02165624313056469,-0.004472456872463226,0.0006205983809195459,0.0011453271145001054,0.014379195868968964,0.01994524523615837,-0.017613859847187996,0.005462903995066881,0.005702079739421606,-0.021057194098830223,-0.021468186751008034,-0.004666909575462341,-0.007595115341246128,-0.009129735641181469,-0.0161031112074852,0.009293882176280022,0.00953285675495863,-0.0013638428645208478,0.0007091081934049726,0.0018222536891698837,0.020376019179821014,0.01186810340732336,-0.013734177686274052,-0.004418510012328625,-0.006746952421963215,-0.0006970430840738118,-0.006644704379141331,-0.04453064501285553,0.003871878841891885,-0.01059865765273571,-0.024984514340758324,0.011757172644138336,-0.016218630596995354,-0.009141125716269016,-0.004623874556273222,-0.009732221253216267,-0.009169373661279678,-0.006947007961571217,-0.005838882178068161,-0.0068959807977080345,-0.000743469747249037,0.008742589503526688,-0.008120769634842873,-0.018119709566235542,-0.004530956968665123,-0.003916825633496046,0.02495340257883072,0.010598400607705116,0.010666633024811745,0.00679260678589344,-0.009019959717988968,-0.004487940575927496,-0.0026543298736214638,0.00286748050712049,0.012851846404373646,0.0012102456530556083,0.014895712956786156,-0.01030716486275196,0.01633496955037117,0.015731101855635643,-0.009079995565116405,0.016830960288643837,0.00940327625721693,-0.0014347939286381006,0.0207867082208395,0.06265891343355179,0.002649270463734865,-0.007526970934122801,0.004714089445769787,0.006397288292646408,-0.0029612022917717695,-0.0015034123789519072,-0.006392269395291805,-0.012309122830629349,0.0040127672255039215,0.001810954650864005,-0.016414696350693703,-0.019156336784362793,0.0003308420709799975,0.007823580875992775,0.0020239183213561773,-0.0024881847202777863,-0.008919963613152504,-0.01775810308754444,-0.012687149457633495,0.0022407048381865025,-0.009261680766940117,0.006048525683581829,0.00518012186512351,0.0029072873294353485,-7.72168641560711e-06,0.012007351964712143,-0.0004918070626445115,0.0013227892341092229,0.006292788311839104,-0.010167273692786694,-0.009050589054822922,0.008057740516960621,0.006250383332371712,0.014853340573608875,0.02723078615963459,-0.02242557890713215,0.04399850592017174,0.00313431303948164,-0.022166002541780472,0.010024639777839184,0.003234871895983815,0.0030383227858692408,0.012888548895716667,0.01507903728634119,0.00479199830442667,-0.0024831658229231834,0.008515636436641216,0.0005489314789883792,0.004214818123728037,0.006590660661458969,-0.012804229743778706,0.011747709475457668,0.002035082783550024,0.0143223125487566,0.0134012121707201,-0.0008568498305976391,0.0025005715433508158,-0.012422841973602772,0.014866000972688198,0.020239505916833878,-0.0034607010893523693,-0.026886560022830963,-0.0023535056971013546,-0.0037942437920719385,0.013139543123543262,0.004902820568531752,0.008357052691280842,-0.011724174953997135,0.005840683821588755,0.009768190793693066,0.00013014259457122535,0.016845345497131348,-0.006546108052134514,-0.00838533416390419,-0.01408461295068264,-0.0022769987117499113,0.010644538328051567,0.002947496483102441,0.02589692734181881,0.012639564462006092,0.004540625493973494,-0.0176566019654274,-0.010239857248961926,0.01839127205312252,0.0031600680667907,0.011127336882054806,0.0036535318940877914,0.015353705734014511,-0.026527339592576027,-0.008746611885726452,0.01886408030986786,0.00887488853186369,-0.0001859961193986237,0.001222877879627049,0.0065072583965957165,-0.009838716126978397,0.008630175143480301,-0.00633110711351037,0.02635054476559162,-0.005968477576971054,-0.013434287160634995,0.01017901673913002,-0.003692896803840995,-0.005410553887486458,-0.006332104559987783,-0.017778540030121803,-0.017085647210478783,-0.005269246641546488,-0.013628004118800163,-0.0005570553475990891,0.010984581895172596,0.000956009142100811,0.009669160470366478,-0.0019082700600847602,-0.05074448138475418,-0.03876679390668869,0.0011635294649749994,-0.012585809454321861,0.008794615045189857,0.00023998660617507994,-0.00455761281773448,-0.0020947649609297514,0.017387693747878075,0.004844747018069029,0.008267332799732685,0.00747610442340374,0.02141532674431801,-0.02262278087437153,-0.014600872062146664,-0.021727152168750763,0.008812149986624718,0.009474638849496841,0.03191479295492172,-0.019652077928185463,0.01944698765873909,0.017112286761403084,0.015296016819775105,0.014461753889918327,-0.019157931208610535,0.009540014900267124,0.004215397406369448,-0.008012793958187103,0.013523118570446968,-0.009407458826899529,-0.029304828494787216,0.012041181325912476,0.015149015933275223,0.0031983305234462023,-0.0003109185490757227,0.03257888928055763,0.007614033296704292,-0.005175750236958265,-0.002383652376011014,0.006435382179915905,0.006068408954888582,-0.007524268701672554,0.02373131737112999,0.004817254841327667,0.005436067469418049,-0.0059105646796524525,-0.005925316829234362,-6.454042886616662e-05,-0.008412199094891548,-0.00655836658552289,-0.0010680218692868948,-0.004262322559952736,0.0015925978077575564,0.00412611523643136,-0.011034490540623665,0.009839101694524288,0.00415002042427659,-0.007727092131972313,-0.010377302765846252,0.0007711391081102192,-0.009322070516645908,0.0035655524116009474,-0.026301125064492226,-0.006197007372975349,0.0006739745149388909,-0.00818476639688015,-0.02090131863951683,-0.002644758205860853,0.006994722411036491,-0.0016304099699482322,0.01705804094672203,-0.016460495069622993,0.017486274242401123,0.013187418691813946,0.0033816162031143904,0.017844069749116898,-0.017695210874080658,-0.011941025033593178,0.009029353968799114,0.0033719318453222513,-0.009064359590411186,0.012252643704414368,0.0011845449917018414,0.003185839159414172,0.003374891821295023,-0.007335654925554991,0.0029391313437372446,0.000280876352917403,0.0048222895711660385,-0.0003767217858694494,-0.045474909245967865,0.004725527483969927,0.0075803473591804504,0.005909985862672329,0.002949362387880683,-0.0036183823831379414,0.0026071954052895308,-0.005563989747315645,-0.012707033194601536,-0.004933884367346764,-0.016659578308463097,-0.0081319659948349,0.012579865753650665,-0.022291865199804306,-0.018159057945013046,-0.0069056968204677105,-0.00018650286074262112,-0.006835494190454483,0.0006484286277554929,0.005561383906751871,0.0062789213843643665,0.029090696945786476,0.002546998206526041,0.009344656951725483,-0.0038842656649649143,-0.012519339099526405,-0.0025535617023706436,-0.003679415676742792,-0.0033875037916004658,0.003728062380105257,-0.014787501655519009,0.0023771373089402914,0.005443841218948364,-0.00957341119647026,-0.015306569635868073,0.0046866778284311295,-0.016635537147521973,-0.01424899697303772,0.001698320615105331,-0.004534294828772545,0.0066452836617827415,0.010703673586249352,0.004293128848075867,-0.009486992843449116,-0.0031507215462625027,0.01611129753291607,-0.015744132921099663,-0.014641146175563335,0.0026989546604454517,0.01565713621675968,-0.005524931009858847,0.006648661568760872,0.0040243822149932384,-0.00559786893427372,-0.014391486532986164,0.026553215458989143,-0.009266120381653309,0.020683180540800095,0.00994131714105606,0.0026739235036075115,0.0038542025722563267,-0.012158502824604511,-0.010751161724328995,-0.00017412402667105198,-0.017064156010746956,-0.010691382922232151,0.00937278475612402,-0.014700417406857014,-0.005352479871362448,0.012342552654445171,0.009191831573843956,-0.011637836694717407,-0.012737436220049858,0.01105053722858429,0.020749129354953766,0.07297933101654053,0.027850160375237465,-0.005428216885775328,-0.019425511360168457,0.0016134463949128985,-0.007674881722778082,0.004896160680800676,-0.006309020332992077,0.0028925116639584303,-0.016418879851698875,-0.012568380683660507,-0.0076565672643482685,-0.002051394898444414,0.011267355643212795,0.01101701334118843,0.02482358179986477,0.011389358900487423,-0.01589033007621765,0.0005615596892312169,-0.027247965335845947,-0.008588980883359909,0.005675439722836018,0.008922569453716278,-0.003106530988588929,0.00925450585782528,-0.00030810333555564284,-0.002115500858053565,-0.007074093911796808,-0.005927231162786484,-0.017885340377688408,-0.016033342108130455,-0.0049004401080310345,0.006337509956210852,0.01978384517133236,0.001572070992551744,-0.0143946073949337,-0.008655560202896595,-0.0011587677290663123,-2.521412170608528e-05,-0.01082194410264492,0.010964666493237019,-0.011412781663239002,0.008038532920181751,0.006299568805843592,-0.008974144235253334,0.006545931100845337,0.0006125871441327035,0.00486041558906436,0.0042688059620559216,0.0018871801439672709,-0.006763682700693607,0.013578971847891808,-0.0020262349862605333,-0.0024552710819989443,-0.01506423857063055,0.0054992204532027245,0.011333892121911049,-0.007717472035437822,-0.005762179847806692,0.0007979075890034437,0.007761630229651928,-0.00952511839568615,-0.010288495570421219,0.014522014185786247,-0.005318223498761654,0.009297103621065617,0.0038411528803408146,0.012293890118598938,0.004698003176599741,-0.007060967851430178,-0.004558722488582134,-0.003963573835790157,0.016085496172308922,0.015816137194633484,0.0027972774114459753,-0.017336538061499596,0.014937583357095718,0.013450084254145622,0.06357342004776001,-0.009506811387836933,0.007877970114350319,0.007048371247947216,0.011457744054496288,0.023370005190372467,0.014203527010977268,-0.004614254459738731,-0.008159955963492393,0.0030794248450547457,-0.0010602197144180536,0.0006093939300626516,-0.010418003425002098,-0.007668149657547474,0.015968769788742065,-0.0015574641292914748,-0.018846578896045685,-0.003667157609015703,0.0019307112088426948,-0.001895931432954967,-0.010295855812728405,0.00023113582574296743,0.007993489503860474,0.0022910244297236204,0.00033837073715403676,-0.005313453264534473,0.0010675875237211585,-0.01549510844051838,0.007410695310682058,0.009646059945225716,-0.012997191399335861,0.010529725812375546,-0.019208982586860657,-0.010029473342001438,-0.013124711811542511,0.029043130576610565,-0.00493550905957818,0.008303387090563774,0.0067044831812381744,0.005133184138685465,-0.008268092758953571,0.0027517518028616905,-0.013479426503181458,-0.01547516044229269,-0.020013773813843727,-0.006451855413615704,0.008133156225085258,-0.006830539554357529,-0.007085484452545643,0.010033013299107552,0.002104497514665127,0.0005678657325915992,0.006996427197009325,-0.00785919837653637,-0.029753299430012703,0.03372034803032875,-0.008247010409832,0.008989491499960423,0.017457574605941772,-0.0059603373520076275,-0.003432418452575803,-0.014526166021823883,0.01819109544157982,-0.007616993971168995,-0.008361894637346268,0.008198246359825134,0.004229682497680187,-0.02080651931464672,0.009076694026589394,-0.006605580914765596,0.0037523536011576653,-0.010452975519001484,-0.012760377489030361,-0.017025675624608994,-0.007993683218955994,0.013692287728190422,0.017206765711307526,0.006106856279075146,0.011746293865144253,-0.009011680260300636,-0.007511272560805082,0.006244495511054993,0.009395747445523739,0.006921007763594389,0.00926200207322836,0.03370635211467743,0.0026780739426612854,0.012087206356227398,0.0012626887764781713,-0.014491417445242405,-0.007984738796949387,-0.02033303491771221,-0.008010058663785458,-0.0027411666233092546,-0.006356299389153719,0.014341359958052635,0.00924749206751585,0.008061794564127922,-0.014423820190131664,-0.0027274927124381065,-0.009464149363338947,0.0032869288697838783,0.028920968994498253,-0.007417581044137478,-0.012927711941301823,-0.006823397241532803,0.0021555088460445404,-0.008643687702715397,-0.0023652170784771442,-0.0060961428098380566,-0.017238536849617958,-0.007533663418143988,0.0022437411826103926,-0.0029654495883733034,0.007918364368379116,-0.004272923804819584,0.022094689309597015,-0.01293826475739479,-0.03929437696933746,-0.05735565349459648,-0.013071688823401928,0.0007404614589177072,-0.000720368989277631,-0.006117763463407755,-0.011282929219305515,0.010729444213211536,-0.014913954772055149,0.00311655318364501,0.006948134861886501,-0.00748022273182869,-0.02309916727244854,-0.0178229883313179,-0.0072602517902851105,0.007839913479983807,0.012868576683104038,0.002075975527986884,0.0007498079212382436,0.005815781187266111,-0.011992518790066242,0.010061261244118214,0.004755143541842699,-0.0014543153811246157,0.014422083273530006,-0.0023919050581753254,0.009424189105629921,-0.01841503195464611,0.008597759529948235,0.023288220167160034,-0.009507520124316216,0.015740947797894478,-0.0004225693119224161,0.02476677857339382,-0.011370633728802204,0.011675688438117504,0.020527847111225128,-0.0073279449716210365,-0.013483609072864056,-0.019474929198622704,-0.004016772843897343,-0.012925073504447937,-0.00565439835190773,0.0104595385491848,-0.007314899004995823,0.010194428265094757,0.0022050561383366585,0.011519340798258781,-0.0059105330146849155,-0.0007297637057490647,-0.016200484707951546,0.015271657146513462,-0.016203250735998154,0.034517351537942886,0.0006107089575380087,-0.013269267976284027,0.01328535471111536,-0.02016814425587654,-0.007773164194077253,-0.007333156652748585,-0.01815428026020527,0.006929537747055292,-0.0034732790663838387,-0.004669690039008856,0.0016878641908988357,-0.03094855323433876,0.0019403311889618635,-0.005923015996813774,-0.0040122526697814465,0.009299001656472683,-0.006708343978971243,0.01585310511291027,0.0010694535449147224,0.0006908577051945031,-0.0015497022541239858,-0.014749257825314999,0.013069666922092438,-0.0003381777205504477,-0.0186776015907526,-0.00869465060532093,-0.005246113985776901,0.004712183494120836,-0.0033125269692391157,0.005922533571720123,0.005009307526051998,-0.002772809471935034,0.0018297180067747831,-0.007289668545126915,-0.025313491001725197,-0.010890730656683445,-0.013207301497459412,-0.015217771753668785,-0.0064299451187253,0.0012019408168271184,0.013148745521903038,-0.022279510274529457,0.008878774009644985,-0.007133841048926115,-0.0007347667124122381,0.007130189798772335,0.0017936835065484047,0.012268022634088993,0.007812416646629572,0.009994118474423885,-0.01274168398231268,-0.000458410766441375,-0.006630516145378351,0.0004267197218723595,0.013977475464344025,-0.003951766062527895,-0.0167144313454628,-0.012754247523844242,0.012914633378386497,0.010781855322420597,0.002908888040110469,-0.007131427992135286,0.017916306853294373,-0.005879903212189674,-0.002502115909010172,-0.0016746085602790117,-0.024386180564761162,-0.008716223761439323,0.003937223460525274,0.004685036838054657,-0.005052074324339628,-0.004745748359709978,-0.004316418897360563,-0.009056701324880123,-0.011055074632167816,0.0087593924254179,-0.016003968194127083,-0.001959120621904731,0.014024545438587666,-0.005205253139138222,-0.0034684527199715376,-0.00704217841848731,0.004913646727800369,0.01903299242258072,-0.007594246882945299,-0.0001278904383070767,-0.00024535658303648233,0.01912636123597622,0.02121288888156414,0.01097018364816904,-0.005211591720581055,-0.004693691153079271,0.0002123745362041518,0.01864037662744522,0.004567956551909447,-0.006998493801802397,0.002807476557791233,-0.0272210780531168,0.008950882591307163,-0.007628897670656443,0.017757385969161987,0.011070613749325275,-0.02169198729097843,0.005343310534954071,0.0013322805752977729,-0.004593148827552795,0.009079867042601109,0.011012605391442776,0.00658367108553648,-0.004797258879989386,-0.006833371240645647,-0.0069283475168049335,-0.009916930459439754,-0.006784595549106598,-0.03476946800947189,0.020896492525935173,0.008564138785004616,-0.0012716330820694566,-0.013008822686970234,-0.000613439769949764,0.0047750589437782764,-0.012346075847744942,0.006973704788833857,-0.013979197479784489,-0.006083691958338022,0.005035505164414644,0.011145804077386856,0.013424682430922985,-0.00019037174934055656,-0.008892635814845562,-0.01950671710073948,-0.010386078618466854,0.01175111997872591,-0.014368708245456219,0.00041413979488424957,-0.014867283403873444,0.0020979661494493484,-0.002743129152804613,0.004309915471822023,-0.012755325064063072,0.013642803765833378,0.008863402530550957,0.0013711462961509824,-0.019572222605347633,0.0036479418631643057,0.1259939968585968,0.01384377758949995,0.015267448499798775,0.014036224223673344,0.0038570465985685587,-0.005283885635435581,0.010237026028335094,-0.011374881491065025,-0.011878897435963154,-0.008971023373305798,-0.009165126830339432,-0.0010226268786936998,-0.007337307557463646,-0.010756309144198895,-0.014150279574096203,0.002133630681782961,-0.015334047377109528,0.00481215538457036,-0.013047880493104458,-0.014511879533529282,-0.0030851999763399363,-0.007749861106276512,-0.006487664300948381,0.013752967119216919,-0.012187069281935692,0.0007167012081481516,-0.0016341822920367122,-0.004467220976948738,0.0042928713373839855,0.022611349821090698,0.0005482397391460836,-0.017850179225206375,-0.014368931762874126,-0.02213916927576065,0.009322037920355797,-0.008927379734814167,0.0012655361788347363,0.003878731979057193,-0.011226431466639042,0.014120969921350479,-0.013007482513785362,-0.027299636974930763,-0.02149207703769207,0.0018350587924942374,0.0014142269501462579,-0.000801382411736995,0.010266175493597984,0.006652788259088993,0.0005369306891225278,-0.006750426720827818,0.0077108764089643955,0.008079683408141136,-0.0018402388086542487,-0.016589464619755745,-0.009489567019045353,-0.006460928358137608,-0.008930034004151821,0.005508729722350836,-0.021854624152183533,0.0021647908724844456,-4.1697108827065676e-05,0.0023772178683429956,-0.015694361180067062,-0.0025681040715426207,0.02343827858567238,-0.007234286982566118,0.011763988994061947,0.006332748103886843,0.01569999009370804,0.0011719107860699296,-0.0026809938717633486,-0.019673682749271393,0.010832150466740131,0.0020819918718189,0.0021434274967759848,0.014149283058941364,-0.018654564395546913,-0.005904508288949728,0.024274280294775963,0.0020302003249526024,0.009324193932116032,-0.0019528145203366876,0.010275795124471188,-0.007945165038108826,0.02523057907819748,-0.0015196279855445027,-0.0033202609047293663,-0.00838176254183054,0.009073046036064625,0.004423896782100201,0.0025238976813852787,0.0009007186163216829,0.012340654619038105,0.013026034459471703,0.0006704675615765154,-0.011622972786426544,0.0025514704175293446,0.0018054911633953452,-0.00021421245764940977,0.0015564989298582077,0.0002535287057980895,-0.007833908312022686,-0.002614386146888137,0.010472987778484821,0.008430087007582188,-0.010319744236767292,-0.007126948330551386,-0.0032228068448603153,-0.005715849809348583,-0.007379905320703983,0.0007485531968995929,-0.020927315577864647,0.0019611797761172056,0.0038484123069792986,-0.006966795306652784,-0.018788157030940056,0.007531090173870325,-0.006524322554469109,0.010099516250193119,-0.004077629651874304,-0.017544057220220566,-0.0056204223074018955,0.0014705952489748597,0.02655109204351902,-0.004098542500287294,0.00679929880425334,-0.009616298601031303,-0.00428798096254468,-0.004214432090520859,0.017463093623518944,0.007254500407725573,0.011614413931965828,-0.015450838021934032,0.01430854294449091,0.011353002861142159,0.0038417966570705175,0.013071335852146149,-0.003091377206146717,-0.0012477281270548701,-0.012130544520914555,-0.0005112078506499529,0.0007805016357451677,0.01115238294005394,-0.011903454549610615,0.01652473211288452,-0.016062499955296516,0.0243363119661808,0.00521033676341176,-0.019244149327278137,0.015055154450237751,-0.0014579187845811248,0.024649038910865784,0.003033657558262348,-0.004459853284060955,-0.0024275374598801136,-0.004720765631645918,-0.008315999060869217,0.01299308892339468,0.003514010924845934,0.00035230195499025285,-0.0016822096658870578,-0.011835559271275997,0.013584377244114876,0.014042497612535954,-0.0021746200509369373,-0.013556176796555519,0.009201740846037865,-0.016880186274647713,0.006788729690015316,0.007318035699427128,0.0079000573605299,-0.0021131120156496763,0.005459972191601992,-0.01956108957529068,-0.003485738066956401,-0.012780935503542423,-0.010953888297080994,-0.0035778111778199673,0.013985262252390385,0.004123058635741472,-0.017365043982863426,0.02569989673793316,-0.0032679142896085978,-0.006953733041882515,-0.020901406183838844,0.003745210822671652,0.004216748755425215,0.007281791884452105,0.01097949780523777,-0.008859830908477306,0.0076435767114162445,-0.002383668441325426,0.003228791058063507,0.000471006816951558,0.021136121824383736,0.006612015888094902,-0.00790025107562542,0.002388188848271966,-0.01046378631144762,0.0019024949287995696,-0.020805569365620613,0.008167678490281105,0.01708216592669487,0.003778784302994609,-0.007486400194466114,0.009304165840148926,0.01634320802986622,-0.015319439582526684,0.012349807657301426,0.008008498698472977,0.004085544031113386,-0.0019550668075680733,-0.0013337925774976611,0.005621806252747774,0.00999923050403595,0.0067540789023041725,0.024973737075924873,-0.013562659732997417,-0.009736709296703339,-0.012089909054338932,-0.016808679327368736,0.008086872287094593,0.008295665495097637,-0.012549092061817646,-0.010748330503702164,3.521411053952761e-05,0.0017467420548200607,0.01626216247677803,0.009219243191182613,-0.006609965115785599,0.010143030434846878,-0.020303402096033096,-0.01044105552136898,-0.013318654149770737,0.00010932621080428362,0.007084518671035767,0.007645950186997652,-0.0032920767553150654,-0.01955648884177208,0.0074850814417004585,0.00894773006439209,0.009001234546303749,0.005829519592225552,-0.0045957546681165695,0.0013910618145018816,-0.012523948214948177,0.013304369524121284,0.01453658938407898,0.017666004598140717,-0.004940214566886425,-0.011730528436601162,-0.015571167692542076,-0.010929387994110584,-0.0006716740899719298,0.02221648395061493,0.021565254777669907,0.01226515881717205,-0.0053292508237063885,0.0007020622142590582,0.0024210221599787474,0.01962619461119175,-0.004420963115990162,-0.015309896320104599,0.0034791347570717335,0.02059043198823929,-0.008116353303194046,-0.0032520205713808537,-0.012169377878308296,0.025940747931599617,-9.851584763964638e-05,0.0036511996295303106,0.0037823636084795,-0.010169846937060356,0.010504196397960186,0.013252376578748226,-0.007866725325584412,-0.0026977320667356253,-0.011583752930164337,-0.006372353993356228,-0.0007445314549840987,-0.0030074622482061386,0.016342146322131157,-0.009066401980817318,0.0021215977612882853,0.008862188085913658,0.015515057370066643,0.009001555852591991,-0.024249698966741562,0.020413951948285103,0.008854007348418236,0.0006535120774060488,0.013391399756073952,-0.01817990653216839,-0.0016513630980625749,-0.011816883459687233,0.007374065928161144,0.02026175521314144,-0.019211476668715477,0.00015504502516705543,-0.007945390418171883,0.001324703567661345,0.025466380640864372,0.006762733682990074,-0.01408602949231863,-0.01516133826225996,-0.0069986796006560326,-0.0004754628462251276,-0.01119284238666296,-0.004222266376018524,-0.014954396523535252,0.0031823322642594576,-0.009523541666567326,-0.011928976513445377,-0.0011272374540567398,-0.009063232690095901,-0.011843233369290829,-0.0030050550121814013,-0.010779651813209057,0.017810650169849396,0.009822757914662361,-0.0130256162956357,-0.002755612600594759,0.010061550885438919,-0.002134740585461259,-0.0004929009592160583,-0.011506262235343456,0.004393350332975388,0.002644677646458149,0.013704448938369751,-0.015646131709218025,-0.005174269899725914,0.017940374091267586,0.006815964821726084,-0.014483116567134857,-0.018775692209601402,-0.017056433483958244,-0.00333380582742393,-0.01628420129418373,-0.02220962941646576,-0.007394126150757074,0.004732364322990179,0.003667865414172411,0.013815898448228836,-0.014784134924411774,0.006790837273001671,-0.005050111562013626,-0.01184664387255907,-0.005963458679616451,0.01068057306110859,0.01837034337222576,6.692128226859495e-05,-0.0020520382095128298,-0.005477442871779203,0.008534909226000309,0.021816853433847427,0.019038107246160507,0.008523069322109222,-0.021777216345071793,-0.01595551334321499,-0.012562203221023083,0.012347427196800709,0.013057525269687176,-0.015681490302085876,0.012324455194175243,-0.0041071330197155476,0.01061281468719244,-0.01118357665836811,-0.001830828026868403,0.0030818136874586344,0.0002257306332467124,0.012498816475272179,0.005094640422612429,0.020110618323087692,0.008550223894417286,0.008692882023751736,0.0034023199696093798,-0.0035538740921765566,0.017047973349690437,-0.008395790122449398,0.0036361422389745712,0.0012567044468596578,-0.012467821128666401,0.015781357884407043,-0.009986070916056633,0.01078745350241661,0.008992418646812439,-0.00894157588481903,-0.009751653298735619,-0.007818657904863358,-0.11352294683456421,0.006673813331872225,0.0006858144770376384,0.012712855823338032,0.017139634117484093,-0.003267174120992422,-0.0037179840728640556,-0.027594735845923424,0.015738407149910927,-0.008096124045550823,0.008535375818610191,-0.006178006995469332,0.0021386174485087395,0.00922358687967062,0.015902427956461906,0.010610240511596203,-0.006293817888945341,0.007873225025832653,-0.009341374039649963,-0.015121137723326683,-0.0025967389810830355,0.0009708734578453004,0.02104487642645836,-0.0034994683228433132,-0.012507845647633076,0.022736024111509323,-0.007137798238545656,0.004183493088930845,-0.005087561905384064,0.005540612153708935,0.011934671550989151,-0.008175094611942768,0.013157593086361885,0.003565874882042408,0.007175907958298922,0.02075435034930706,-0.008561364375054836,0.0018133737612515688,-0.0031988373957574368,0.0026560029946267605,-0.015025373548269272,0.0025075653102248907,-0.020946715027093887,0.002409552223980427,0.0030347283463925123,-0.008436071686446667,0.011734389699995518,0.005770737770944834,0.0027340843807905912,0.009276704862713814,0.014263113029301167,0.005924335680902004,-0.013983492739498615,-0.0073938933201134205,-0.0037190215662121773,-0.007606761995702982,0.00866461731493473,-0.00787393283098936,0.004571785684674978,-0.01736222766339779,0.0011665115598589182,-0.018963271751999855,0.002434736117720604,0.023223616182804108,0.013454395346343517,-0.007077569141983986,0.006989220157265663,0.0016794668044894934,-0.0029226583428680897,0.015770161524415016,-0.007460178807377815,0.02135499194264412,-0.0067621381022036076,0.006347097456455231,0.01143655739724636,-0.009779580868780613,0.0011012412142008543,0.022937849164009094,0.03317839652299881,0.002777715912088752,0.0014309572288766503,-0.004011448472738266,-0.020232975482940674,-0.0036248492542654276,0.009381849318742752,-0.004546706099063158,0.01232175249606371,-0.02003932185471058,0.005393791012465954,0.007975440472364426,-0.02001962997019291,0.00812353566288948,0.004558304324746132,0.012361841276288033,-0.00022309240011963993,-0.005494816228747368,-0.005414157174527645,-0.0007955267792567611,-0.006178250070661306,0.0011265840148553252,0.014568240381777287,-0.015398587100207806,-0.009784664027392864,0.002724339719861746,-0.012673153541982174,-0.0022227196022868156,0.012834923341870308,0.011582594364881516,0.0023665439803153276,0.006087005604058504,-0.0014784777304157615,0.004853080026805401,0.004227772355079651,0.005455693230032921,-0.0038181168492883444,-0.009257722645998001,0.006031699012964964,0.0033167218789458275,-0.0009175615850836039,0.023257719352841377,-0.0028650029562413692,0.002901359461247921,0.002793062711134553,0.01102980226278305,0.0026135335210710764,0.028918616473674774,0.015613989904522896,-0.0029948721639811993,-0.009738076478242874,0.018055813387036324,0.0043314797803759575,0.008178786374628544,-0.011788956820964813,0.011455508880317211,0.01573013886809349,0.00820583663880825,0.01591729186475277,0.002678733319044113,-0.017613554373383522,-0.00441357959061861,-0.010343971662223339,0.003275121096521616,-0.004354435950517654,-0.016168376430869102,-0.016327762976288795,0.010710583068430424,-0.0002415279159322381,-0.005174752790480852,-0.010321610607206821,2.5521783754811622e-05,-0.005093996413052082,0.00427284324541688,-0.00925386231392622,-0.022916292771697044,-0.005452363286167383,-0.005463994108140469,-0.00032996939262375236,-0.0056364452466368675,-0.01507771946489811,-0.0140626709908247,-0.001988076837733388,0.010080339387059212,-0.008691756054759026,0.001160038635134697,-0.0021076020784676075,-0.012562798336148262,-0.002622719155624509,0.0030087551567703485,-0.007625970058143139,-0.002947271103039384,0.018139785155653954,0.02823634259402752,-0.0030986485071480274,-0.0026572253555059433,-0.009556874632835388,-0.0120854452252388,-0.016098687425255775,0.004706657491624355,0.018779207020998,-0.008696485310792923,0.02307201363146305,0.008763439022004604,-0.014935833401978016,-0.010818082839250565,-0.2784213721752167,-0.007361662574112415,-0.009495736099779606,-0.023461056873202324,-0.008934522047638893,0.015963122248649597,0.0016804963815957308,-0.009592200629413128,-0.011385498568415642,0.010840379633009434,0.0007005499792285264,0.0030378401279449463,0.01442185789346695,0.0060276128351688385,0.011916878633201122,0.0019495971500873566,0.010881658643484116,0.010174351744353771,0.002560058841481805,-0.011619336903095245,0.005709640681743622,-0.019679618999361992,0.008580016903579235,-0.020601846277713776,-0.003206663765013218,-0.009325030259788036,0.010211093351244926,0.02160986326634884,-0.0012345046270638704,-0.0058813090436160564,0.02697822079062462,-0.009422902949154377,-0.013682184740900993,-0.0015802914276719093,0.020953504368662834,-0.003903919830918312,-0.00243631680496037,-0.020303402096033096,0.01755078323185444,0.024769868701696396,0.0016339250141754746,0.02251550555229187,0.004645044915378094,-0.010381357744336128,-0.014821520075201988,-0.010959195904433727,0.00934459175914526,-0.010714001022279263,0.018016111105680466,-0.00970667414367199,-0.007309091277420521,-0.012314545921981335,-0.02047012746334076,0.027432451024651527,-0.0009060755837708712,0.07745006680488586,-0.0023823976516723633,0.01124457735568285,0.0096189696341753,-0.0008077527745626867,-0.0035770712420344353,-0.0034886582288891077,0.011778567917644978,-0.008943229913711548,0.003386442083865404,-0.00024284704704768956,0.010145587846636772,0.007330470718443394,0.003942918032407761,0.0022819836158305407,-0.0008272781851701438,0.007588133215904236,0.005243266467005014,-0.014266717247664928,-0.005166773218661547,0.0074570500291883945,-0.0016363218892365694,-0.019104288890957832,-0.005167931783944368,0.008953874930739403,-0.007413430605083704,-0.013545575551688671,-0.017633790150284767,0.026401540264487267,-0.0021100472658872604,-0.010175767354667187,0.009788733907043934,-0.014036711305379868,0.003915506415069103,-0.003761973464861512,-0.004975275602191687,0.002093156334012747,-0.001363328075967729,-0.0029019585344940424,-0.009283140301704407,-0.006503944285213947,-0.011568261310458183,0.02174294926226139,-0.014086995273828506,0.0033965124748647213,0.0035606948658823967,0.003461358603090048,0.010544992983341217,0.010210482403635979,-0.002245498588308692,0.019960559904575348,-0.007419897243380547,-0.007997768931090832,0.00904663186520338,0.02357649616897106,-0.011239221319556236,-0.00011569660273380578,-0.0029487835709005594,0.007448234129697084,0.016541525721549988,-0.0001295312977163121,0.009020346216857433,-0.020686302334070206,0.015325473621487617,-0.0016831347020342946,-0.008773420937359333,0.016255050897598267,-0.0012025240575894713,0.01161193661391735,-0.016618099063634872,0.012996693141758442,-0.004140432924032211,-0.007176905404776335,0.020722240209579468,-0.010730667039752007,0.01690627448260784,-0.0032811376731842756,0.010093660093843937,-0.0027236961759626865,-0.03603730350732803,-0.004680242855101824,0.006091711111366749,-0.012325975112617016,-0.014773516915738583,-0.012536093592643738,0.0029048342257738113,-0.02004828117787838,-0.007857202552258968,-0.012408236041665077,-0.005879549775272608,-0.003138889791443944,-0.015323558822274208,-0.0001826178777264431,0.004041365813463926,-0.015603084117174149,0.008681814186275005,0.01134839653968811,0.0006241817027330399,-0.026418721303343773,0.0036757681518793106,0.0031010936945676804,-0.0018149744719266891,-0.0038577064406126738,-0.010925833135843277,-0.006739520467817783,-0.014096260070800781,-0.005563016515225172,0.016652911901474,-0.0007585270213894546,0.011374784633517265,-0.009055189788341522,0.014467866159975529,0.021866194903850555,-0.011922026984393597,-0.006064226385205984,0.014592982828617096,0.012229286134243011,0.007419169414788485,-0.003800228238105774,0.005821636877954006,0.005980832036584616,0.019860951229929924,0.0005983874434605241,-0.021042626351118088,-0.011280648410320282,-0.0034789254423230886,-0.005904307123273611,0.00940112117677927,-0.01505252718925476,-0.007798091508448124,-0.005041247699409723,-0.020565425977110863,0.002939002588391304,-0.010503344237804413,0.006530262529850006,-0.00948650948703289,0.006920433137565851,-0.013644187711179256,-0.01110368873924017,-0.0007017726311460137,-0.011356927454471588,-0.009044218808412552,0.004168874584138393,0.014494956471025944,0.007382184267044067,-0.01204177737236023,-0.0026305855717509985,0.00237200572155416,-0.011614670976996422,0.0075203352607786655,-0.007654733490198851,-0.018017364665865898,-0.007952709682285786,0.009685106575489044,0.016591427847743034,0.008159216493368149,-0.004515109583735466,0.019129447638988495,-0.1756141632795334,-0.024899190291762352,0.0018353804480284452,0.008671293035149574,-0.01384413056075573,0.01001817174255848,-0.012732546776533127,0.005506077315658331,0.0014535110676661134,-0.00014272250700742006,-0.02563503570854664,0.0071355667896568775,-0.02158156782388687,-0.00474808132275939,0.018071835860610008,0.023083724081516266,0.009568641893565655,0.006390306632965803,-0.005066118203103542,-0.01592129096388817,0.017062868922948837,-0.01115796621888876,-0.015767812728881836,-0.005238134413957596,0.006928991060703993,0.006582673639059067,-0.008210115134716034,-0.0006850744248367846,0.003518740413710475,0.02363714389503002,0.014902275986969471,-0.00873962976038456,-0.00457162456586957,0.008439594879746437,0.004671009257435799,0.006651798263192177,0.007029373198747635,0.010178695432841778,-0.01541563868522644,0.005330503452569246,0.005778331309556961,0.010172613896429539,-0.0029294793494045734,-0.005375274922698736,0.015940893441438675,-0.01708410307765007,0.02029111236333847,0.020185356959700584,0.003809751709923148,0.010334190912544727,0.004035063553601503,-0.013017106801271439,-0.009174071252346039,0.0011511747725307941,0.003145364811643958,-0.004294078331440687,0.01332454290241003,-0.013086714781820774,0.016923105344176292,-0.012309269048273563,-0.012259078212082386,0.0015276713529601693,0.00023750621767248958,-0.00841486919671297,-0.012003683485090733,-0.02218620665371418,-0.006810398772358894,-0.05309946462512016,-0.016830896958708763,0.008899983949959278,0.013663781806826591,-0.008498359471559525,-0.009214417077600956,-0.005358291324228048,-0.019415665417909622,-0.0016335167456418276,-0.01287610549479723,-0.005925686564296484,0.007678573951125145,0.004894197918474674,-0.005250392947345972,0.01937422715127468,0.03884986415505409,0.007704956457018852,0.004224277101457119,-0.010258260183036327,0.012103293091058731,0.0007560174562968314,0.009477147832512856,0.005485904403030872,0.011781315319240093,0.005216819699853659,-0.01289766188710928,-0.00058182911016047,-0.006487181875854731,0.010025066323578358,0.01070936769247055,0.008055237121880054,0.009198716841638088,-0.0050565944984555244,0.01677780970931053,-0.004822997841984034,-0.0006103349733166397,-0.010622531175613403,-0.007425166200846434,-0.0016098107444122434,-0.006618257611989975,0.0011639798758551478,-0.08570022881031036,0.020885812118649483,-0.025955354794859886,0.018434884026646614,-0.0073579950258135796,0.005618041846901178,0.005165067967027426,0.0032188494224101305,-0.0012533745029941201,0.015155804343521595,-0.004030752461403608,-0.0077774110250175,0.0008675797143951058,-0.0021942458115518093,0.005814365576952696,0.0067954701371490955,-0.0116463303565979,-0.004899860825389624,0.012563779018819332,-0.02336389385163784,0.0006979600293561816,-0.004649227485060692,-0.012502971105277538,-0.010896007530391216,0.0012360489927232265,-0.012883569113910198,0.025206802412867546,0.011092202737927437,-0.01052560843527317,-0.006687352433800697,-0.01787686161696911,0.004141188692301512,0.0106991371139884,-0.00821922067552805,-0.02622329816222191,0.006792123895138502,-0.013250929303467274,0.007654957938939333,0.008035637438297272,-0.005465570371598005,-0.013763535767793655,-0.01950150541961193,0.008698672987520695,0.0057535613887012005,-0.019228672608733177,-0.011553805321455002,-0.0003967660013586283,0.0012686088448390365,0.006336930673569441,-0.005957281216979027,-0.002579220337793231,-0.002936155302450061,0.0036823435220867395,0.005852008704096079,0.017855370417237282,-0.00011639236618066207,0.0004218293179292232,0.001062761410139501,0.0018936148844659328,0.0179592277854681,0.006386397872120142,0.009569131769239902,0.00946755986660719,0.0031641540117561817,-0.019553659483790398,0.0029401606880128384,-0.014651062898337841,-0.009318306110799313,0.01822330802679062,0.019901007413864136,0.002202707575634122,0.003464141394942999,0.0073665534146130085,-0.014449591748416424,-0.0014002956449985504,0.01639820821583271,0.010666480287909508,0.00931896548718214,-0.0015187592944130301,-0.023576384410262108,-0.00443253805860877,0.014584994874894619,-0.0053917961195111275,0.01415127795189619,0.011401182971894741,-0.0006382536957971752,0.018119532614946365,0.009133468382060528,0.012955060228705406,-0.0014709169045090675,-0.016649436205625534,0.02026389352977276,0.0006713725160807371,0.015495236963033676,0.003925270866602659,0.00319079402834177,-0.003925030119717121,-0.021138904616236687,-0.00461933808401227,-0.005469720810651779,0.00739274313673377,0.019258851185441017,0.02616351842880249,0.023124778643250465,-0.00566488690674305,0.01773357018828392,0.023644834756851196,0.0047590043395757675,0.017013562843203545,-0.0032865749672055244,-0.018152205273509026,-0.010509730316698551,0.004198023583739996,0.011710388585925102,-0.00446705985814333,0.002852680627256632,-0.002007831586524844,-0.000134904301376082,-0.01944751851260662,0.017555125057697296,0.007372296415269375,0.013482901267707348,-0.01416250690817833,0.009404434822499752,0.002286749193444848,0.005182494409382343,-0.0028514256700873375,0.004553719889372587,-0.0026370203122496605,-0.0011353131849318743,0.011851341463625431,-0.00646215071901679,-0.013426951132714748,0.020288217812776566,0.006485862657427788,0.01353476569056511,-0.015545669943094254,0.006692144554108381,0.0026561636477708817,0.0048660943284630775,-0.018292417749762535,-0.007460114546120167,0.022227099165320396,0.0106017105281353,0.05320962518453598,-0.02265460416674614,-0.01131453923881054,0.012853817082941532,-0.0002959979756269604,0.025417005643248558,-0.00955783948302269,0.0014118781546130776,-0.00904284231364727,-0.008947938680648804,-0.007168934214860201,-0.00964303594082594,-0.004022146109491587,-0.005613087210804224,-0.12938329577445984,-0.0043584736995399,0.020456742495298386,0.0071443296037614346,-0.011277008801698685,-0.02349260449409485,-0.010244361124932766,-0.00665429187938571,-0.010064574889838696,0.005249082110822201,0.005279236473143101,0.017985159531235695,-0.02883007377386093,0.010324330069124699,-0.012035149149596691,0.008913593366742134,0.008274752646684647,-0.0018126015784218907,-0.004603218752890825,0.00580825237557292,0.008159039542078972,0.01880655251443386,0.0002549282507970929,-0.004038217011839151,0.005237426608800888,-0.018459560349583626,-0.00046851334627717733,0.0023338748142123222,-0.0042199338786304,-0.006385834887623787,0.011244351975619793,0.0007573044276796281,0.01756402850151062,-0.008600994013249874,-0.0022277063690125942,-0.0030407358426600695,-0.007221739273518324,0.01820104382932186,-0.02493535354733467,0.01585320197045803,-0.0005586881306953728,0.0033721248619258404,-0.00026433906168676913,-0.000743469747249037,0.005868381354957819,0.006111698690801859,-0.0011203524190932512,0.011258958838880062,-0.0008901173714548349,-0.011496561579406261,-0.008037720806896687,0.016194118186831474,0.011407424695789814,-0.014084485359489918,0.017604801803827286,0.002007188042625785,-0.006658796686679125,-0.009705387987196445,0.015173210762441158,0.006459673400968313,-0.00285873725079,0.019698521122336388,0.012200135737657547,-0.008034748956561089,0.0028521015774458647,-0.00245031644590199,-0.006310049910098314,-0.00373665289953351,0.008135923184454441,-0.0090325390920043,-0.0002607999776955694,0.0046803392469882965,-0.01800999790430069,-0.008924789726734161,0.01823682151734829,-0.007351914420723915,-0.019322993233799934,0.012701595202088356,0.0053284624591469765,-0.0064052678644657135,0.019654009491205215,0.00013570864393841475,0.016256112605333328,0.007728443015366793,0.010437853634357452,0.00808533001691103,0.019011886790394783,0.012183984741568565,0.033292051404714584,0.005902435164898634,-0.018925726413726807,-0.00701944762840867,0.011261066421866417,0.005332435946911573,0.0031362916342914104,0.0005442180554382503,-0.0032328530214726925,-0.010592673905193806,-0.018920287489891052,-0.009756236337125301,-0.005785324610769749,-0.030977396294474602,0.001599933486431837,0.00013377821596805006,0.008112323470413685,-0.0063599590212106705,-0.005695757456123829,0.00597459077835083,0.01210800651460886,-0.006559251341968775,0.0007339463336393237,0.011125277727842331,0.022035440430045128,0.017060229554772377,0.01003420352935791,-0.0034310349728912115,0.00637843506410718,0.011094809509813786,-0.013998170383274555,-0.014564729295670986,0.01242771651595831,-0.0036663247738033533,-0.000654135481454432,0.00626980047672987,-0.0076171220280230045,-0.0020285514183342457,0.006653873715549707,0.012656455859541893,-0.01786595582962036,-0.008405892178416252,0.01965014822781086,-0.0021813763305544853,0.010792931541800499,-0.015798313543200493,-0.015769999474287033,-0.006753129884600639,-0.015076013281941414,0.007592670153826475,0.006454171612858772,0.02763102576136589,-0.008400551043450832,-0.0049078394658863544,-0.024386631324887276,0.006857115309685469,0.001914125750772655,-0.01439663302153349,-0.020056629553437233,0.008954518474638462,0.013706443831324577,0.007875348441302776,0.012146084569394588,-0.009473125450313091,0.009648504666984081,0.015645135194063187,0.01922854408621788,0.0068963672965765,0.008811811916530132,0.013530968688428402,-0.017957940697669983,-0.01021209079772234,0.0022633387707173824,-0.007277818396687508,-0.0031573977321386337,-0.11325757950544357,-0.0026099944952875376,0.01439537201076746,-0.004530924838036299,0.001019970397464931,-0.0020006245467811823,-0.004129558335989714,0.015971921384334564,-0.044551171362400055,0.0030149968806654215,0.007847486063838005,-0.01554462406784296,0.007680688984692097,-0.00788731686770916,-0.017942272126674652,-0.000786610587965697,0.005577197298407555,0.009266538545489311,-0.009329116903245449,-0.04451880231499672,-0.0037785109598189592,0.0028084840159863234,-0.009803786873817444,-0.010790380649268627,0.002866531489416957,0.0017853827448561788,0.007238357327878475,-0.007430804427713156,-0.004662869498133659,0.004536635708063841,0.01837938465178013,0.01211519818753004,0.0014415101613849401,-5.029150634072721e-05,0.021934866905212402,-0.010267108678817749,-0.013645731844007969,-0.015742121264338493,0.008256089873611927,-0.04040089249610901,0.07481249421834946,0.007236475590616465,0.009462444111704826,-0.027326276525855064,0.003720212262123823,0.000653174240142107,-0.002285812282934785,-0.0037178313359618187,0.012064619921147823,0.006163128651678562,-4.221188646624796e-05,-0.004891624208539724,-0.009622621349990368,0.0006778354290872812,0.013634954579174519,-0.020278330892324448,-0.004124345723539591,0.007662141229957342,0.018916331231594086,-0.0036245116498321295,0.01430609729140997,-0.01053135097026825,-0.012238960713148117,-0.016030864790081978,0.002648538677021861,0.014399755746126175,-0.008265534415841103,0.017143085598945618,-0.014470246620476246,-5.842742757522501e-05,-0.004861831199377775,-0.015087821520864964,-0.006019762251526117,0.01629151962697506,0.010227116756141186,-0.003751903073862195,-0.01222227606922388,0.0076263234950602055,0.042506661266088486,-0.01409455481916666,-0.0125817796215415,0.006965314969420433,-0.1917276829481125,0.00950542837381363,-0.01586632803082466,0.0023973588831722736,0.005743181332945824,-0.0027462500147521496,0.013118598610162735,0.011540125124156475,-4.4238830014364794e-05,0.0049981833435595036,0.010282487608492374,0.0003759496030397713,0.01399040874093771,0.018821081146597862,-0.014726671390235424,0.004507406149059534,0.011466688476502895,-0.005345562938600779,0.003956358879804611,-0.0034813869278877974,-0.0006390218622982502,-0.012699902057647705,0.006115961819887161,-0.00699468981474638,-0.00933891348540783,0.0034024324268102646,0.0066421241499483585,-0.002772600157186389,-0.00560781080275774,0.0124791469424963,0.008322587236762047,-0.009324386715888977,0.019184015691280365,-0.01484056655317545,0.004880982916802168,0.009200002998113632,-0.004697439726442099,-0.0016762494342401624,0.005595938302576542,0.0051397476345300674,0.015112820081412792,0.0016515520401299,0.0027893949300050735,0.004518795292824507,0.02610747143626213,0.010790864005684853,-0.00240150885656476,0.0018596394220367074,-0.00877827126532793,0.016919050365686417,-0.006034755613654852,0.004655871074646711,-0.007221192587167025,-0.010618927888572216,-0.010135614313185215,0.0057146274484694,-0.0011658620787784457,8.326552051585168e-05,-0.0037010847590863705,0.007693116553127766,-0.011633782647550106,-0.0017288855742663145,0.008993348106741905,0.006360128056257963,-0.006610793061554432,0.02352437563240528,0.001936598913744092,-0.011150550097227097,-0.01644146628677845,0.0009796085068956017,0.0030192439444363117,-0.0053696841932833195,0.013059624470770359,-0.0033805544953793287,0.016168439760804176,0.0018524626502767205,0.012617220170795918,0.005636119283735752,-0.016038715839385986,0.010487047955393791,-0.007545631844550371,-0.001429348485544324,-0.0017839670181274414,-0.008450678549706936,0.005330666434019804,-0.02991759404540062,0.00345455389469862,0.018851209431886673,-0.009807764552533627,0.027462579309940338,0.007071391679346561,0.0019209625897929072,-0.018841171637177467,-0.005503535736352205,0.02069077454507351,-0.020384222269058228,0.00936795026063919,0.007733526639640331,-0.009904591366648674,-0.004870839882642031,-0.03102888911962509,0.010977471247315407,0.015817424282431602,0.0011372757144272327,0.0072667705826461315,0.00784523319453001,-0.003772204741835594,0.015585226006805897,0.006962628103792667,-0.005917835980653763,-0.004866400267928839,-0.002367018721997738,0.005616626236587763,0.008822798728942871,-0.012629799544811249,-0.011987242847681046,0.0032996777445077896,0.0023828642442822456,0.012849369086325169,0.010437403805553913,0.008191507309675217,0.014551647007465363,-0.00907558761537075,-0.012082315981388092,-0.01734895631670952,-0.025283891707658768,0.011902658268809319,0.01442468911409378,-0.00960622914135456,0.009892510250210762,0.006284326780587435,0.09945326298475266,-0.000902246858458966,0.010209871456027031,0.006395020522177219,-0.014969841577112675,0.006021085660904646,0.005478468257933855,0.006624804809689522,-0.005861262790858746,0.018376680091023445,-0.005344887264072895,-0.008701054379343987,0.017867742106318474,0.02290046401321888,0.004558425396680832,-0.0031763159204274416,0.009653178043663502,0.017748555168509483,0.0004191588668618351,-0.020645441487431526,-0.0037479782477021217,0.01151856780052185,-0.018366899341344833,0.013412505388259888,-0.006302890833467245,0.006716001313179731,-0.00566723570227623,0.021751975640654564,-0.009203510358929634,-0.005479597952216864,-0.0036258467007428408,0.011007815599441528,-0.019736887887120247,0.0033232851419597864,-0.00348482932895422,0.005073791369795799,0.017230041325092316,0.020670218393206596,0.004283766727894545,-0.0009454562095925212,0.002031994052231312,-0.017311764881014824,-0.013582253828644753,-0.012368597090244293,0.010673816315829754,-0.0031707175076007843,0.008417531847953796,-0.004093330819159746,-0.01342865638434887,0.006839676760137081,0.007039966061711311,0.002886531176045537,-0.010179306380450726,0.01376741286367178,0.003229884896427393,-0.002050425624474883,-0.006090544629842043,-0.01241382211446762,-0.004899153020232916,-0.007758493069559336,-0.007976759225130081,-0.01766863465309143,0.0025243479758501053,0.0038350399117916822,0.011882581748068333,0.004422273952513933,-0.03836751729249954,-0.01081705279648304,-0.007251629140228033,-0.007358638569712639,0.007515196222811937,0.021443774923682213,-0.011086410842835903,0.003115957835689187,0.01913968101143837,0.023567553609609604,0.0044838543981313705,0.002975921845063567,-0.01662723533809185,-0.006301764864474535,0.011563225649297237,-0.007714479696005583,0.007416438311338425,-0.035197507590055466,0.009823915548622608,-0.017413947731256485,0.011747097596526146,-0.0038893171586096287,0.021576901897788048,0.01757732592523098,0.013345262035727501,-0.006837489083409309,0.029992317780852318,-0.011094197630882263,0.010682325810194016,0.002443913836032152,-0.0005208277725614607,-0.01606852374970913,0.010624848306179047,0.0047839065082371235,0.01419053040444851,-0.01350423227995634,0.012274585664272308,0.012537653557956219,0.007614258676767349,-0.0039986432529985905,0.010640677064657211,-0.0038547625299543142,-0.006087520159780979,0.027305202558636665,0.006098201964050531,-0.00494043156504631,0.004934415221214294,-0.01824975572526455,0.001602957840077579,0.026787754148244858,0.005400836933404207,0.008201074786484241,0.022710701450705528,0.005333361215889454,0.007449979893863201,-0.00023634797253180295,-0.011554860509932041,0.00011505313159432262,0.006364085711538792,0.0009316215291619301,0.012276645749807358,-0.002286005299538374,0.007153740152716637,-0.00578177347779274,-0.003366011893376708,0.016108853742480278,-0.007560239173471928,-0.012466534040868282,5.5177883041324094e-05,0.013790159486234188,-0.012926618568599224,1.878943839983549e-05,0.0008286013035103679,-0.0036813300102949142,-0.0005811856244690716,-0.0008696871809661388,-0.008247340098023415,0.02868564799427986,-0.014315041713416576,-0.017415814101696014,0.006972618401050568,-0.024270612746477127,-0.009373226203024387,0.0051077669486403465,0.0038382895290851593,-0.01722528040409088,0.015512949787080288,0.01026356965303421,0.00711700227111578,-0.010315561667084694,0.01249308604747057,0.014615736901760101,-0.002677438547834754,0.005468305200338364,-0.005088237579911947,-0.018737059086561203,-0.003193721640855074,0.0038784947246313095,0.0009255004115402699,0.006019891239702702,0.0115288645029068,-0.018515832722187042,-0.005315995309501886,0.0148364482447505,0.009229088202118874,-0.002652656752616167,0.005572419613599777,0.007090028841048479,-0.00805481243878603,0.027019791305065155,-0.005165357608348131,0.01384897343814373,-0.01675380766391754,0.014895391650497913,0.001922378083691001,-0.007131235208362341,0.010457383468747139,-0.0060896435752511024,-0.0035761059261858463,-0.017283009365200996,0.013179706409573555,0.01639494299888611,0.0069476836360991,-0.010041441768407822,-0.004489645827561617,-0.01367124542593956,-0.0003028188075404614,0.012466919608414173,-0.010653103701770306,0.008282281458377838,0.003187681082636118,-0.01343492977321148,-0.010245668701827526,-0.011471674777567387,-0.01613684557378292,-0.0010712954681366682,-0.0027505853213369846,-0.001911632250994444,-0.0011440966045483947,-0.02027985267341137,-0.003082658164203167,-0.0005120121641084552,-0.004386079031974077,-0.010168688371777534,0.0036431557964533567,0.006260099820792675,-0.010663633234798908,-0.002148623578250408,-0.002349805785343051,0.0030768970027565956,-0.0034179803915321827,-0.008466539904475212,-0.011844230815768242,-0.005494784563779831,0.0010436181910336018,0.011641600169241428,-0.011137792840600014,7.610687316628173e-05,0.005389544181525707,-0.023192087188363075,-0.005416119936853647,-0.009617231786251068,0.008793344721198082,-0.024386076256632805,0.020657410845160484,5.134117236593738e-05,-0.007362756412476301,-0.009800750762224197,0.006533399689942598,-0.010050579905509949,0.006684471387416124,0.011441572569310665,0.006047689355909824,0.016310229897499084,-0.005246692802757025,0.007157488260418177,0.0017344196094200015,-0.00866750068962574,0.0006803951691836119,0.00713065592572093,-0.0014674743870273232,0.0203915573656559,-0.005685457959771156,-0.007061901036649942,-0.016780640929937363,0.001550675486214459,-0.008510038256645203,-0.011533658020198345,-0.008761588484048843,0.022064397111535072,-0.0017128309700638056,0.0062705883756279945,0.0048079160042107105,0.018406344577670097,0.010051971301436424,0.003991404082626104,0.012091951444745064,-0.005227489396929741,-0.0035770712420344353,-0.009186764247715473,-0.0038295702543109655,-0.00698986416682601,0.012210141867399216,0.005487545393407345,-0.0013136116322129965,0.0018605402437970042,-0.011810770258307457,-0.001065592747181654,0.0004330579249653965,0.024547435343265533,-0.0043790326453745365,-0.0002492174389772117,-0.0189106035977602,-0.010918785817921162,0.020448731258511543,0.007792806718498468,-0.002034664386883378,0.008813790045678616,-0.01989891566336155,0.001182962441816926,0.000261572131421417,-0.0074978540651500225,0.0019776527769863605,-0.011139015667140484,-0.02664639614522457,0.0028707943856716156,0.007007550913840532,-0.017508666962385178,-0.014156038872897625,-0.02033647708594799,0.016214512288570404,0.006000136490911245,-0.016533177345991135,0.018597586080431938,0.005563668441027403,-0.00725555419921875,0.01448176521807909,0.016186457127332687,-0.016622057184576988,0.007171966601163149,0.009879093617200851,0.014025414362549782,0.015332052484154701,0.018447238951921463,0.01657157577574253,-0.01883309707045555,0.0012578627793118358,-0.01160209160298109,-0.0029103304259479046,-0.024813447147607803,-0.008269749581813812,0.019136399030685425,0.12509235739707947,0.00992282573133707,-0.010059620253741741,-0.006295362021774054,-0.009466594085097313,-0.005341983400285244,-0.006175258196890354,-0.00834791548550129,0.0037003285251557827,-0.009935236535966396,-0.022054295986890793,-0.021636681631207466,0.00747463246807456,0.0023884624242782593,0.0020293877460062504,0.000621370563749224,-0.010186834260821342,0.0025970444548875093,0.004555682651698589,0.010875705629587173,-0.00799268577247858,-0.010559020563960075,-0.018151158466935158,0.006607222370803356,0.00013699558621738106,0.0032064514234662056,-0.01213186327368021,0.017665095627307892,-0.001385656651109457,-0.013753159902989864,-0.0032455134205520153,0.004236889537423849,0.011882774531841278,-0.014331771992146969,0.007972095161676407,0.0015528311487287283,0.0077825915068387985,0.0031973575241863728,0.007028214633464813,-0.014710456132888794,0.019549252465367317,-0.013456358574330807,0.006737617775797844,-0.015732519328594208,0.0006138741155155003,0.0037009399384260178,0.011282256804406643,0.010245632380247116,0.002517430577427149,0.007911423221230507,0.00890109408646822,-0.010392270050942898,-0.017399711534380913,-0.02358563430607319,-0.006632172502577305,0.010217915289103985,-0.022281570360064507,0.007806669920682907,0.013242524117231369,-0.0033365730196237564,0.026809824630618095,-0.013774974271655083,-0.00872904434800148,-0.010284706950187683,-0.014805947430431843,0.015970248728990555,0.017862962558865547,0.015086662955582142,0.0027441910933703184,0.010856385342776775,-0.004200211260467768,-0.0081545514985919,0.0031795732211321592,-0.026753583922982216,0.014192008413374424,-0.012117899954319,-0.0035813823342323303,0.015963943675160408,-0.0860016718506813,0.03140305355191231,0.007273109629750252,-0.00939896609634161,0.008446688763797283,-0.00541621632874012,-0.0522768460214138,-0.0012892642989754677,-0.009854674339294434,-0.0076980385929346085,-0.015288103371858597,-0.03279374539852142,-0.014441356062889099,-0.005670452956110239,-0.0029624251183122396,-0.012520995922386646,-0.0102844825014472,-0.017415877431631088,-0.015840580686926842,-0.013365293852984905,-0.009166606701910496,-0.005349005106836557,-0.005249958485364914,0.019897757098078728,-0.007069654297083616,-0.009444724768400192,0.004441514145582914,-0.01018715649843216,0.009931439533829689,0.002962167840451002,-0.013154460117220879,0.014917655847966671,-0.015001467429101467,0.009532036259770393,-0.0044509246945381165,0.028517216444015503,0.00990370661020279,-0.010221325792372227,-0.010877507738769054,0.0023901837412267923,0.02150103636085987,-0.014040149748325348,-0.0007246803143061697,0.00785189401358366,0.0014458857476711273,-0.0006708737928420305,0.004349204711616039,-0.01244916021823883,-0.01190697681158781,-0.1309737116098404,-0.0030378401279449463,0.005152037832885981,-0.025020644068717957,0.013737556524574757,0.01354216504842043,-0.010803540237247944,-0.020594704896211624,-0.010123742744326591,-0.005482333246618509,0.007814539596438408,0.0062471660785377026,0.011471273377537727,0.014933951199054718,0.010366315953433514,-0.017068468034267426,0.0075530968606472015,0.0021459211129695177,-0.005174430552870035,0.004797837696969509,-0.0006980726611800492,-0.01761162281036377,-0.011748763732612133,0.007687899749726057,-0.015306426212191582,0.007811580318957567,-0.004673641175031662,0.019404791295528412,0.006644575856626034,-0.009581189602613449,0.01846865750849247,-0.00799687672406435,-0.008734514936804771,0.025797318667173386,0.004079817328602076,0.01512935757637024,-0.0006804736331105232,-0.0038689833600074053,0.006711303722113371,-0.014750850386917591,0.016202479600906372,0.01031462848186493,-0.005430308170616627,0.01708185113966465,0.008559875190258026,-0.005445751361548901,-0.0028198380023241043,-0.0038498397916555405,-0.006423091981559992,0.013393329456448555,0.008289198391139507,0.019474737346172333,0.013462373986840248,-0.009793463163077831,-0.013543033972382545,0.03380116820335388,0.057620640844106674,0.0037551848217844963,0.01428164541721344,0.011203941889107227,-0.00013776373816654086,-0.007206891197711229,0.011069182306528091,-0.0032131224870681763,0.009809983894228935,0.006570447236299515,-0.002480398863554001,0.022422587499022484,0.011351908557116985,-0.01595130003988743,-0.019222430884838104,0.00509705301374197,-0.006570335011929274,0.0017189440550282598,0.027080731466412544,-0.011916235089302063,0.0015000663697719574,-0.0020198484417051077,-0.02209283970296383,0.006771082524210215,0.0002977755793835968,-0.019696606323122978,0.008564154617488384,-0.0007474914309568703,0.011921319179236889,0.009810338728129864,0.014718177728354931,0.0014345606323331594,0.008807356469333172,-0.006630355026572943,-0.003958745859563351,-0.009559383615851402,-0.005430855322629213,-0.014630086719989777,-0.011925501748919487,0.0004732106754090637,0.018642853945493698,-0.013681734912097454,0.010839325375854969,-0.014961443841457367,0.0016361128073185682,0.0032435106113553047,-0.002405848354101181,-0.018609875813126564,0.0033618290908634663,0.011865722015500069,-0.012829582206904888,0.008958829566836357,-0.011033131740987301,0.007112349383533001,-0.007317069917917252,-0.003843147773295641,0.015338101424276829,0.0060599129647016525,0.013022753410041332,0.022979997098445892,-0.010455581359565258,0.003293846268206835,0.011678189970552921,0.03189416974782944,-0.0003863417077809572,0.006824394688010216,-0.008517374284565449,0.012291766703128815,-0.008964218199253082,0.007173221092671156,0.019597060978412628,0.0208904929459095,-0.008607679978013039,0.02034304104745388,0.010004634968936443,0.011900341138243675,-0.00043498832383193076,0.0033996535930782557,-0.002569137839600444,0.009322158992290497,-0.002651530783623457,-0.008777949027717113,-0.005856899078935385,-0.013607734814286232,0.0010277243563905358,-0.011572104878723621,-0.023325929418206215,0.008436039090156555,0.0016878400929272175,-0.0035754949785768986,0.010810618288815022,0.020025212317705154,-0.009496903046965599,0.01064186729490757,0.0021814408246427774,-0.0061418297700583935,-0.006570986472070217,0.01253622304648161,0.01944899745285511,-0.010414046235382557,0.00017785617092158645,0.006716644857078791,0.011308281682431698,0.014264336787164211,-0.0031749242916703224,-0.020774956792593002,-0.0003114172432105988,0.011388715356588364,-0.009031891822814941,-0.006522138603031635,0.018276477232575417,0.0024473723024129868,0.002980136778205633,-0.007986669428646564,0.010007386095821857,0.009231405332684517,-0.018392913043498993,-0.020028775557875633,0.012274328619241714,-0.008668269030749798,0.0041609592735767365,-0.0037708855234086514,-0.009803260676562786,-0.004945358261466026,-0.01740073226392269,0.0035423238296061754,-0.007416149135679007,0.023602621629834175,0.005355633329600096,-0.0019859694875776768,0.01988109014928341,7.979076144692954e-06,-0.006595607381314039,0.0053070830181241035,0.008229612372815609,0.016438249498605728,0.006289506796747446,0.00754022691398859,0.011281898245215416,0.00024167270748876035,0.006314409431070089,-0.0031186926644295454,-0.02108895592391491,-0.013352083042263985,0.020173614844679832,0.008024762384593487,0.013543741777539253,-0.015686606988310814,-0.008190031163394451,0.015606686472892761,-0.008021931163966656,-0.015871604904532433,0.0037902863696217537,0.0008586193434894085,0.003796238452196121,-0.010971165262162685,0.007283883169293404,-0.016522156074643135,0.0055426545441150665,-0.018035799264907837,-0.009387576021254063,-0.00015417633403558284,-0.009344720281660557,-0.005082639399915934,0.007296253461390734,-0.009880026802420616,-0.002254636026918888,0.02115420438349247,-0.00485372357070446,0.004400492645800114,-0.00884152390062809,-0.006040804088115692,0.011755109764635563,0.008026177994906902,-0.006253858096897602,-0.0029635189566761255,0.007403810508549213,0.0043754614889621735,0.026068542152643204,-0.024823419749736786,-0.004859900567680597,0.0077138361521065235,0.0007009119726717472,-0.018028592690825462,-0.011082421988248825,-0.007141128182411194,-0.01778709888458252,0.009043511003255844,0.0008742235950194299,0.019595323130488396,-0.00226938771083951,-0.0021313303150236607,0.0028745909221470356,0.013393265195190907,0.0035802884958684444,-0.0015817874809727073,0.006639556493610144,0.006195977795869112,-0.007812898606061935,-0.008897827938199043,-0.012519138865172863,0.014377216808497906,0.00478403503075242,-0.004690281115472317,0.003118644468486309,0.027247516438364983,-0.002435001777485013,0.033513087779283524,0.01822897233068943,0.007350771687924862,0.0011077403323724866,0.013501819223165512,-0.015879904851317406,0.013183299452066422,0.011308056302368641,-0.0003690966113936156,-5.669004895025864e-05,0.006077144294977188,-0.0071005732752382755,0.005103584378957748,0.012177292257547379,-0.0015176330925896764,0.00743842963129282,0.006680489517748356,0.004452131222933531,0.004653377924114466,-0.008840574882924557,-0.0031223606783896685,-0.013772077858448029,-0.005994860082864761,0.0052159992046654224,0.00597047246992588,-0.004418735392391682,-0.009556038305163383,-0.005633131135255098,0.02587483637034893,-0.002589789219200611,-0.0176318921148777,-0.009988966397941113,-0.015307571738958359,-0.009621800854802132,-0.002565787872299552,-0.01531350426375866,0.014097933657467365,-0.0033172364346683025,0.001826854539103806,0.0018190363189205527,-0.008359553292393684,-0.0038599425461143255,-0.004618598148226738,-0.0021358828525990248,-0.0039221663028001785,-0.0034684045240283012,-0.004433149006217718,0.006080731749534607,-0.0017949383473023772,-0.008630593307316303,0.001273048692382872,-0.019467659294605255,-6.12587173236534e-05,-0.018115075305104256,-0.006602621171623468,-0.007384441327303648,-0.007939839735627174,0.0019286199240013957,0.0008089773473329842,-0.01783713512122631,0.010118434205651283,-0.014237920753657818,0.01597065106034279,0.016588177531957626,-0.01785440556704998,0.01155418436974287,-0.005966603755950928,-0.014077438972890377,-0.013903025537729263,-0.002557036466896534,-0.021007491275668144,-0.005378428380936384,0.012218442745506763,0.004273728467524052,0.011610778979957104,-0.004312143661081791,0.01642666570842266,-0.023566925898194313,0.013862889260053635,0.015911821275949478,0.004173909313976765,-0.024028481915593147,-0.01222963910549879,-0.005391822662204504,0.011719332076609135,-0.007083456497639418,-0.0073945121839642525,0.010108668357133865,0.013066895306110382,-0.0004766210913658142,-0.006762267090380192,-0.0007032324792817235,0.0023309518583118916,0.012527922168374062,-0.006683377083390951,0.012418627738952637,-0.008594752289354801,-0.0089180339127779,-0.0018390804762020707,-0.01272482518106699,0.015199174173176289,-0.012042034417390823,-0.010652774013578892,0.001955002313479781,0.009363831952214241,-0.009031509980559349,-0.0028586569242179394,-0.0013132980093359947,0.009787592105567455,0.008148052729666233,0.004363750107586384,0.009258558973670006,-0.024081429466605186,0.01084060501307249,0.02108844183385372,-0.01939285360276699,0.011464710347354412,-0.010239985771477222,-0.009829654358327389,0.02925250120460987,-0.006770503241568804,-0.0068392264656722546,0.0012964068446308374,-0.016846660524606705,0.0068872300907969475,-0.003937834873795509,-8.339421765413135e-05,0.008675314486026764,-0.005402928218245506,-0.009232563897967339,0.011987275443971157,0.006109446752816439,-0.006341531407088041,0.007804907858371735,-0.007662084884941578,0.006093183066695929,-0.018207769840955734,-0.006304789334535599,0.000968299456872046,0.011293482035398483,0.0006706284475512803,0.00998291838914156,-0.016655774787068367,0.004729790613055229,0.008077752776443958,-0.0064179119653999805,-0.006763167679309845,0.0055464874021708965,-0.006630998104810715,-0.006346454378217459,0.0029069576412439346,0.004286420997232199,-0.00612212298437953,0.009613017551600933,-0.007194488774985075,-0.014121548272669315,-0.013963254168629646,0.008268116973340511,0.018683167174458504,0.00021566831856034696,0.010583395138382912,0.0023251124657690525,0.005577534902840853,-0.005223962478339672,-0.010808792896568775,-0.00891019869595766,0.0025711446069180965,-0.009238084778189659,0.00847254041582346,0.002356433542445302,-0.020508840680122375,0.008203793317079544,-0.013110458850860596,-0.00429300032556057,0.00894743949174881,-0.0010654800571501255,0.007953747175633907,0.0008857498760335147,0.008226757869124413,0.006239090580493212,-0.003030576976016164,-0.011644785292446613,-0.016018863767385483,0.0014197607524693012,0.012671319767832756,-0.014869586564600468,-0.011633380316197872,-0.0008804009412415326,0.005208792630583048,-0.009140313602983952,-0.004907278809696436,-0.01574484072625637,0.007207204587757587,-0.025614989921450615,0.010377657599747181,0.005622417200356722,0.020156607031822205,-8.534072549082339e-05,-0.013232074677944183,0.0025512452702969313,0.0074208625592291355,0.003769534407183528,0.006363023538142443,0.001976124243810773,-0.009836303070187569,0.014816982671618462,-0.02623211219906807,-0.013312103226780891,0.018329545855522156,0.011043942533433437,0.004413313698023558,-0.0026370524428784847,-0.006824623793363571,-0.01342408824712038,0.01530361082404852,0.02297188900411129,-0.015759512782096863,-0.0038370348047465086,0.008708260953426361,0.0386798270046711,0.006922588218003511,-0.014513103291392326,0.006315784528851509,0.0011656669666990638,-0.00011241488391533494,-0.0043263561092317104,0.006935876328498125,0.01871299184858799,-0.0018523683538660407,0.01645565964281559,0.0006411654176190495,-0.017343293875455856,0.01558641716837883,0.003914637491106987,-0.003911966923624277,0.010716164484620094,0.010333998128771782,0.009289140813052654,0.002327702473849058,-0.0016474217409268022,0.0085306940600276,-0.006147765554487705,-0.0027541646268218756,0.012298844754695892,-0.011853464879095554,0.0022197917569428682,0.009226707741618156,0.02173178642988205,-0.017738966271281242,-0.010917370207607746,-0.0029402251821011305,0.0004863214853685349,-0.0067732385359704494,-0.009347519837319851,-0.0026199843268841505,0.00044122201506979764,0.007049706764519215,-0.005566982086747885,-0.009083359502255917,0.005341717973351479,0.0016353566898033023,0.0075265211053192616,-0.025540797039866447,-0.00833797361701727,-0.00534829730167985,-0.004227929282933474,0.016433872282505035,0.006095499265938997,0.0034416201524436474,0.006703711114823818,-0.013493518345057964,-0.00048759233322925866,0.02160598710179329,-0.018758028745651245,-0.013188640587031841,0.00872473418712616,0.01274280995130539,-0.002263290574774146,-0.0006550966063514352,-0.01119509432464838,-0.010811157524585724,-0.007531395647674799,0.0025357375852763653,0.01623639091849327,0.012533069588243961,-0.11452934145927429,-0.014385758899152279,-0.0036055126693099737,0.002186845988035202,0.013855954632163048,-0.0006583944195881486,0.0048728990368545055,0.009528513066470623,0.003839930286630988,0.01954481191933155,0.001959699671715498,-0.00801488570868969,0.01553120743483305,0.010433783754706383,0.00287243933416903,0.0030284454114735126,0.0027071910444647074,0.005127111449837685,0.007968137040734291,0.004281257279217243,-0.011975499801337719,-0.017328623682260513,0.008220185525715351,0.007401622831821442,-0.013764807023108006,0.007864666171371937,-0.004687312990427017,-0.004217983223497868,-0.01190197467803955,0.005709093064069748,0.012869670987129211,-0.013801033608615398,-0.011998728848993778,0.20357556641101837,-0.0030479426495730877,0.012771195732057095,-0.0171239972114563,0.005747669842094183,0.00899829063564539,-0.014829105697572231,0.00494075333699584,-0.008008965291082859,-0.0036376866046339273,-0.033662255853414536,0.0065314690582454205,-0.009848415851593018,0.013626010157167912,0.012002847157418728,-0.013834439218044281,0.02108149044215679,0.016931405290961266,-0.0017394707538187504,-0.00963470246642828,-0.005704395938664675,0.01754046231508255,-0.015337469056248665,0.015215389430522919,-0.005915905814617872,-0.025276893749833107,-0.005014732480049133,-0.00463339826092124,-0.020541712641716003,-0.001968644093722105,0.000676644966006279,0.01785305328667164,-0.011794249527156353,0.016294624656438828,-0.004089083522558212,0.006442975252866745,-0.02364637888967991,-0.010055324994027615,0.008496284484863281,0.005891228560358286,0.010857462882995605,-0.0347641259431839,-0.014917171560227871,0.017434941604733467,-0.01820305548608303,-0.02300403080880642,-0.01460286695510149,-0.026439635083079338,-0.005786696448922157,0.005840812344104052,-0.002880639396607876,0.005296160001307726,-0.004211021587252617,-0.002037527970969677,-0.010035361163318157,0.004914330784231424,0.004394669085741043,0.005622674711048603,0.0011111185885965824,0.009060111828148365,-0.01080778706818819,-0.014376429840922356,-0.008422542363405228,0.0036981890443712473,-0.026923397555947304,0.009801522828638554,-0.0014322763308882713,-0.013493984937667847,0.012008155696094036,0.012425931170582771,0.009741486981511116,0.02373787946999073,0.0018142102053388953,-0.0050240508280694485,0.01613137498497963,0.005036276765167713,0.0027613716665655375,0.005145667586475611,-0.005073678679764271,0.00631151394918561,0.015935149043798447,0.005443435162305832,-0.0074535515159368515,0.012360554188489914,0.009225227870047092,0.010121893137693405,0.0003564523358363658,0.0020175480749458075,0.0005545940366573632,-0.018256383016705513,-0.0015494207618758082,-0.004463328048586845,0.010256974026560783,0.005540004465728998,-0.005248623434454203,0.005901942495256662,0.010503585450351238,-0.008990907110273838,0.008495476096868515,-0.029623478651046753,-0.0010746014304459095,0.010479615069925785,0.007128741126507521,-0.004881907254457474,-0.012746831402182579,-0.005546809174120426,-0.004563066177070141,0.0002746024983935058,-0.012642459943890572,-0.003734111087396741,0.01777506433427334,0.0049340128898620605,-0.0012290994636714458,-0.00021181550982873887,0.0020156176760792732,0.0010072377044707537,0.003468742361292243,-0.003944575320929289,0.014315459877252579,-0.005033606663346291,0.004686838481575251,-0.012386228889226913,0.0018407534807920456,0.004675609990954399,-0.0087699294090271,-0.005062884651124477,-0.0077690305188298225,0.00480366125702858,-0.012847527861595154,-0.007804791443049908,-0.0020366229582577944,0.010552520863711834,0.0009618164622224867,-0.02200361341238022,-0.02055400423705578,0.007025834172964096,0.005628401413559914,-0.003323606913909316,-0.00350605184212327,0.006432036403566599,0.004809271544218063,0.010274733416736126,0.04477909207344055,-0.009266168810427189,-0.014458194375038147,0.003407451556995511,-0.003966630436480045,0.00690626073628664,-0.005162558518350124,-0.017314080148935318,-0.0033658831380307674,-0.019236072897911072,-0.010986302979290485,-0.009487057104706764,-0.0126802958548069,0.009735309518873692,0.04154672846198082,-0.018142199143767357,0.002596642356365919,-0.0076661063358187675,0.013936100527644157,0.058171678334474564,-0.025674721226096153,-0.006219496950507164,-0.014702396467328072,0.007355244364589453,-0.01217672135680914,-0.01009633019566536,0.008379188366234303,-0.00898730382323265,-0.0017007015412673354,0.003610322717577219,0.0026148527394980192,0.0058074044063687325,-0.016003387048840523,-0.011510750278830528,0.0013994108885526657,-0.005675825756043196,-0.010906624607741833,0.003757855389267206,0.008256155997514725,0.0037957236636430025,0.0004637596430256963,0.0059378482401371,-0.006037457846105099,-0.018181998282670975,0.0013030506670475006,0.007541135419160128,0.009224391542375088,0.010982869192957878,-0.0036199912428855896,-0.002958113793283701,0.01651797443628311,-0.03149764612317085,0.004628603812307119,0.00334406946785748,-0.007923029363155365,0.015490380115807056,0.020828863605856895,0.016824204474687576,-0.0038670848589390516,0.014724436216056347,0.000400498160161078,0.0663076639175415,0.00567030580714345,-0.013410317711532116,0.008589716628193855,-0.008427352644503117,-0.01424303650856018,0.0008962303982116282,-0.009365360252559185,0.008820024318993092,0.013941312208771706,-0.007390265353024006,0.015612092800438404,0.008377837017178535,-0.006962129846215248,0.01604386232793331,0.004204136785119772,0.0069089229218661785,-0.0185052789747715,-0.013314954936504364,0.007275469601154327,0.014722811058163643,0.008437100797891617,0.011726523749530315,0.016620544716715813,0.015615695156157017,0.0120353102684021,0.006396838463842869,-0.008448812179267406,-0.00602632574737072,0.010790380649268627,0.002144247991964221,-0.014843912795186043,0.013109751045703888,-0.0005983744049444795,-0.01191713660955429,-0.0060539147816598415,0.007560625206679106,0.018343864008784294,-0.02141418308019638,-0.0038201757706701756,-0.0008210405358113348,0.0037896588910371065,0.00903385877609253,0.02255813404917717,0.0149000883102417,0.010207773186266422,0.01298686396330595,0.01658656820654869,-0.009689725004136562,-0.000968685548286885,-0.0354095958173275,-0.0020211192313581705,0.0172839667648077,0.017595110461115837,-0.007312276400625706,-0.009096597321331501,-0.012832960113883018,0.006029736716300249,0.01993134617805481,-0.007445869967341423,-0.013995345681905746,-0.021392418071627617,0.013174227438867092,0.0006699688965454698,0.0026909918524324894,0.0032831323333084583,0.012930993922054768,0.0012651460710912943,0.000811227539088577,0.01763002574443817,-0.00523826340213418,0.016636181622743607,-0.011958190239965916,-0.00934743881225586,0.011710581369698048,-0.009352635592222214,0.001517037977464497,0.022132251411676407,-0.0027835392393171787,-0.021134112030267715,0.000661684141959995,0.0020901961252093315,0.008411427959799767,-0.02320259064435959,-0.023216569796204567,-0.02040291577577591,-0.0019324647728353739,-0.012253865599632263,-0.012067129835486412,-0.012556578032672405,-0.006384226027876139,0.008578809909522533,-0.0006862648879177868,0.018786733970046043,0.008309703320264816,-0.004579378291964531,0.008779493160545826,-0.012430795468389988,0.010612075217068195,0.006497509777545929,0.00468828622251749,0.020637301728129387,0.014828919433057308,0.008801830001175404,-0.0012163587380200624,0.011090272106230259,0.00605464493855834,-0.00599315483123064,0.003595965448766947,0.0026772695127874613,0.007111930754035711,-0.0021474009845405817,-0.15517501533031464,-0.007093977648764849,0.016207048669457436,-0.003689244855195284,0.02290702797472477,-0.024147450923919678,0.02058466523885727,-0.003728344105184078,0.0020039579831063747,0.0036031962372362614,-0.00701624620705843,0.001598936039954424,-0.015112241730093956,-0.026839423924684525,-0.0005213304539211094,0.04432762786746025,0.0021426393650472164,0.008228357881307602,0.0006260357331484556,-0.0051366910338401794,0.0046644131653010845,-0.0015309208538383245,0.007084615062922239,-0.010650690644979477,-0.01891385205090046,-0.017962105572223663,-0.019904641434550285,-0.003021359210833907,0.00939719658344984,0.014427713118493557,0.0003639488131739199,0.01590440608561039,-0.007913827896118164,-0.008794532157480717,-0.004160219803452492,-0.00011183575406903401,-0.023288607597351074,0.001976816216483712,0.022937526926398277,-0.009748597629368305,-0.014059019275009632,-0.022420817986130714,0.014181907288730145,0.0013818360166624188,0.0023023937828838825,-0.007540484424680471,0.01842080056667328,0.006028867792338133,-0.022552955895662308,-0.005644746124744415,-0.0043883309699594975,-0.004599744454026222,-0.008561484515666962,0.014006786048412323,-0.011542826890945435,-0.009602931328117847,-0.036284975707530975,0.0013754897518083453,0.012572064064443111,0.006309454329311848,-0.0002941721468232572,-0.004653667565435171,-0.013862421736121178,0.004336177371442318,0.010433993302285671,0.009525666013360023,-0.006532643456012011,-0.0015942708123475313,0.014698229730129242,0.013635436072945595,0.01483591366559267,0.004928945563733578,0.011660551652312279,0.00346562173217535,-0.009555619210004807,0.01836557686328888,0.011766644194722176,0.005703310016542673,-0.005696287844330072,0.008640498854219913,0.00856035016477108,-0.03719845414161682,0.016891704872250557,0.009445746429264545,-0.0034338664263486862,-0.005024726502597332,-0.016796855255961418,-0.008475210517644882,-0.017073003575205803,0.004128266125917435,0.016665266826748848,0.00954902358353138,0.010982382111251354,-0.008389675989747047,-0.012186558917164803,0.008364107459783554,0.017737936228513718,0.01394137553870678,0.013139929622411728,-0.008969285525381565,-0.01151264924556017,-0.007080208044499159,-0.02486042119562626,0.00451834499835968,0.01454064343124628,-0.0027549047954380512,-0.01847361959517002,0.012725340202450752,0.02681497111916542,0.0022874209098517895,0.0060871499590575695,-0.012228837236762047,-0.01910441741347313,-0.02300979010760784,0.004791234154254198,-0.00982105266302824,-0.007742567453533411,0.01883193850517273,0.0016032794956117868,-0.0007860033656470478,-0.00030844920547679067,0.0010288181947544217,-0.01645890437066555,0.014252045191824436,-0.01001357939094305,0.002469572238624096,-0.025139495730400085,-0.007612746674567461,-0.05701448768377304,0.008700916543602943,0.01902882568538189,-0.02189522795379162,0.015759384259581566,0.010229690931737423,-0.013251837342977524,-0.013460122980177402,-0.01524634100496769,0.0020383321680128574,0.014956198632717133,-0.007906491868197918,-0.013498730957508087,0.006993595976382494,0.003018873743712902,0.001712734461762011,0.03202492371201515,0.026156842708587646,0.008240841329097748,-0.017780285328626633,0.006188404746353626,-0.014345478266477585,0.0025132661685347557,0.011938242241740227,-0.00015267223352566361,0.0147481644526124,-0.00812479481101036,-0.0010659064864739776,-0.0005582457524724305,0.006272712256759405,-0.004541509784758091,0.0014816629700362682,-0.02871515043079853,0.0016121916705742478,-0.02394980750977993,0.0008420820813626051,-0.007255136035382748,-0.006515704095363617,-0.005095303524285555,-0.005030743312090635,-0.011658716946840286,0.028127659112215042,0.00975873228162527,0.021014409139752388,-0.0160182137042284,0.008259791880846024,-0.00808415561914444,-0.011482791975140572,-0.0018780268728733063,-0.0016436574514955282,0.01837550289928913,0.0003763035056181252,0.009928029961884022,-0.008596843108534813,-0.0039632199332118034,0.01536337286233902,0.0038513196632266045,0.01520631741732359,-0.012446328997612,0.01358643639832735,-0.01477467454969883,0.0018546526553109288,-0.013842265121638775,-0.0008109700866043568,0.015721803531050682,0.006470515858381987,-0.01047314889729023,-0.017738599330186844,-0.002085148822516203,-0.00151948316488415,0.000500236579682678,-0.011062928475439548,-0.012429083697497845,-0.008604375645518303,-0.0033165609929710627,0.0162813700735569,-0.00872577540576458,0.006237449590116739,0.0014139856211841106,0.00227738288231194,0.007259607780724764,-0.0024163410998880863,-0.000929530244320631,0.01526214275509119,0.0005013305344618857,0.012352321296930313,0.0024202982895076275,-0.004930940456688404,0.005372138228267431,0.013471262529492378,0.011361593380570412,0.020780909806489944,-0.016667872667312622,-0.01875338703393936,-0.0006402565049938858,-0.0038189534097909927,-0.0173107348382473,-0.0007631341577507555,-0.004413474816828966,0.006579649168998003,-0.0007289272034540772,-0.016239607706665993,0.007476409897208214,5.302224599290639e-05,-0.01624462567269802,-0.014696476981043816,-0.0008294378640130162,6.569868855876848e-05,-0.006026261951774359,-0.0035658427514135838,0.00035259153810329735,-0.003949449863284826,0.009364716708660126,-0.010776331648230553,0.002928385278210044,-0.009490063413977623,-0.01819232851266861,0.004032875876873732,-0.0032316383440047503,0.00964342150837183,-0.0010484643280506134,-0.016542362049221992,-0.013282490894198418,-0.02188814990222454,0.014662325382232666,0.003973450977355242,0.01259040366858244,0.003396448213607073,0.0023380222264677286,-0.01695997640490532,0.012070347554981709,0.007248966954648495,0.011380953714251518,-0.009349804371595383,0.005258500576019287,0.01802116073668003,0.00570098590105772,-0.011989140883088112,0.011402743868529797,0.010607988573610783,0.008799505420029163,-0.009475105442106724,0.008064079098403454,-0.012264966033399105,-0.006731090601533651,0.00045869231689721346,-0.014379839412868023,-0.007578159682452679,-0.019541822373867035,0.02880922518670559,-0.01217967364937067,-0.0017422698438167572,0.009241893887519836,0.011424331925809383,-0.0059761349111795425,-0.10590112954378128,0.01093854196369648,-0.019668808206915855,-0.008417797274887562,-0.012183469720184803,-0.015398330055177212,0.022412968799471855,-0.014847170561552048,0.012399098835885525,-0.011321166530251503,-0.020581383258104324,-0.012875880114734173,0.009312482550740242,-0.01491408422589302,0.010381936095654964,0.014163745567202568,-0.00536081288009882,0.0030865189619362354,-0.017042148858308792,0.009154188446700573,0.003824438899755478,0.004048094153404236,-0.005840908735990524,-0.004764570388942957,-0.0011096063535660505,-0.01651327684521675,0.004218435846269131,0.0076619721949100494,0.016768736764788628,-0.010754378512501717,-0.007011130917817354,-0.0018741177627816796,0.004677861928939819,-0.0013004607753828168,0.02279837615787983,0.015664083883166313,-0.003047492355108261,-0.006805235054343939,-0.023204054683446884,0.011979939416050911,-0.01936367340385914,0.020488401874899864,0.0002779807255137712,0.01603945530951023,0.011033518239855766,-0.0034474434796720743,0.003860779106616974,0.0030094629619270563,-0.0025448587257415056,0.016781283542513847,0.0010827252408489585,-0.02335255965590477,0.000616254925262183,-0.0035649340134114027,0.0007393514970317483,-0.008183765225112438,0.0014471083413809538,0.0038755787536501884,0.007099337410181761,-0.012667966075241566,0.006208354607224464,-0.011235825717449188,-0.005788819864392281,-0.013990281149744987,-0.005277065094560385,-0.019661838188767433,-0.011538130231201649,0.011401553638279438,0.0067108855582773685,0.001396434847265482,0.0769028514623642,-0.0029904483817517757,0.002209946746006608,0.009979894384741783,-0.0010606379946693778,-0.016086678951978683,0.007984510622918606,0.018508948385715485,0.0032983184792101383,-0.004930043593049049,0.013569834642112255,1.877335125755053e-05,0.0041457414627075195,-0.0065275197848677635,0.01902691088616848,0.0049742781557142735,-0.008188189007341862,-0.004906102083623409,-0.0191107876598835,0.016605230048298836,-0.017471250146627426,0.010408093221485615,-0.008595138788223267,0.00039457817911170423,0.0075583732686936855,0.01484600454568863,0.011490130797028542,0.0035124020650982857,-0.006972779054194689,0.0128085408359766,0.006472124718129635,-0.011789342388510704,0.006717384327203035,-0.0022378091234713793,0.00325773935765028,0.0053901877254247665,0.008246632292866707,0.0030436997767537832,0.0072782342322170734,0.0012802877463400364,-0.00802643597126007,0.004147414583712816,0.008670682087540627,0.004049904178828001,0.0038673868402838707,0.014705437235534191,0.0026979250833392143,0.001775945769622922,-0.01869085803627968,0.0037806022446602583,0.012721864506602287,0.015738211572170258,-0.008133381605148315,-0.007445990107953548,-0.006062779109925032,0.005171599797904491,-0.007623749785125256,-0.001971603836864233,-0.03202363848686218,0.0014124091248959303,0.00964097585529089,-0.0062558529898524284,0.12542743980884552,-0.023395422846078873,-0.02142343297600746,0.00010404972999822348,0.0040498957969248295,0.009305443614721298,-0.005175766069442034,-0.006316371727734804,0.01862599514424801,0.01787419244647026,0.03209351748228073,-0.013965249061584473,-0.01298594195395708,0.003942033741623163,0.007697572000324726,-0.0037004253827035427,0.001353675965219736,0.004194419831037521,0.038188375532627106,-0.006305979564785957,0.008670156821608543,-0.011301315389573574,0.022354990243911743,0.011309697292745113,-0.006025111768394709,-0.02238098718225956,-0.014605054631829262,0.009788730181753635,-0.02146783284842968,-0.026633543893694878,0.008195299655199051,5.627179052680731e-05,-0.006054638884961605,0.018990008160471916,0.0018300878582522273,-0.006439500488340855,0.0015690467553213239,-0.004935315810143948,-0.005042776465415955,-0.008323850110173225,0.01732305809855461,0.004760194569826126,0.009951967746019363,0.002688618842512369,-0.02490813285112381,0.013938416726887226,-0.008612480014562607,0.017687037587165833,0.0007003569626249373,0.003144141985103488,0.00028641021344810724,0.006280304864048958,0.01704099029302597,-0.031904399394989014,-0.01954682171344757,0.006692659109830856,-0.0029927969444543123,-0.019856123253703117,0.01037242915481329,0.007297733798623085,-0.00034432284883223474,9.271252201870084e-05,3.400759305804968e-05,-0.008098633028566837,-0.017516130581498146,0.0009811046766117215,-0.007083006668835878,-0.013434672728180885,0.006502609234303236,0.00046227165148593485,-0.006619544234126806,-0.011502401903271675,-0.01764489896595478,-0.018358498811721802,-0.016132373362779617,0.01945388875901699,-0.004716904833912849,0.016170112416148186,0.002639401238411665,-0.008305462077260017,-0.030113548040390015,0.014484983868896961,0.049616213887929916,0.0026693870313465595,0.015345823019742966,0.0026869860012084246,0.019824400544166565,0.00838514044880867,0.0023412152659147978,-0.0035702185705304146,-0.007228761445730925,0.009889356791973114,-0.01150357536971569,0.006204118020832539,-0.007316265255212784,0.005138332024216652,-0.004389585927128792,-0.006546832155436277,-0.004268612712621689,0.022032320499420166,-0.014779822900891304,0.011949374340474606,0.0014258417068049312,0.0048449402675032616,0.02138534002006054,-0.0369078628718853,-0.0007908937404863536,-0.009307898581027985,0.009610539302229881,0.010517065413296223,-0.005397812929004431,-0.0021158468443900347,-0.003497409401461482,-0.0037914770655333996,-0.019967637956142426,0.002439747331663966,-0.020455583930015564,-0.006008759140968323,-0.008751148357987404,-0.018866462633013725,0.008806422352790833,-0.0035796293523162603,-0.003078668611124158,-0.004720652941614389,-0.010492903180420399],\"index\":0}],\"model\":\"vicuna-7b-v1.5\",\"usage\":{\"prompt_tokens\":13,\"total_tokens\":13}}" - ] - } - ], - "source": [ - "!curl http://127.0.0.1:8000/v1/embeddings \\\n", - " -H \"Content-Type: application/json\" \\\n", - " -d '{ \\\n", - " \"model\": \"vicuna-7b-v1.5\", \\\n", - " \"input\": \"Hello, can you tell me a joke for me?\" \\\n", - " }'" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "-U2SZWTghxzc" - }, - "source": [ - "Try text completion with" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "85T5NO7Wh03R", - "outputId": "1a2c9568-2aa3-4a89-ecd8-8af496be1a41" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\"id\":\"cmpl-kB3gg4KtgcGdif9V4eNbh6\",\"object\":\"text_completion\",\"created\":1705782008,\"model\":\"vicuna-7b-v1.5\",\"choices\":[{\"index\":0,\"text\":\", there was a little girl named Alice. Alice lived in a small village nestled in a valley\",\"logprobs\":null,\"finish_reason\":\"length\"}],\"usage\":{\"prompt_tokens\":5,\"total_tokens\":24,\"completion_tokens\":19}}" - ] - } - ], - "source": [ - "!curl http://127.0.0.1:8000/v1/completions \\\n", - " -H \"Content-Type: application/json\" \\\n", - " -d '{ \\\n", - " \"model\": \"vicuna-7b-v1.5\", \\\n", - " \"prompt\": \"Once upon a time\", \\\n", - " \"max_tokens\": 20, \\\n", - " \"temperature\": 0.5 \\\n", - " }'" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "EDxLbQDKVLiQ" - }, - "source": [ - "Try create_embeddings to analyze the prompts!" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "bbrFoxgaplhK", - "outputId": "48e23158-1468-445d-a4cd-b5bd67bd3bde" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Cosine similarity between 'The quick brown fox' and 'The quick brown dog': 0.90\n", - "Cosine similarity between 'The quick brown fox' and 'The fast brown fox': 0.86\n", - "Cosine similarity between 'The quick brown fox' and 'A completely different sentence': 0.58\n", - "Cosine similarity between 'The quick brown dog' and 'The fast brown fox': 0.84\n", - "Cosine similarity between 'The quick brown dog' and 'A completely different sentence': 0.66\n", - "Cosine similarity between 'The fast brown fox' and 'A completely different sentence': 0.62\n" - ] - } - ], - "source": [ - "import json\n", - "import numpy as np\n", - "import requests\n", - "from scipy.spatial.distance import cosine\n", - "\n", - "\n", - "def get_embedding_from_api(word, model=\"vicuna-7b-v1.5\"):\n", - " url = \"http://127.0.0.1:8000/v1/embeddings\"\n", - " headers = {\"Content-Type\": \"application/json\"}\n", - " data = json.dumps({\"model\": model, \"input\": word})\n", - "\n", - " response = requests.post(url, headers=headers, data=data)\n", - " if response.status_code == 200:\n", - " embedding = np.array(response.json()[\"data\"][0][\"embedding\"])\n", - " return embedding\n", - " else:\n", - " print(f\"Error: {response.status_code} - {response.text}\")\n", - " return None\n", - "\n", - "\n", - "def cosine_similarity(vec1, vec2):\n", - " return 1 - cosine(vec1, vec2)\n", - "\n", - "\n", - "def print_cosine_similarity(embeddings, texts):\n", - " for i in range(len(texts)):\n", - " for j in range(i + 1, len(texts)):\n", - " sim = cosine_similarity(embeddings[texts[i]], embeddings[texts[j]])\n", - " print(f\"Cosine similarity between '{texts[i]}' and '{texts[j]}': {sim:.2f}\")\n", - "\n", - "\n", - "texts = [\n", - " \"The quick brown fox\",\n", - " \"The quick brown dog\",\n", - " \"The fast brown fox\",\n", - " \"A completely different sentence\",\n", - "]\n", - "\n", - "embeddings = {}\n", - "for text in texts:\n", - " embeddings[text] = get_embedding_from_api(text)\n", - "\n", - "print_cosine_similarity(embeddings, texts)" - ] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "gpuType": "T4", - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} From bb248008516254c4ccdb1c4bbef9120cb9a74e82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cyixin-huang1=E2=80=9D?= Date: Sat, 15 Feb 2025 22:54:44 -0800 Subject: [PATCH 32/40] revert the spacing changes --- fastchat/data/clean_sharegpt.py | 1 - fastchat/data/extract_gpt4_only.py | 1 - fastchat/data/extract_single_round.py | 1 - fastchat/data/filter_wrong_format.py | 1 - fastchat/data/hardcoded_questions.py | 1 - fastchat/data/inspect_data.py | 1 - fastchat/data/optional_clean.py | 1 - fastchat/data/optional_replace.py | 1 - fastchat/data/sample.py | 1 - fastchat/data/split_long_conversation.py | 1 - fastchat/data/split_train_test.py | 1 - fastchat/llm_judge/clean_judgment.py | 1 - fastchat/llm_judge/compute_agreement.py | 1 - .../download_mt_bench_pregenerated.py | 1 - fastchat/llm_judge/gen_api_answer.py | 1 - fastchat/llm_judge/gen_judgment.py | 1 - fastchat/llm_judge/gen_model_answer.py | 1 - fastchat/llm_judge/show_result.py | 1 - fastchat/model/apply_delta.py | 1 - fastchat/model/apply_lora.py | 1 - fastchat/model/convert_fp16.py | 1 - fastchat/model/make_delta.py | 1 - fastchat/model/model_chatglm.py | 1 - fastchat/model/model_registry.py | 1 - fastchat/model/monkey_patch_non_inplace.py | 1 - fastchat/model/upload_hub.py | 1 - fastchat/serve/cli.py | 1 - fastchat/serve/controller.py | 1 - fastchat/serve/gradio_block_arena_vision.py | 22 ++++++++----------- .../serve/gradio_block_arena_vision_named.py | 8 +++---- fastchat/serve/huggingface_api.py | 1 - fastchat/serve/huggingface_api_worker.py | 1 - fastchat/serve/inference.py | 7 +++--- fastchat/serve/launch_all_serve.py | 7 +++--- fastchat/serve/model_worker.py | 1 - fastchat/serve/monitor/clean_battle_data.py | 1 - fastchat/serve/monitor/clean_chat_data.py | 1 - .../arena_33k/filter_bad_conv.py | 1 - .../arena_33k/upload_hf_dataset.py | 1 - .../lmsys_chat_1m/compute_stats.py | 1 - .../lmsys_chat_1m/filter_bad_conv.py | 1 - .../lmsys_chat_1m/upload_hf_dataset.py | 1 - fastchat/serve/monitor/elo_analysis.py | 14 +++++------- .../serve/monitor/leaderboard_csv_to_html.py | 1 - fastchat/serve/monitor/summarize_cluster.py | 1 - .../serve/monitor/tag_openai_moderation.py | 1 - fastchat/serve/monitor/topic_clustering.py | 1 - fastchat/serve/multi_model_worker.py | 1 - fastchat/serve/openai_api_server.py | 1 - fastchat/serve/shutdown_serve.py | 2 +- fastchat/serve/test_message.py | 1 - fastchat/serve/test_throughput.py | 1 - fastchat/serve/vllm_worker.py | 8 +++---- fastchat/train/train_lora.py | 18 +++++++-------- fastchat/train/train_lora_t5.py | 18 +++++++-------- fastchat/utils.py | 9 +++----- .../benchmark/benchmark_api_provider.py | 1 - tests/launch_openai_api_test_server.py | 1 - tests/test_cli.py | 1 - tests/test_openai_api.py | 1 - 60 files changed, 47 insertions(+), 116 deletions(-) diff --git a/fastchat/data/clean_sharegpt.py b/fastchat/data/clean_sharegpt.py index c14127af6..af6ffb695 100644 --- a/fastchat/data/clean_sharegpt.py +++ b/fastchat/data/clean_sharegpt.py @@ -5,7 +5,6 @@ Usage: python3 -m fastchat.data.clean_sharegpt --in sharegpt_html.json --out sharegpt_clean.json """ - import argparse from concurrent.futures import ProcessPoolExecutor import json diff --git a/fastchat/data/extract_gpt4_only.py b/fastchat/data/extract_gpt4_only.py index 9bf185a20..bab53bcc7 100644 --- a/fastchat/data/extract_gpt4_only.py +++ b/fastchat/data/extract_gpt4_only.py @@ -3,7 +3,6 @@ Usage: python3 -m fastchat.data.extract_gpt4_only --in sharegpt.json """ - import argparse import json diff --git a/fastchat/data/extract_single_round.py b/fastchat/data/extract_single_round.py index a7c93ac7f..5da803656 100644 --- a/fastchat/data/extract_single_round.py +++ b/fastchat/data/extract_single_round.py @@ -3,7 +3,6 @@ Usage: python3 -m fastchat.data.extract_single_round --in sharegpt.json """ - import argparse import json diff --git a/fastchat/data/filter_wrong_format.py b/fastchat/data/filter_wrong_format.py index 90df80b94..46588ba84 100644 --- a/fastchat/data/filter_wrong_format.py +++ b/fastchat/data/filter_wrong_format.py @@ -5,7 +5,6 @@ python3 -m fastchat.data.filter_wrong_format --in input.json --out output.json """ - import argparse import json import re diff --git a/fastchat/data/hardcoded_questions.py b/fastchat/data/hardcoded_questions.py index c6a89a45b..a2bcff42e 100644 --- a/fastchat/data/hardcoded_questions.py +++ b/fastchat/data/hardcoded_questions.py @@ -1,7 +1,6 @@ """ Hardcoded question and answers. """ - import json diff --git a/fastchat/data/inspect_data.py b/fastchat/data/inspect_data.py index dd5f33789..df9227106 100644 --- a/fastchat/data/inspect_data.py +++ b/fastchat/data/inspect_data.py @@ -2,7 +2,6 @@ Usage: python3 -m fastchat.data.inspect_data --in sharegpt_20230322_clean_lang_split.json """ - import argparse import json import random diff --git a/fastchat/data/optional_clean.py b/fastchat/data/optional_clean.py index 81469f4c9..47aecc111 100644 --- a/fastchat/data/optional_clean.py +++ b/fastchat/data/optional_clean.py @@ -8,7 +8,6 @@ Requirement: pip3 install polyglot pyicu pycld2 """ - import argparse import json import re diff --git a/fastchat/data/optional_replace.py b/fastchat/data/optional_replace.py index 4640f0d6e..1114151a9 100644 --- a/fastchat/data/optional_replace.py +++ b/fastchat/data/optional_replace.py @@ -7,7 +7,6 @@ Requirement: pip3 install transformers tqdm """ - import argparse import json import traceback diff --git a/fastchat/data/sample.py b/fastchat/data/sample.py index 7a8501a9b..5ea94fada 100644 --- a/fastchat/data/sample.py +++ b/fastchat/data/sample.py @@ -3,7 +3,6 @@ Usage: python3 -m fastchat.data.sample --in sharegpt.json --out sampled.json """ - import argparse import json diff --git a/fastchat/data/split_long_conversation.py b/fastchat/data/split_long_conversation.py index 9a4c04f95..413fa8bce 100644 --- a/fastchat/data/split_long_conversation.py +++ b/fastchat/data/split_long_conversation.py @@ -6,7 +6,6 @@ --out sharegpt_split.json \ --model-name-or-path $ """ - import argparse from concurrent.futures import ProcessPoolExecutor import json diff --git a/fastchat/data/split_train_test.py b/fastchat/data/split_train_test.py index eafbc5e11..60b8960b5 100644 --- a/fastchat/data/split_train_test.py +++ b/fastchat/data/split_train_test.py @@ -3,7 +3,6 @@ Usage: python3 -m fastchat.data.split_train_test --in sharegpt.json """ - import argparse import json diff --git a/fastchat/llm_judge/clean_judgment.py b/fastchat/llm_judge/clean_judgment.py index 338fe9fbf..d139ed7f2 100644 --- a/fastchat/llm_judge/clean_judgment.py +++ b/fastchat/llm_judge/clean_judgment.py @@ -1,7 +1,6 @@ """ Clean model judgment files. """ - import argparse import json diff --git a/fastchat/llm_judge/compute_agreement.py b/fastchat/llm_judge/compute_agreement.py index d6b252b10..1b940bf5a 100644 --- a/fastchat/llm_judge/compute_agreement.py +++ b/fastchat/llm_judge/compute_agreement.py @@ -5,7 +5,6 @@ python compute_agreement.py --judges gpt4-pair human --votefiles human_judgments.json gpt4_pair_judgments.json python compute_agreement.py --judges human human --votefiles human_judgments.json """ - import argparse import json import os diff --git a/fastchat/llm_judge/download_mt_bench_pregenerated.py b/fastchat/llm_judge/download_mt_bench_pregenerated.py index 84232ff6a..9546311f5 100644 --- a/fastchat/llm_judge/download_mt_bench_pregenerated.py +++ b/fastchat/llm_judge/download_mt_bench_pregenerated.py @@ -1,7 +1,6 @@ """ Download the pre-generated model answers and judgments for MT-bench. """ - import os from fastchat.utils import run_cmd diff --git a/fastchat/llm_judge/gen_api_answer.py b/fastchat/llm_judge/gen_api_answer.py index adb21680d..8f9c62624 100644 --- a/fastchat/llm_judge/gen_api_answer.py +++ b/fastchat/llm_judge/gen_api_answer.py @@ -3,7 +3,6 @@ Usage: python3 gen_api_answer.py --model gpt-3.5-turbo """ - import argparse import json import os diff --git a/fastchat/llm_judge/gen_judgment.py b/fastchat/llm_judge/gen_judgment.py index 1cd2dcd49..a1c70b295 100644 --- a/fastchat/llm_judge/gen_judgment.py +++ b/fastchat/llm_judge/gen_judgment.py @@ -2,7 +2,6 @@ Usage: python gen_judgment.py --model-list [LIST-OF-MODEL-ID] --parallel [num-concurrent-api-call] --mode [single|pairwise-baseline|pairwise-all] """ - import argparse from concurrent.futures import ThreadPoolExecutor import json diff --git a/fastchat/llm_judge/gen_model_answer.py b/fastchat/llm_judge/gen_model_answer.py index 4294f9d97..44166fb08 100644 --- a/fastchat/llm_judge/gen_model_answer.py +++ b/fastchat/llm_judge/gen_model_answer.py @@ -3,7 +3,6 @@ Usage: python3 gen_model_answer.py --model-path lmsys/fastchat-t5-3b-v1.0 --model-id fastchat-t5-3b-v1.0 """ - import argparse import json import os diff --git a/fastchat/llm_judge/show_result.py b/fastchat/llm_judge/show_result.py index 8a4e9232c..f20801b11 100644 --- a/fastchat/llm_judge/show_result.py +++ b/fastchat/llm_judge/show_result.py @@ -2,7 +2,6 @@ Usage: python3 show_result.py --mode [single|pairwise-baseline|pairwise-all] """ - import argparse import pandas as pd diff --git a/fastchat/model/apply_delta.py b/fastchat/model/apply_delta.py index 776853f00..a42ada71d 100644 --- a/fastchat/model/apply_delta.py +++ b/fastchat/model/apply_delta.py @@ -4,7 +4,6 @@ Usage: python3 -m fastchat.model.apply_delta --base ~/model_weights/llama-7b --target ~/model_weights/vicuna-7b --delta lmsys/vicuna-7b-delta-v1.1 """ - import argparse import gc import glob diff --git a/fastchat/model/apply_lora.py b/fastchat/model/apply_lora.py index 0bea2449e..01263dcc7 100644 --- a/fastchat/model/apply_lora.py +++ b/fastchat/model/apply_lora.py @@ -7,7 +7,6 @@ Dependency: pip3 install git+https://github.com/huggingface/peft.git@2822398fbe896f25d4dac5e468624dc5fd65a51b """ - import argparse import torch diff --git a/fastchat/model/convert_fp16.py b/fastchat/model/convert_fp16.py index ec8e3af92..efc40aa83 100644 --- a/fastchat/model/convert_fp16.py +++ b/fastchat/model/convert_fp16.py @@ -2,7 +2,6 @@ Usage: python3 -m fastchat.model.convert_fp16 --in in-folder --out out-folder """ - import argparse from transformers import AutoTokenizer, AutoModelForCausalLM diff --git a/fastchat/model/make_delta.py b/fastchat/model/make_delta.py index 88a588b30..480ba8f1a 100644 --- a/fastchat/model/make_delta.py +++ b/fastchat/model/make_delta.py @@ -4,7 +4,6 @@ Usage: python3 -m fastchat.model.make_delta --base ~/model_weights/llama-13b --target ~/model_weights/vicuna-13b --delta ~/model_weights/vicuna-13b-delta --hub-repo-id lmsys/vicuna-13b-delta-v1.1 """ - import argparse import torch diff --git a/fastchat/model/model_chatglm.py b/fastchat/model/model_chatglm.py index bb5c5a5a8..2cbac8bc5 100644 --- a/fastchat/model/model_chatglm.py +++ b/fastchat/model/model_chatglm.py @@ -2,7 +2,6 @@ Inference code for ChatGLM. Adapted from https://huggingface.co/THUDM/chatglm-6b/blob/main/modeling_chatglm.py. """ - import re import torch diff --git a/fastchat/model/model_registry.py b/fastchat/model/model_registry.py index 74a2cc9df..2eed9649e 100644 --- a/fastchat/model/model_registry.py +++ b/fastchat/model/model_registry.py @@ -1,5 +1,4 @@ """Additional information of the models.""" - from collections import namedtuple, OrderedDict from typing import List diff --git a/fastchat/model/monkey_patch_non_inplace.py b/fastchat/model/monkey_patch_non_inplace.py index 0a8dca04c..413dd3b30 100644 --- a/fastchat/model/monkey_patch_non_inplace.py +++ b/fastchat/model/monkey_patch_non_inplace.py @@ -2,7 +2,6 @@ Monkey patch the llama implementation in the huggingface/transformers library. Avoid bugs in mps backend by not using in-place operations. """ - import math from typing import List, Optional, Tuple diff --git a/fastchat/model/upload_hub.py b/fastchat/model/upload_hub.py index d2e1a9616..b1519652e 100644 --- a/fastchat/model/upload_hub.py +++ b/fastchat/model/upload_hub.py @@ -4,7 +4,6 @@ Usage: python3 -m fastchat.model.upload_hub --model-path ~/model_weights/vicuna-13b --hub-repo-id lmsys/vicuna-13b-v1.3 """ - import argparse import tempfile diff --git a/fastchat/serve/cli.py b/fastchat/serve/cli.py index 1863f8bc5..78f7f51b1 100644 --- a/fastchat/serve/cli.py +++ b/fastchat/serve/cli.py @@ -13,7 +13,6 @@ - Type "!!save " to save the conversation history to a json file. - Type "!!load " to load a conversation history from a json file. """ - import argparse import os import re diff --git a/fastchat/serve/controller.py b/fastchat/serve/controller.py index 8fefd4ec2..42d928403 100644 --- a/fastchat/serve/controller.py +++ b/fastchat/serve/controller.py @@ -2,7 +2,6 @@ A controller manages distributed workers. It sends worker addresses to clients. """ - import argparse import asyncio import dataclasses diff --git a/fastchat/serve/gradio_block_arena_vision.py b/fastchat/serve/gradio_block_arena_vision.py index ee7964c15..d253050a8 100644 --- a/fastchat/serve/gradio_block_arena_vision.py +++ b/fastchat/serve/gradio_block_arena_vision.py @@ -211,7 +211,6 @@ def wrap_pdfchat_query(query, document): "languages": ",".join(PDFPARSE_SUPPORTED_LANGS.values()), } - def convert_base64_to_pil_image(b64_string): from PIL import Image import numpy as np @@ -219,33 +218,30 @@ def convert_base64_to_pil_image(b64_string): image_data = np.frombuffer(base64.b64decode(b64_string), dtype=np.uint8) image_bytes = BytesIO(image_data) image = Image.open(image_bytes) - + return image - def batch_convert_base64_to_images(base64_dict): import concurrent.futures - with concurrent.futures.ThreadPoolExecutor() as executor: return list(executor.map(convert_base64_to_pil_image, base64_dict.values())) - def parse_pdf(file_path): import requests url = "https://www.datalab.to/api/v1/marker" form_data = { - "file": ("test.pdf", open(file_path, "rb"), "application/pdf"), + 'file': ('test.pdf', open(file_path, 'rb'), 'application/pdf'), "force_ocr": (None, False), "paginate": (None, False), - "output_format": (None, "markdown"), + 'output_format': (None, 'markdown'), "use_llm": (None, True), "strip_existing_ocr": (None, False), - "disable_image_extraction": (None, False), + "disable_image_extraction": (None, False) } - headers = {"X-Api-Key": str(os.getenv("MARKER_API_KEY"))} + headers = {"X-Api-Key": str(os.getenv("MARKER_API_KEY"))} response = requests.post(url, files=form_data, headers=headers) data = response.json() @@ -259,7 +255,7 @@ def parse_pdf(file_path): if data["status"] == "complete": break - + output_md = data["markdown"] output_images = batch_convert_base64_to_images(data["images"]) @@ -454,9 +450,9 @@ def build_single_vision_language_model_ui( with gr.Row(elem_id="model_selector_row"): model_selector = gr.Dropdown( choices=text_and_vision_models, - value=( - text_and_vision_models[0] if len(text_and_vision_models) > 0 else "" - ), + value=text_and_vision_models[0] + if len(text_and_vision_models) > 0 + else "", interactive=True, show_label=False, container=False, diff --git a/fastchat/serve/gradio_block_arena_vision_named.py b/fastchat/serve/gradio_block_arena_vision_named.py index 80b9ea056..7c653acf3 100644 --- a/fastchat/serve/gradio_block_arena_vision_named.py +++ b/fastchat/serve/gradio_block_arena_vision_named.py @@ -355,11 +355,9 @@ def build_side_by_side_vision_ui_named(context: Context, random_questions=None): with gr.Column(): model_selectors[i] = gr.Dropdown( choices=text_and_vision_models, - value=( - text_and_vision_models[i] - if len(text_and_vision_models) > i - else "" - ), + value=text_and_vision_models[i] + if len(text_and_vision_models) > i + else "", interactive=True, show_label=False, container=False, diff --git a/fastchat/serve/huggingface_api.py b/fastchat/serve/huggingface_api.py index a7bd11c53..8022fbc93 100644 --- a/fastchat/serve/huggingface_api.py +++ b/fastchat/serve/huggingface_api.py @@ -5,7 +5,6 @@ python3 -m fastchat.serve.huggingface_api --model lmsys/vicuna-7b-v1.5 python3 -m fastchat.serve.huggingface_api --model lmsys/fastchat-t5-3b-v1.0 """ - import argparse import torch diff --git a/fastchat/serve/huggingface_api_worker.py b/fastchat/serve/huggingface_api_worker.py index 9ce076ab7..6ed8e6c8c 100644 --- a/fastchat/serve/huggingface_api_worker.py +++ b/fastchat/serve/huggingface_api_worker.py @@ -21,7 +21,6 @@ "model_path", "api_base", "token", and "context_length" are necessary, while others are optional. """ - import argparse import asyncio import json diff --git a/fastchat/serve/inference.py b/fastchat/serve/inference.py index a60207b2f..6d155aab7 100644 --- a/fastchat/serve/inference.py +++ b/fastchat/serve/inference.py @@ -1,5 +1,4 @@ """Inference for FastChat models.""" - import abc import gc import json @@ -228,9 +227,9 @@ def generate_stream( output_ids if echo else output_ids[input_echo_len:] ) ], - "token_logprobs": ( - token_logprobs if echo else token_logprobs[input_echo_len:] - ), + "token_logprobs": token_logprobs + if echo + else token_logprobs[input_echo_len:], "top_logprobs": [{}] * len(token_logprobs if echo else token_logprobs[input_echo_len:]), } diff --git a/fastchat/serve/launch_all_serve.py b/fastchat/serve/launch_all_serve.py index 1842a7c8e..2f4ad7b0b 100644 --- a/fastchat/serve/launch_all_serve.py +++ b/fastchat/serve/launch_all_serve.py @@ -1,14 +1,13 @@ """ -Usage: python launch_all_serve_by_shell.py --model-path-address "THUDM/chatglm2-6b@localhost@2021" "huggyllama/llama-7b@localhost@2022" +Usage: python launch_all_serve_by_shell.py --model-path-address "THUDM/chatglm2-6b@localhost@2021" "huggyllama/llama-7b@localhost@2022" -Workers are listed in format of `model-path`@`host`@`port` +Workers are listed in format of `model-path`@`host`@`port` -The key mechanism behind this scripts is: +The key mechanism behind this scripts is: 1, execute shell cmd to launch the controller/worker/openai-api-server; 2, check the log of controller/worker/openai-api-server to ensure that the serve is launched properly. Note that a few of non-critical `fastchat.serve` cmd options are not supported currently. """ - import sys import os diff --git a/fastchat/serve/model_worker.py b/fastchat/serve/model_worker.py index 464aefca1..683a78556 100644 --- a/fastchat/serve/model_worker.py +++ b/fastchat/serve/model_worker.py @@ -1,7 +1,6 @@ """ A model worker that executes the model. """ - import argparse import base64 import gc diff --git a/fastchat/serve/monitor/clean_battle_data.py b/fastchat/serve/monitor/clean_battle_data.py index 7134aab29..270f981cc 100644 --- a/fastchat/serve/monitor/clean_battle_data.py +++ b/fastchat/serve/monitor/clean_battle_data.py @@ -4,7 +4,6 @@ Usage: python3 clean_battle_data.py --mode conv_release """ - import argparse import datetime import json diff --git a/fastchat/serve/monitor/clean_chat_data.py b/fastchat/serve/monitor/clean_chat_data.py index 569f79322..ec6da4a65 100644 --- a/fastchat/serve/monitor/clean_chat_data.py +++ b/fastchat/serve/monitor/clean_chat_data.py @@ -4,7 +4,6 @@ Usage: python3 clean_chat_data.py """ - import argparse import json import os diff --git a/fastchat/serve/monitor/dataset_release_scripts/arena_33k/filter_bad_conv.py b/fastchat/serve/monitor/dataset_release_scripts/arena_33k/filter_bad_conv.py index 909dc9e85..6d12d7c65 100644 --- a/fastchat/serve/monitor/dataset_release_scripts/arena_33k/filter_bad_conv.py +++ b/fastchat/serve/monitor/dataset_release_scripts/arena_33k/filter_bad_conv.py @@ -3,7 +3,6 @@ Usage: python3 filter_bad_conv.py --in clean_battle_conv_20230630_tagged_v1_pii.json """ - import argparse from collections import defaultdict from enum import Enum, auto diff --git a/fastchat/serve/monitor/dataset_release_scripts/arena_33k/upload_hf_dataset.py b/fastchat/serve/monitor/dataset_release_scripts/arena_33k/upload_hf_dataset.py index 0c5b8d48f..e37aadcea 100644 --- a/fastchat/serve/monitor/dataset_release_scripts/arena_33k/upload_hf_dataset.py +++ b/fastchat/serve/monitor/dataset_release_scripts/arena_33k/upload_hf_dataset.py @@ -1,7 +1,6 @@ """ Upload to huggingface. """ - import json from datasets import Dataset, DatasetDict, load_dataset diff --git a/fastchat/serve/monitor/dataset_release_scripts/lmsys_chat_1m/compute_stats.py b/fastchat/serve/monitor/dataset_release_scripts/lmsys_chat_1m/compute_stats.py index 1f4e85391..97abaaa0d 100644 --- a/fastchat/serve/monitor/dataset_release_scripts/lmsys_chat_1m/compute_stats.py +++ b/fastchat/serve/monitor/dataset_release_scripts/lmsys_chat_1m/compute_stats.py @@ -2,7 +2,6 @@ From colab: https://colab.research.google.com/drive/1oMdw_Lqgmd6DletSOLHsyD-Rc96cRShs?usp=sharing """ - import argparse import datetime import json diff --git a/fastchat/serve/monitor/dataset_release_scripts/lmsys_chat_1m/filter_bad_conv.py b/fastchat/serve/monitor/dataset_release_scripts/lmsys_chat_1m/filter_bad_conv.py index 82af3459c..3ccde1ca5 100644 --- a/fastchat/serve/monitor/dataset_release_scripts/lmsys_chat_1m/filter_bad_conv.py +++ b/fastchat/serve/monitor/dataset_release_scripts/lmsys_chat_1m/filter_bad_conv.py @@ -7,7 +7,6 @@ Usage: python3 filter_bad_conv_lmsys_chat_1m.py --in clean_battle_conv_20230630_tagged_v1_pii.json """ - import argparse from concurrent.futures import ProcessPoolExecutor from collections import defaultdict diff --git a/fastchat/serve/monitor/dataset_release_scripts/lmsys_chat_1m/upload_hf_dataset.py b/fastchat/serve/monitor/dataset_release_scripts/lmsys_chat_1m/upload_hf_dataset.py index f4aa3dc87..41d0fbdb5 100644 --- a/fastchat/serve/monitor/dataset_release_scripts/lmsys_chat_1m/upload_hf_dataset.py +++ b/fastchat/serve/monitor/dataset_release_scripts/lmsys_chat_1m/upload_hf_dataset.py @@ -1,7 +1,6 @@ """ Upload to huggingface. """ - import argparse import json from datasets import Dataset, DatasetDict, load_dataset diff --git a/fastchat/serve/monitor/elo_analysis.py b/fastchat/serve/monitor/elo_analysis.py index 82712578d..eac5368d3 100644 --- a/fastchat/serve/monitor/elo_analysis.py +++ b/fastchat/serve/monitor/elo_analysis.py @@ -455,14 +455,12 @@ def report_elo_analysis_results( "last_updated_tstamp": last_updated_tstamp, "bootstrap_df": bootstrap_df, "leaderboard_table_df": leaderboard_table_df, - "style_coefficients": ( - { - "bootstrap": np.vstack(boostrap_coef), - "final": coef_final, - } - if rating_system == "bt" and style_control - else {} - ), + "style_coefficients": { + "bootstrap": np.vstack(boostrap_coef), + "final": coef_final, + } + if rating_system == "bt" and style_control + else {}, } diff --git a/fastchat/serve/monitor/leaderboard_csv_to_html.py b/fastchat/serve/monitor/leaderboard_csv_to_html.py index 7b357ce80..ad52e7b2b 100644 --- a/fastchat/serve/monitor/leaderboard_csv_to_html.py +++ b/fastchat/serve/monitor/leaderboard_csv_to_html.py @@ -4,7 +4,6 @@ Usage: python3 leaderboard_csv_to_html.py --in leaderboard_table_20230619.csv """ - import argparse import numpy as np diff --git a/fastchat/serve/monitor/summarize_cluster.py b/fastchat/serve/monitor/summarize_cluster.py index 9d9f87c0b..b461a68b2 100644 --- a/fastchat/serve/monitor/summarize_cluster.py +++ b/fastchat/serve/monitor/summarize_cluster.py @@ -3,7 +3,6 @@ python3 summarize_cluster.py --in results_c20_kmeans_cluster.pkl --model gpt-4 --num-prompts 100 python3 summarize_cluster.py --in results_c20_kmeans_cluster.pkl --model azure-gpt-4-32k --num-prompts 200 """ - import argparse import pickle diff --git a/fastchat/serve/monitor/tag_openai_moderation.py b/fastchat/serve/monitor/tag_openai_moderation.py index 44f9640c2..b80703388 100644 --- a/fastchat/serve/monitor/tag_openai_moderation.py +++ b/fastchat/serve/monitor/tag_openai_moderation.py @@ -1,7 +1,6 @@ """ Add OpenAI moderation API results to all conversations. """ - import argparse from concurrent.futures import ThreadPoolExecutor import json diff --git a/fastchat/serve/monitor/topic_clustering.py b/fastchat/serve/monitor/topic_clustering.py index f5431854f..3d58e56bf 100644 --- a/fastchat/serve/monitor/topic_clustering.py +++ b/fastchat/serve/monitor/topic_clustering.py @@ -4,7 +4,6 @@ python3 topic_clustering.py --in arena.json --english-only --min-length 32 python3 topic_clustering.py --in clean_conv_20230809_100k.json --english-only --min-length 32 --max-length 1536 """ - import argparse import json import pickle diff --git a/fastchat/serve/multi_model_worker.py b/fastchat/serve/multi_model_worker.py index b4f8e0310..5e6266fe0 100644 --- a/fastchat/serve/multi_model_worker.py +++ b/fastchat/serve/multi_model_worker.py @@ -11,7 +11,6 @@ We recommend using this with multiple Peft models (with `peft` in the name) where all Peft models are trained on the exact same base model. """ - import argparse import asyncio import dataclasses diff --git a/fastchat/serve/openai_api_server.py b/fastchat/serve/openai_api_server.py index 5b5374d92..a6ffee96b 100644 --- a/fastchat/serve/openai_api_server.py +++ b/fastchat/serve/openai_api_server.py @@ -7,7 +7,6 @@ Usage: python3 -m fastchat.serve.openai_api_server """ - import asyncio import argparse import json diff --git a/fastchat/serve/shutdown_serve.py b/fastchat/serve/shutdown_serve.py index f8d37b8a4..95e2b704f 100644 --- a/fastchat/serve/shutdown_serve.py +++ b/fastchat/serve/shutdown_serve.py @@ -1,7 +1,7 @@ """ Usage： python shutdown_serve.py --down all -options: "all","controller","model_worker","openai_api_server"， `all` means to stop all related servers +options: "all","controller","model_worker","openai_api_server"， `all` means to stop all related servers """ import argparse diff --git a/fastchat/serve/test_message.py b/fastchat/serve/test_message.py index 1d969b65b..203a44901 100644 --- a/fastchat/serve/test_message.py +++ b/fastchat/serve/test_message.py @@ -1,5 +1,4 @@ """Send a test message.""" - import argparse import json diff --git a/fastchat/serve/test_throughput.py b/fastchat/serve/test_throughput.py index e9e4735c8..3796a6e2a 100644 --- a/fastchat/serve/test_throughput.py +++ b/fastchat/serve/test_throughput.py @@ -1,5 +1,4 @@ """Benchmarking script to test the throughput of serving workers.""" - import argparse import json diff --git a/fastchat/serve/vllm_worker.py b/fastchat/serve/vllm_worker.py index 44e029175..0af680bb5 100644 --- a/fastchat/serve/vllm_worker.py +++ b/fastchat/serve/vllm_worker.py @@ -156,11 +156,9 @@ async def generate_stream(self, params): "cumulative_logprob": [ output.cumulative_logprob for output in request_output.outputs ], - "finish_reason": ( - request_output.outputs[0].finish_reason - if len(request_output.outputs) == 1 - else [output.finish_reason for output in request_output.outputs] - ), + "finish_reason": request_output.outputs[0].finish_reason + if len(request_output.outputs) == 1 + else [output.finish_reason for output in request_output.outputs], } # Emit twice here to ensure a 'finish_reason' with empty content in the OpenAI API response. # This aligns with the behavior of model_worker. diff --git a/fastchat/train/train_lora.py b/fastchat/train/train_lora.py index cfb76d3a7..9ecb47c29 100644 --- a/fastchat/train/train_lora.py +++ b/fastchat/train/train_lora.py @@ -135,16 +135,14 @@ def train(): model_args.model_name_or_path, cache_dir=training_args.cache_dir, device_map=device_map, - quantization_config=( - BitsAndBytesConfig( - load_in_4bit=True, - bnb_4bit_use_double_quant=True, - bnb_4bit_quant_type="nf4", - bnb_4bit_compute_dtype=compute_dtype, - ) - if lora_args.q_lora - else None - ), + quantization_config=BitsAndBytesConfig( + load_in_4bit=True, + bnb_4bit_use_double_quant=True, + bnb_4bit_quant_type="nf4", + bnb_4bit_compute_dtype=compute_dtype, + ) + if lora_args.q_lora + else None, ) lora_config = LoraConfig( r=lora_args.lora_r, diff --git a/fastchat/train/train_lora_t5.py b/fastchat/train/train_lora_t5.py index 5342a7dfc..21abc92cb 100644 --- a/fastchat/train/train_lora_t5.py +++ b/fastchat/train/train_lora_t5.py @@ -137,16 +137,14 @@ def train(): model_args.model_name_or_path, cache_dir=training_args.cache_dir, device_map=device_map, - quantization_config=( - BitsAndBytesConfig( - load_in_4bit=True, - bnb_4bit_use_double_quant=True, - bnb_4bit_quant_type="nf4", - bnb_4bit_compute_dtype=compute_dtype, - ) - if lora_args.q_lora - else None - ), + quantization_config=BitsAndBytesConfig( + load_in_4bit=True, + bnb_4bit_use_double_quant=True, + bnb_4bit_quant_type="nf4", + bnb_4bit_compute_dtype=compute_dtype, + ) + if lora_args.q_lora + else None, ) lora_config = LoraConfig( diff --git a/fastchat/utils.py b/fastchat/utils.py index 9a4eccc54..60527099d 100644 --- a/fastchat/utils.py +++ b/fastchat/utils.py @@ -1,7 +1,6 @@ """ Common utilities. """ - from asyncio import AbstractEventLoop from io import BytesIO import base64 @@ -483,21 +482,19 @@ def get_pdf_num_page(file_path): return len(reader.pages) - def resize_image_if_needed(image_bytes): image = Image.open(BytesIO(image_bytes)) if min(image.size) < 128: print(f"⚠️ Resizing image from {image.size} to minimum required size") new_size = (max(128, image.size[0]), max(128, image.size[1])) image = image.resize(new_size, Image.LANCZOS) - + img_byte_arr = BytesIO() image.save(img_byte_arr, format="PNG") return img_byte_arr.getvalue() - + return image_bytes - def image_moderation_request(image_bytes, endpoint, api_key): headers = {"Content-Type": "image/jpeg", "Ocp-Apim-Subscription-Key": api_key} @@ -532,7 +529,7 @@ def image_moderation_filter(image): print(f"moderating image") image_bytes = base64.b64decode(image.base64_str) - image_bytes = resize_image_if_needed(image_bytes) + image_bytes = resize_image_if_needed(image_bytes) nsfw_flagged = image_moderation_provider(image_bytes, "nsfw") csam_flagged = False diff --git a/playground/benchmark/benchmark_api_provider.py b/playground/benchmark/benchmark_api_provider.py index d66207ef1..89ca02ece 100644 --- a/playground/benchmark/benchmark_api_provider.py +++ b/playground/benchmark/benchmark_api_provider.py @@ -2,7 +2,6 @@ Usage: python3 -m playground.benchmark.benchmark_api_provider --api-endpoint-file api_endpoints.json --output-file ./benchmark_results.json --random-questions metadata_sampled.json """ - import argparse import json import time diff --git a/tests/launch_openai_api_test_server.py b/tests/launch_openai_api_test_server.py index ba622b27e..e3461a935 100644 --- a/tests/launch_openai_api_test_server.py +++ b/tests/launch_openai_api_test_server.py @@ -1,7 +1,6 @@ """ Launch an OpenAI API server with multiple model workers. """ - import os import argparse diff --git a/tests/test_cli.py b/tests/test_cli.py index a0f9473e9..113e497a4 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,5 +1,4 @@ """Test command line interface for model inference.""" - import argparse import os diff --git a/tests/test_openai_api.py b/tests/test_openai_api.py index b64eaa79f..3e7879bb5 100644 --- a/tests/test_openai_api.py +++ b/tests/test_openai_api.py @@ -4,7 +4,6 @@ Launch: python3 launch_openai_api_test_server.py """ - import warnings import openai From 4902d71fe3b3c04f3882548a5080dd4c33ac8579 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cyixin-huang1=E2=80=9D?= Date: Sat, 15 Feb 2025 23:00:26 -0800 Subject: [PATCH 33/40] Update setup_pdfchat.sh --- fastchat/serve/setup_pdfchat.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fastchat/serve/setup_pdfchat.sh b/fastchat/serve/setup_pdfchat.sh index 2511b939d..6a5a7324a 100644 --- a/fastchat/serve/setup_pdfchat.sh +++ b/fastchat/serve/setup_pdfchat.sh @@ -2,4 +2,5 @@ # Install Python packages pip install llama-index-core llama-parse llama-index-readers-file python-dotenv -pip install pdf2image \ No newline at end of file +pip install pdf2image +pip install pypdf filetype marker-pdf \ No newline at end of file From 35289190923a257995e021380cf93736b8477b06 Mon Sep 17 00:00:00 2001 From: CodingWithTim Date: Wed, 19 Feb 2025 05:12:32 +0000 Subject: [PATCH 34/40] fix formatting --- fastchat/serve/gradio_block_arena_vision.py | 18 ++++++++++++------ fastchat/utils.py | 8 +++++--- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/fastchat/serve/gradio_block_arena_vision.py b/fastchat/serve/gradio_block_arena_vision.py index d253050a8..1a4ad56f8 100644 --- a/fastchat/serve/gradio_block_arena_vision.py +++ b/fastchat/serve/gradio_block_arena_vision.py @@ -194,6 +194,7 @@ def wrap_pdfchat_query(query, document): return reformatted_query_context + PDFPARSE_MAX_RETRY = 2 PDFPARSE_SUPPORTED_LANGS = { "English": "en", @@ -211,6 +212,7 @@ def wrap_pdfchat_query(query, document): "languages": ",".join(PDFPARSE_SUPPORTED_LANGS.values()), } + def convert_base64_to_pil_image(b64_string): from PIL import Image import numpy as np @@ -218,30 +220,33 @@ def convert_base64_to_pil_image(b64_string): image_data = np.frombuffer(base64.b64decode(b64_string), dtype=np.uint8) image_bytes = BytesIO(image_data) image = Image.open(image_bytes) - + return image + def batch_convert_base64_to_images(base64_dict): import concurrent.futures + with concurrent.futures.ThreadPoolExecutor() as executor: return list(executor.map(convert_base64_to_pil_image, base64_dict.values())) + def parse_pdf(file_path): import requests url = "https://www.datalab.to/api/v1/marker" form_data = { - 'file': ('test.pdf', open(file_path, 'rb'), 'application/pdf'), + "file": ("test.pdf", open(file_path, "rb"), "application/pdf"), "force_ocr": (None, False), "paginate": (None, False), - 'output_format': (None, 'markdown'), + "output_format": (None, "markdown"), "use_llm": (None, True), "strip_existing_ocr": (None, False), - "disable_image_extraction": (None, False) + "disable_image_extraction": (None, False), } - headers = {"X-Api-Key": str(os.getenv("MARKER_API_KEY"))} + headers = {"X-Api-Key": str(os.getenv("MARKER_API_KEY"))} response = requests.post(url, files=form_data, headers=headers) data = response.json() @@ -255,7 +260,7 @@ def parse_pdf(file_path): if data["status"] == "complete": break - + output_md = data["markdown"] output_images = batch_convert_base64_to_images(data["images"]) @@ -272,6 +277,7 @@ def _prepare_text_with_image(state, text, images, csam_flag): return text + def _prepare_text_with_pdf(text, pdfs): if len(pdfs) > 0: parsed_text, imgs = parse_pdf(pdfs[0]) diff --git a/fastchat/utils.py b/fastchat/utils.py index 60527099d..c83d93436 100644 --- a/fastchat/utils.py +++ b/fastchat/utils.py @@ -482,19 +482,21 @@ def get_pdf_num_page(file_path): return len(reader.pages) + def resize_image_if_needed(image_bytes): image = Image.open(BytesIO(image_bytes)) if min(image.size) < 128: print(f"⚠️ Resizing image from {image.size} to minimum required size") new_size = (max(128, image.size[0]), max(128, image.size[1])) image = image.resize(new_size, Image.LANCZOS) - + img_byte_arr = BytesIO() image.save(img_byte_arr, format="PNG") return img_byte_arr.getvalue() - + return image_bytes + def image_moderation_request(image_bytes, endpoint, api_key): headers = {"Content-Type": "image/jpeg", "Ocp-Apim-Subscription-Key": api_key} @@ -529,7 +531,7 @@ def image_moderation_filter(image): print(f"moderating image") image_bytes = base64.b64decode(image.base64_str) - image_bytes = resize_image_if_needed(image_bytes) + image_bytes = resize_image_if_needed(image_bytes) nsfw_flagged = image_moderation_provider(image_bytes, "nsfw") csam_flagged = False From bac54f01b350d2b68872cae6538e88f9d55e0b94 Mon Sep 17 00:00:00 2001 From: CodingWithTim Date: Wed, 19 Feb 2025 06:08:55 +0000 Subject: [PATCH 35/40] FIX: add missing pdf_id --- fastchat/serve/gradio_web_server.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fastchat/serve/gradio_web_server.py b/fastchat/serve/gradio_web_server.py index 4dd302bea..9e12326d7 100644 --- a/fastchat/serve/gradio_web_server.py +++ b/fastchat/serve/gradio_web_server.py @@ -121,6 +121,9 @@ def __init__(self, model_name, is_vision=False, pdf_id=None): self.is_vision = is_vision self.ans_models = [] self.router_outputs = [] + self.pdf_id = ( + pdf_id # NOTE(Tim): Version 1 PDFChat Architecture, could be revised later. + ) # NOTE(chris): This could be sort of a hack since it assumes the user only uploads one image. If they can upload multiple, we should store a list of image hashes. self.has_csam_image = False From 1c6f91150e3e5407f9ff3b156e8af1bbaac3d0c6 Mon Sep 17 00:00:00 2001 From: CodingWithTim Date: Wed, 19 Feb 2025 06:49:56 +0000 Subject: [PATCH 36/40] FIX: add_text logic and make cleaner --- fastchat/serve/gradio_block_arena_vision.py | 4 +-- .../serve/gradio_block_arena_vision_anony.py | 25 ++++++++----------- 2 files changed, 12 insertions(+), 17 deletions(-) diff --git a/fastchat/serve/gradio_block_arena_vision.py b/fastchat/serve/gradio_block_arena_vision.py index 1a4ad56f8..b28ac9b6d 100644 --- a/fastchat/serve/gradio_block_arena_vision.py +++ b/fastchat/serve/gradio_block_arena_vision.py @@ -288,8 +288,8 @@ def _prepare_text_with_pdf(text, pdfs): if len(imgs) > 0: return wrapped_text, imgs - return wrapped_text - return text + return wrapped_text, [] + return text, [] def convert_images_to_conversation_format(images): diff --git a/fastchat/serve/gradio_block_arena_vision_anony.py b/fastchat/serve/gradio_block_arena_vision_anony.py index 304105502..2b31bff7a 100644 --- a/fastchat/serve/gradio_block_arena_vision_anony.py +++ b/fastchat/serve/gradio_block_arena_vision_anony.py @@ -358,19 +358,16 @@ def add_text( images = convert_images_to_conversation_format(images) - post_processed_text = _prepare_text_with_pdf( - text[:BLIND_MODE_INPUT_CHAR_LEN_LIMIT], pdfs + text, pdf_imgs = _prepare_text_with_pdf(text, pdfs) + + text, text_flagged, image_flagged, csam_flag = moderate_input( + state0, text, text, model_list, images + pdf_imgs, ip ) - if type(post_processed_text) is tuple: - text += post_processed_text[0] - text, text_flagged, image_flagged, csam_flag = moderate_input( - state0, text, text, model_list, images + post_processed_text[1], ip - ) - else: - text += post_processed_text - text, text_flagged, image_flagged, csam_flag = moderate_input( - state0, text, text, model_list, images, ip - ) + + text = text[:BLIND_MODE_INPUT_CHAR_LEN_LIMIT] # Hard cut-off + + if len(pdf_imgs) > 0: + text = (text, pdf_imgs) conv = states[0].conv if (len(conv.messages) - conv.offset) // 2 >= CONVERSATION_TURN_LIMIT: @@ -427,8 +424,6 @@ def add_text( + [""] ) - text = text[:BLIND_MODE_INPUT_CHAR_LEN_LIMIT] # Hard cut-off - upload_pdf_file_to_gcs( pdf_file_path=pdfs[0], filename=unique_id, @@ -436,7 +431,7 @@ def add_text( for i in range(num_sides): post_processed_text = _prepare_text_with_image( - states[i], post_processed_text, images, csam_flag=csam_flag + states[i], text, images, csam_flag=csam_flag ) states[i].conv.append_message(states[i].conv.roles[0], post_processed_text) From 85ed193dec9c0a4be0d6435ed45b9bc26fd39a95 Mon Sep 17 00:00:00 2001 From: CodingWithTim Date: Wed, 19 Feb 2025 07:26:40 +0000 Subject: [PATCH 37/40] FIX: small bug with push to gc --- fastchat/serve/gradio_block_arena_vision_anony.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/fastchat/serve/gradio_block_arena_vision_anony.py b/fastchat/serve/gradio_block_arena_vision_anony.py index 2b31bff7a..3f88f3dd7 100644 --- a/fastchat/serve/gradio_block_arena_vision_anony.py +++ b/fastchat/serve/gradio_block_arena_vision_anony.py @@ -424,10 +424,11 @@ def add_text( + [""] ) - upload_pdf_file_to_gcs( - pdf_file_path=pdfs[0], - filename=unique_id, - ) + if len(pdfs) > 0: + upload_pdf_file_to_gcs( + pdf_file_path=pdfs[0], + filename=unique_id, + ) for i in range(num_sides): post_processed_text = _prepare_text_with_image( From 0919376ec6e926a24346278cd4f00844d3cca31a Mon Sep 17 00:00:00 2001 From: PranavB-11 Date: Thu, 20 Feb 2025 00:51:18 +0000 Subject: [PATCH 38/40] Updated PDF Character Length Limit --- fastchat/constants.py | 2 ++ .../serve/gradio_block_arena_vision_anony.py | 20 ++++++++++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/fastchat/constants.py b/fastchat/constants.py index 1a98f9a1b..042ee29cd 100644 --- a/fastchat/constants.py +++ b/fastchat/constants.py @@ -38,11 +38,13 @@ BLIND_MODE_INPUT_CHAR_LEN_LIMIT = int( os.getenv("FASTCHAT_BLIND_MODE_INPUT_CHAR_LEN_LIMIT", 30000) ) +PDF_CHAR_LEN_LIMIT = 320000 # Maximum conversation turns CONVERSATION_TURN_LIMIT = 50 # Maximum PDF Page Limit PDF_PAGE_LIMIT = 50 PDF_LIMIT_MSG = f"YOU HAVE REACHED THE MAXIMUM PDF PAGE LIMIT ({PDF_PAGE_LIMIT} PAGES). PLEASE UPLOAD A SMALLER DOCUMENT." +PDF_CHAR_LIMIT_MSG = f"YOU HAVE REACHED THE MAXIMUM PDF CHARACTER LIMIT ({PDF_CHAR_LEN_LIMIT} CHARACTERS). PLEASE UPLOAD A SMALLER DOCUMENT OR PROMPT." # Session expiration time SESSION_EXPIRATION_TIME = 3600 # The output dir of log files diff --git a/fastchat/serve/gradio_block_arena_vision_anony.py b/fastchat/serve/gradio_block_arena_vision_anony.py index 3f88f3dd7..7edfacf69 100644 --- a/fastchat/serve/gradio_block_arena_vision_anony.py +++ b/fastchat/serve/gradio_block_arena_vision_anony.py @@ -17,6 +17,8 @@ PDF_MODERATION_MSG, PDF_LIMIT_MSG, PDF_PAGE_LIMIT, + PDF_CHAR_LEN_LIMIT, + PDF_CHAR_LIMIT_MSG, MODERATION_MSG, CONVERSATION_LIMIT_MSG, SLOW_MODEL_MSG, @@ -364,7 +366,23 @@ def add_text( state0, text, text, model_list, images + pdf_imgs, ip ) - text = text[:BLIND_MODE_INPUT_CHAR_LEN_LIMIT] # Hard cut-off + if len(pdfs) > 0 and len(text) > PDF_CHAR_LEN_LIMIT: + logger.info(f"pdf character limit. ip: {get_ip(request)}. text: {text}") + for i in range(num_sides): + states[i].skip_next = True + return ( + states + + [x.to_gradio_chatbot() for x in states] + + [ + {"text": PDF_CHAR_LIMIT_MSG}, + "", + no_change_btn, + ] + + [no_change_btn] * 7 + + [""] + ) + else: + text = text[:BLIND_MODE_INPUT_CHAR_LEN_LIMIT] if len(pdf_imgs) > 0: text = (text, pdf_imgs) From a4b3b6bcbc7b0a95840b38aca1134a8bcc50c966 Mon Sep 17 00:00:00 2001 From: PranavB-11 Date: Fri, 21 Feb 2025 05:48:30 +0000 Subject: [PATCH 39/40] Fixed bug with character limits --- .../serve/gradio_block_arena_vision_anony.py | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/fastchat/serve/gradio_block_arena_vision_anony.py b/fastchat/serve/gradio_block_arena_vision_anony.py index 7edfacf69..573039686 100644 --- a/fastchat/serve/gradio_block_arena_vision_anony.py +++ b/fastchat/serve/gradio_block_arena_vision_anony.py @@ -270,6 +270,7 @@ def add_text( text = chat_input files = [] + user_query = text ip = get_ip(request) logger.info(f"add_text (anony). ip: {ip}. len: {len(text)}") states = [state0, state1] @@ -366,21 +367,22 @@ def add_text( state0, text, text, model_list, images + pdf_imgs, ip ) - if len(pdfs) > 0 and len(text) > PDF_CHAR_LEN_LIMIT: - logger.info(f"pdf character limit. ip: {get_ip(request)}. text: {text}") - for i in range(num_sides): - states[i].skip_next = True - return ( - states - + [x.to_gradio_chatbot() for x in states] - + [ - {"text": PDF_CHAR_LIMIT_MSG}, - "", - no_change_btn, - ] - + [no_change_btn] * 7 - + [""] - ) + if len(pdfs) > 0: + if len(text) > PDF_CHAR_LEN_LIMIT: + logger.info(f"pdf character limit. ip: {get_ip(request)}. text: {text}") + for i in range(num_sides): + states[i].skip_next = True + return ( + states + + [x.to_gradio_chatbot() for x in states] + + [ + {"text": PDF_CHAR_LIMIT_MSG}, + "", + no_change_btn, + ] + + [no_change_btn] * 7 + + [""] + ) else: text = text[:BLIND_MODE_INPUT_CHAR_LEN_LIMIT] From 5a314de0d8fccf94227ca9de14869f8025717013 Mon Sep 17 00:00:00 2001 From: PranavB-11 Date: Fri, 21 Feb 2025 05:49:16 +0000 Subject: [PATCH 40/40] Fixed bug with character limits --- fastchat/serve/gradio_block_arena_vision_anony.py | 1 - 1 file changed, 1 deletion(-) diff --git a/fastchat/serve/gradio_block_arena_vision_anony.py b/fastchat/serve/gradio_block_arena_vision_anony.py index 573039686..f1301122f 100644 --- a/fastchat/serve/gradio_block_arena_vision_anony.py +++ b/fastchat/serve/gradio_block_arena_vision_anony.py @@ -270,7 +270,6 @@ def add_text( text = chat_input files = [] - user_query = text ip = get_ip(request) logger.info(f"add_text (anony). ip: {ip}. len: {len(text)}") states = [state0, state1]