From a25745ffaf4f329d2781e4107bdc1207dc84374e Mon Sep 17 00:00:00 2001 From: magikRUKKOLA <124943879+magikRUKKOLA@users.noreply.github.com> Date: Mon, 22 Jun 2026 15:29:47 +0000 Subject: [PATCH] implement perplexity in llama-server --- examples/server/llama-perplexity.sh | 102 ++++++++ examples/server/server-context.cpp | 349 ++++++++++++++++++++++++++++ examples/server/server-context.h | 23 ++ examples/server/server-task.h | 27 +++ examples/server/server.cpp | 93 ++++++++ 5 files changed, 594 insertions(+) create mode 100644 examples/server/llama-perplexity.sh diff --git a/examples/server/llama-perplexity.sh b/examples/server/llama-perplexity.sh new file mode 100644 index 0000000000..18240a0c87 --- /dev/null +++ b/examples/server/llama-perplexity.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +set -euo pipefail + +SERVER="${SERVER:-http://localhost:8080}" +INPUT_FILE="${1:-}" +STATE_FILE="${2:-/tmp/perplexity_state.bin}" +N_CTX="${N_CTX:-512}" +SAVE_EVERY="${SAVE_EVERY:-10}" +TMPDIR="${TMPDIR:-/tmp}" + +if [[ -z "$INPUT_FILE" ]]; then + echo "Usage: $0 [state-file]" >&2 + exit 1 +fi + +TOKEN_FILE="$TMPDIR/perplexity_tokens_$$.json" +REQ_FILE="$TMPDIR/perplexity_req_$$.json" +trap 'rm -f "$TOKEN_FILE" "$REQ_FILE"' EXIT + +jq -s -R '{"content": ., "add_special": true}' < "$INPUT_FILE" > "$REQ_FILE" +curl -sf -X POST "$SERVER/tokenize" -H "Content-Type: application/json" -d @"$REQ_FILE" > "$TOKEN_FILE" || { + echo "[!] Tokenization failed" >&2; exit 1 +} + +TOTAL_TOKENS=$(jq -e '.tokens | length' "$TOKEN_FILE") +N_CHUNKS=$(( TOTAL_TOKENS / N_CTX )) + +print_final() { + curl -sf "$SERVER/perplexity/state" | jq -r '[.nll, .nll2, .count, .perplexity, .chunk_index] | @tsv' | awk -v n_ctx="$N_CTX" ' + { + nll=$1; nll2=$2; count=$3; ppl=$4; chunk_index=$5; + if (count>1) { + mean=nll/count; + var=(nll2/count)-(mean*mean); + if (var>0) { + unc=(sqrt(var/(count-1)))*ppl; + printf("Final estimate: PPL over %d chunks for n_ctx=%d = %.4lf +/- %.5lf\n", chunk_index, n_ctx, ppl, unc); + } else { + printf("Final estimate: PPL over %d chunks for n_ctx=%d = %.4lf\n", chunk_index, n_ctx, ppl); + } + } + }' +} + +save_checkpoint() { + curl -sf -X POST "$SERVER/perplexity/save" -H "Content-Type: application/json" -d "{\"filename\":\"$STATE_FILE\"}" > /dev/null 2>&1 || true +} + +START_CHUNK=0 +if [[ -f "$STATE_FILE" ]]; then + curl -sf -X POST "$SERVER/perplexity/load" -H "Content-Type: application/json" -d "{\"filename\":\"$STATE_FILE\"}" > /dev/null 2>&1 || true + START_CHUNK=$(curl -sf "$SERVER/perplexity/state" | jq -r '.chunk_index // 0') +fi + +if (( START_CHUNK >= N_CHUNKS )); then + echo "perplexity: calculating perplexity over $N_CHUNKS chunks, n_ctx=$N_CTX, batch_size=$N_CTX, n_seq=1" + print_final + read -p "Invalidate state and start over? [y/N] " answer < /dev/tty + if [[ "$answer" == [yY] ]]; then + rm -f "$STATE_FILE" + curl -sf -X POST "$SERVER/perplexity/reset" -H "Content-Type: application/json" -d '{}' > /dev/null 2>&1 || true + echo "[*] State invalidated. Restart the script to begin from scratch." + fi + exit 0 +fi + +echo "perplexity: calculating perplexity over $N_CHUNKS chunks, n_ctx=$N_CTX, batch_size=$N_CTX, n_seq=1" + +shutdown() { + echo "" + save_checkpoint + exit 0 +} +trap shutdown SIGINT SIGTERM + +chunk_idx=$START_CHUNK +while (( chunk_idx < N_CHUNKS )); do + start=$(( chunk_idx * N_CTX )) + jq "{\"tokens\": .tokens[$start:$((start + N_CTX))], \"n_ctx\": $N_CTX}" "$TOKEN_FILE" > "$REQ_FILE" + + resp=$(curl -sf -X POST "$SERVER/perplexity" -H "Content-Type: application/json" -d @"$REQ_FILE") || { + echo "[!] Request failed at chunk $((chunk_idx + 1))" >&2; exit 1 + } + + if echo "$resp" | jq -e '.error' > /dev/null 2>&1; then + echo "[!] Server error: $(echo "$resp" | jq -c '.error')" >&2; exit 1 + fi + + idx=$(echo "$resp" | jq -r '.chunk_index') + ppl=$(echo "$resp" | jq -r '.perplexity') + printf "[%d]%.4lf," "$idx" "$ppl" + + if (( (chunk_idx + 1) % SAVE_EVERY == 0 )); then + save_checkpoint + fi + + chunk_idx=$((chunk_idx + 1)) +done + +echo "" +print_final +save_checkpoint diff --git a/examples/server/server-context.cpp b/examples/server/server-context.cpp index c2563b2306..67039654e5 100644 --- a/examples/server/server-context.cpp +++ b/examples/server/server-context.cpp @@ -18,6 +18,69 @@ #include #include +bool server_perplexity_state::save(const std::string & filename) const { + std::ofstream file(filename, std::ios::binary); + if (!file) return false; + + const uint32_t magic = 0x504C5459; + const uint32_t version = 2; + file.write((const char *)&magic, sizeof(magic)); + file.write((const char *)&version, sizeof(version)); + + file.write((const char *)&n_ctx, sizeof(n_ctx)); + file.write((const char *)&n_batch, sizeof(n_batch)); + file.write((const char *)&ppl_stride, sizeof(ppl_stride)); + file.write((const char *)&nll, sizeof(nll)); + file.write((const char *)&nll2, sizeof(nll2)); + file.write((const char *)&count, sizeof(count)); + file.write((const char *)&chunk_index, sizeof(chunk_index)); + + size_t n_tokens = tokens.size(); + file.write((const char *)&n_tokens, sizeof(n_tokens)); + if (n_tokens) file.write((const char *)tokens.data(), n_tokens * sizeof(llama_token)); + + size_t n_all = all_tokens.size(); + file.write((const char *)&n_all, sizeof(n_all)); + if (n_all) file.write((const char *)all_tokens.data(), n_all * sizeof(llama_token)); + + return file.good(); +} + +bool server_perplexity_state::load(const std::string & filename) { + std::ifstream file(filename, std::ios::binary); + if (!file) return false; + + uint32_t magic, version; + file.read((char *)&magic, sizeof(magic)); + file.read((char *)&version, sizeof(version)); + if (magic != 0x504C5459 || version < 1 || version > 2) return false; + + file.read((char *)&n_ctx, sizeof(n_ctx)); + file.read((char *)&n_batch, sizeof(n_batch)); + file.read((char *)&ppl_stride, sizeof(ppl_stride)); + file.read((char *)&nll, sizeof(nll)); + file.read((char *)&nll2, sizeof(nll2)); + file.read((char *)&count, sizeof(count)); + + if (version >= 2) { + file.read((char *)&chunk_index, sizeof(chunk_index)); + } else { + chunk_index = 0; + } + + size_t n_tokens; + file.read((char *)&n_tokens, sizeof(n_tokens)); + tokens.resize(n_tokens); + if (n_tokens) file.read((char *)tokens.data(), n_tokens * sizeof(llama_token)); + + size_t n_all; + file.read((char *)&n_all, sizeof(n_all)); + all_tokens.resize(n_all); + if (n_all) file.read((char *)all_tokens.data(), n_all * sizeof(llama_token)); + + return file.good(); +} + static void server_prompt_checkpoint_update(server_prompt_checkpoint & ckpt, llama_context * ctx, int id, int64_t n_tokens, llama_pos pos_min = -1, llama_pos pos_max = -1, int32_t offset = 0) { if (pos_min == -1) { pos_min = llama_kv_cache_seq_pos_min(ctx, id); @@ -2717,6 +2780,173 @@ static size_t load_server_tokens_from_file(const std::string & filename, server return pos; } +void server_context::process_perplexity_chunk(const std::vector & new_tokens, + server_task_result_perplexity & result) { + if (!perplexity_state) { + perplexity_state = std::make_unique(); + perplexity_state->n_ctx = params_base.n_ctx; + perplexity_state->n_batch = params_base.n_batch; + } + + auto & state = *perplexity_state; + state.all_tokens.insert(state.all_tokens.end(), new_tokens.begin(), new_tokens.end()); + state.tokens.insert(state.tokens.end(), new_tokens.begin(), new_tokens.end()); + + const int n_ctx = state.n_ctx; + const int n_batch = state.n_batch; + const int n_vocab = llama_n_vocab(model); + const int first = n_ctx / 2; + const bool add_bos = llama_should_add_bos_token(model); + + LLAMA_LOG_DEBUG("[PPL DEBUG] process_perplexity_chunk: n_ctx=%d n_batch=%d n_vocab=%d first=%d buffered=%zu\n", + n_ctx, n_batch, n_vocab, first, state.tokens.size()); + + llama_batch batch = llama_batch_init(std::min(n_batch, n_ctx), 0, 1); + + while ((int)state.tokens.size() >= n_ctx) { + std::vector chunk(state.tokens.begin(), state.tokens.begin() + n_ctx); + + llama_kv_cache_clear(ctx); + + const int num_batches = (n_ctx + n_batch - 1) / n_batch; + std::vector logits; + if (num_batches > 1) { + logits.reserve((size_t)(n_ctx - first) * n_vocab); + } + + bool decode_failed = false; + for (int j = 0; j < num_batches; ++j) { + const int batch_start = j * n_batch; + const int batch_size = std::min(n_ctx - batch_start, n_batch); + + int n_outputs = 0; + + batch.n_tokens = 0; + const int seq_start = batch_start; + const llama_token token_org = chunk[seq_start]; + if (add_bos && j == 0) { + chunk[seq_start] = llama_token_bos(model); + } + + for (int k = 0; k < batch_size; ++k) { + const int idx = k; + batch.token[idx] = chunk[seq_start + k]; + batch.pos[idx] = j * n_batch + k; + batch.n_seq_id[idx] = 1; + batch.seq_id[idx][0] = 0; + batch.logits[idx] = batch.pos[idx] >= first ? 1 : 0; + n_outputs += batch.logits[idx] != 0; + } + batch.n_tokens = batch_size; + + chunk[seq_start] = token_org; + + LLAMA_LOG_DEBUG("[PPL DEBUG] batch %d/%d n_tokens=%d pos=[%d..%d] first_token=%d n_outputs=%d\n", + j + 1, num_batches, batch_size, + batch.pos[0], batch.pos[batch_size - 1], + batch.token[0], n_outputs); + + int decode_ret = llama_decode(ctx, batch); + + if (decode_ret != 0) { + decode_failed = true; + LLAMA_LOG_DEBUG("[PPL DEBUG] decode FAILED on batch %d/%d with ret=%d\n", j + 1, num_batches, decode_ret); + break; + } + + if (num_batches > 1 && n_outputs > 0) { + const float * batch_logits = llama_get_logits(ctx); + logits.insert(logits.end(), batch_logits, batch_logits + (size_t)n_outputs * n_vocab); + } + } + + if (decode_failed) { + result.error = true; + llama_batch_free(batch); + return; + } + + // Reference uses llama_get_logits_ith(ctx, first) directly when num_batches == 1 + const float * all_logits = num_batches > 1 + ? logits.data() + : llama_get_logits_ith(ctx, first); + + if (!all_logits) { + LLAMA_LOG_DEBUG("[PPL DEBUG] failed to get logits\n"); + result.error = true; + llama_batch_free(batch); + return; + } + + // Compute log-probabilities for tokens [first .. n_ctx-2] + double chunk_nll = 0.0; + double chunk_nll2 = 0.0; + for (int j = first; j < n_ctx - 1; ++j) { + const float * tok_logits = all_logits + (size_t)(j - first) * n_vocab; + + // ---- identical to perplexity.cpp::log_softmax() ---- + float max_logit = tok_logits[0]; + for (int k = 1; k < n_vocab; ++k) { + max_logit = std::max(max_logit, tok_logits[k]); + } + + double sum_exp = 0.0; + for (int k = 0; k < n_vocab; ++k) { + sum_exp += expf(tok_logits[k] - max_logit); + } + + // keep full double precision, do NOT cast to float + double log_softmax = tok_logits[chunk[j + 1]] - max_logit - log(sum_exp); + double v = -log_softmax; + // -------------------------------------------------- + + if (j < first + 3) { + LLAMA_LOG_DEBUG("[PPL DEBUG] j=%d next_token=%d max_logit=%f log_softmax=%.10f v=%.10f\n", + j, chunk[j+1], max_logit, log_softmax, v); + } + + chunk_nll += v; + chunk_nll2 += v * v; + } + + state.nll += chunk_nll; + state.nll2 += chunk_nll2; + state.count += n_ctx - 1 - first; + state.chunk_index++; + + LLAMA_LOG_DEBUG("[PPL DEBUG] chunk_nll=%f chunk_nll2=%f count=%ld perplexity_so_far=%f\n", + chunk_nll, chunk_nll2, state.count, state.count > 0 ? std::exp(state.nll / state.count) : 0.0); + + state.tokens.erase(state.tokens.begin(), state.tokens.begin() + n_ctx); + } + + llama_batch_free(batch); + + result.nll = state.nll; + result.nll2 = state.nll2; + result.count = state.count; + result.perplexity = state.count > 0 ? std::exp(state.nll / state.count) : 0.0; + result.tokens_processed = state.count; + result.tokens_total = state.all_tokens.size(); + result.chunk_index = state.chunk_index; + result.error = false; +} + +bool server_context::perplexity_save(const std::string & filename) { + return perplexity_state ? perplexity_state->save(filename) : false; +} + +bool server_context::perplexity_load(const std::string & filename) { + if (!perplexity_state) { + perplexity_state = std::make_unique(); + } + return perplexity_state->load(filename); +} + +void server_context::perplexity_reset() { + perplexity_state.reset(); +} + void server_context::process_single_task(server_task&& task) { switch (task.type) { case SERVER_TASK_TYPE_COMPLETION: @@ -3129,6 +3359,125 @@ void server_context::process_single_task(server_task&& task) { result.data = json{{ "success", true }}; queue_results.send(result); } break; + case SERVER_TASK_TYPE_PERPLEXITY: + { + // perplexity needs exclusive access to the model context + if (!slots_idle()) { + queue_tasks.defer(std::move(task)); + break; + } + + std::vector new_tokens; + if (task.data.contains("tokens")) { + new_tokens = task.data["tokens"].get>(); + } else if (task.data.contains("text")) { + new_tokens = ::common_tokenize(ctx, task.data["text"].get(), false); + } else { + send_error(task, "perplexity requires 'text' or 'tokens'", ERROR_TYPE_INVALID_REQUEST); + break; + } + + if (!perplexity_state) { + perplexity_state = std::make_unique(); + perplexity_state->n_ctx = json_value(task.data, "n_ctx", params_base.n_ctx); + perplexity_state->n_batch = json_value(task.data, "n_batch", params_base.n_batch); + } + + LLAMA_LOG_DEBUG("[PPL DEBUG] task: requested n_ctx=%d n_batch=%d actual_llama_n_ctx=%d n_new_tokens=%zu\n", + perplexity_state->n_ctx, perplexity_state->n_batch, + llama_n_ctx(ctx), new_tokens.size()); + + server_task_result_perplexity ppl_res; + ppl_res.id = task.id; + ppl_res.stop = true; + try { + process_perplexity_chunk(new_tokens, ppl_res); + } catch (const std::exception & e) { + LLAMA_LOG_DEBUG("[PPL DEBUG] EXCEPTION in process_perplexity_chunk: %s\n", e.what()); + ppl_res.error = true; + } + + server_task_result res; + res.id = task.id; + res.stop = true; + if (ppl_res.error) { + res.error = true; + res.data = format_error_response("perplexity evaluation failed", ERROR_TYPE_SERVER); + } else { + try { + res.error = false; + res.data = ppl_res.to_json(); + LLAMA_LOG_DEBUG("[PPL DEBUG] result json prepared ok\n"); + } catch (const std::exception & e) { + LLAMA_LOG_DEBUG("[PPL DEBUG] EXCEPTION in to_json/dump: %s\n", e.what()); + res.error = true; + res.data = format_error_response("perplexity json serialization failed", ERROR_TYPE_SERVER); + } + } + queue_results.send(res); + } break; + + case SERVER_TASK_TYPE_PERPLEXITY_STATE: + { + server_task_result_perplexity ppl_res; + ppl_res.id = task.id; + ppl_res.stop = true; + if (perplexity_state) { + ppl_res.nll = perplexity_state->nll; + ppl_res.nll2 = perplexity_state->nll2; + ppl_res.count = perplexity_state->count; + ppl_res.perplexity = perplexity_state->count > 0 + ? std::exp(perplexity_state->nll / perplexity_state->count) + : 0.0; + ppl_res.tokens_processed = perplexity_state->count; + ppl_res.tokens_total = perplexity_state->all_tokens.size(); + ppl_res.chunk_index = perplexity_state->chunk_index; + } + + server_task_result res; + res.id = task.id; + res.stop = true; + res.data = ppl_res.to_json(); + queue_results.send(res); + } break; + + case SERVER_TASK_TYPE_PERPLEXITY_SAVE: + { + std::string filename = json_value(task.data, "filename", std::string("perplexity_state.bin")); + bool ok = perplexity_save(filename); + + server_task_result res; + res.id = task.id; + res.stop = true; + res.error = !ok; + res.data = { {"saved", ok}, {"filename", filename} }; + queue_results.send(res); + } break; + + case SERVER_TASK_TYPE_PERPLEXITY_LOAD: + { + std::string filename = json_value(task.data, "filename", std::string("perplexity_state.bin")); + bool ok = perplexity_load(filename); + + server_task_result res; + res.id = task.id; + res.stop = true; + res.error = !ok; + res.data = { {"loaded", ok}, {"filename", filename} }; + queue_results.send(res); + } break; + + case SERVER_TASK_TYPE_PERPLEXITY_RESET: + { + perplexity_reset(); + + server_task_result res; + res.id = task.id; + res.stop = true; + res.error = false; + res.data = { {"reset", true} }; + queue_results.send(res); + } break; } } diff --git a/examples/server/server-context.h b/examples/server/server-context.h index 68e0d215db..b3d1ca9fcc 100644 --- a/examples/server/server-context.h +++ b/examples/server/server-context.h @@ -231,6 +231,22 @@ struct server_metrics { void reset_bucket(); }; +struct server_perplexity_state { + std::vector tokens; // buffered, not yet evaluated + std::vector all_tokens; // complete history for reference + + double nll = 0.0; + double nll2 = 0.0; + int64_t count = 0; + int32_t n_ctx = 512; + int32_t n_batch = 512; + int32_t ppl_stride = 0; // 0 = non-strided (standard) + int64_t chunk_index = 0; + + bool save(const std::string & filename) const; + bool load(const std::string & filename); +}; + struct server_context { llama_model* model = nullptr; llama_context* ctx = nullptr; @@ -279,6 +295,8 @@ struct server_context { int32_t cache_ram_n_min = 0; float cache_ram_similarity = 0.5f; + std::unique_ptr perplexity_state; + ~server_context(); bool load_model(const gpt_params& params_); @@ -392,4 +410,9 @@ struct server_context { void create_checkpoint_at_interval(server_slot & slot, const gpt_params & params_base); void release_slot_after_final_response(server_slot & slot); + + void process_perplexity_chunk(const std::vector& tokens, server_task_result_perplexity& result); + bool perplexity_save(const std::string& filename); + bool perplexity_load(const std::string& filename); + void perplexity_reset(); }; diff --git a/examples/server/server-task.h b/examples/server/server-task.h index 76a6bad36a..0ec8f1a473 100644 --- a/examples/server/server-task.h +++ b/examples/server/server-task.h @@ -34,6 +34,11 @@ enum server_task_type { SERVER_TASK_TYPE_LOAD_CONTROL_VECTOR, SERVER_TASK_TYPE_UNLOAD_CONTROL_VECTOR, SERVER_TASK_TYPE_SET_CONTROL_VECTOR, + SERVER_TASK_TYPE_PERPLEXITY, + SERVER_TASK_TYPE_PERPLEXITY_STATE, + SERVER_TASK_TYPE_PERPLEXITY_SAVE, + SERVER_TASK_TYPE_PERPLEXITY_LOAD, + SERVER_TASK_TYPE_PERPLEXITY_RESET, }; enum oaicompat_type { @@ -343,6 +348,28 @@ struct server_task_result_embd : server_task_result { } }; +struct server_task_result_perplexity : server_task_result { + double nll = 0.0; + double nll2 = 0.0; + int64_t count = 0; + double perplexity = 0.0; + int64_t tokens_processed = 0; + int64_t tokens_total = 0; + int64_t chunk_index = 0; + + virtual json to_json() override { + return { + {"nll", nll}, + {"nll2", nll2}, + {"count", count}, + {"perplexity", perplexity}, + {"tokens_processed", tokens_processed}, + {"tokens_total", tokens_total}, + {"chunk_index", chunk_index}, + }; + } +}; + // using shared_ptr for polymorphism of server_task_result using server_task_result_ptr = std::unique_ptr; diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 1e33a8be59..785a235e4f 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -1013,6 +1013,93 @@ int main(int argc, char ** argv) { } }; + // + // Perplexity handlers + // + + const auto handle_perplexity = [&ctx_server](const httplib::Request & req, httplib::Response & res) { + json body = json::parse(req.body); + server_task task; + task.type = SERVER_TASK_TYPE_PERPLEXITY; + task.data = body; + + const int id_task = ctx_server.queue_tasks.post(std::move(task)); + ctx_server.queue_results.add_waiting_task_id(id_task); + + server_task_result result = ctx_server.queue_results.recv(id_task); + ctx_server.queue_results.remove_waiting_task_id(id_task); + + if (result.error) { + res_err(res, result.data); + } else { + res.set_content(result.data.dump(), "application/json"); + } + }; + + const auto handle_perplexity_state = [&ctx_server](const httplib::Request &, httplib::Response & res) { + server_task task; + task.type = SERVER_TASK_TYPE_PERPLEXITY_STATE; + + const int id_task = ctx_server.queue_tasks.post(std::move(task)); + ctx_server.queue_results.add_waiting_task_id(id_task); + + server_task_result result = ctx_server.queue_results.recv(id_task); + ctx_server.queue_results.remove_waiting_task_id(id_task); + + res.set_content(result.data.dump(), "application/json"); + }; + + const auto handle_perplexity_save = [&ctx_server](const httplib::Request & req, httplib::Response & res) { + json body = json::parse(req.body); + server_task task; + task.type = SERVER_TASK_TYPE_PERPLEXITY_SAVE; + task.data = body; + + const int id_task = ctx_server.queue_tasks.post(std::move(task)); + ctx_server.queue_results.add_waiting_task_id(id_task); + + server_task_result result = ctx_server.queue_results.recv(id_task); + ctx_server.queue_results.remove_waiting_task_id(id_task); + + if (result.error) { + res_err(res, result.data); + } else { + res.set_content(result.data.dump(), "application/json"); + } + }; + + const auto handle_perplexity_load = [&ctx_server](const httplib::Request & req, httplib::Response & res) { + json body = json::parse(req.body); + server_task task; + task.type = SERVER_TASK_TYPE_PERPLEXITY_LOAD; + task.data = body; + + const int id_task = ctx_server.queue_tasks.post(std::move(task)); + ctx_server.queue_results.add_waiting_task_id(id_task); + + server_task_result result = ctx_server.queue_results.recv(id_task); + ctx_server.queue_results.remove_waiting_task_id(id_task); + + if (result.error) { + res_err(res, result.data); + } else { + res.set_content(result.data.dump(), "application/json"); + } + }; + + const auto handle_perplexity_reset = [&ctx_server](const httplib::Request &, httplib::Response & res) { + server_task task; + task.type = SERVER_TASK_TYPE_PERPLEXITY_RESET; + + const int id_task = ctx_server.queue_tasks.post(std::move(task)); + ctx_server.queue_results.add_waiting_task_id(id_task); + + server_task_result result = ctx_server.queue_results.recv(id_task); + ctx_server.queue_results.remove_waiting_task_id(id_task); + + res.set_content(result.data.dump(), "application/json"); + }; + const auto handle_props = [&ctx_server](const httplib::Request & req, httplib::Response & res) { std::string template_key = "tokenizer.chat_template", curr_tmpl; int32_t tlen = llama_model_meta_val_str(ctx_server.model, template_key.c_str(), nullptr, 0); @@ -2152,6 +2239,12 @@ int main(int argc, char ** argv) { svr->Post("/rename_prompt", rename_saved_prompt); } + // Perplexity routes + svr->Post("/perplexity", handle_perplexity); + svr->Get ("/perplexity/state", handle_perplexity_state); + svr->Post("/perplexity/save", handle_perplexity_save); + svr->Post("/perplexity/load", handle_perplexity_load); + svr->Post("/perplexity/reset", handle_perplexity_reset); svr->Get ("/version", handle_version); if (!params.sql_save_file.empty()) {