aparler/c-llm
1
0
Fork 0
mirror of https://github.com/karpathy/llm.c.git synced 2026-07-28 20:35:09 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Got serving ggufs working

Browse source
This commit is contained in:
Cruthaifios 2026-03-21 22:33:41 -05:00
parent f1e2ace651
commit 17562bf011
7 changed files with 2133 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

5
Makefile
View file

@ -285,6 +285,9 @@ test_gpt2fp32cu: test_gpt2_fp32.cu
profile_gpt2cu: profile_gpt2.cu $(NVCC_CUDNN)
$(NVCC) $(NVCC_FLAGS) $(PFLAGS) -lineinfo $^ $(NVCC_LDFLAGS) $(NVCC_INCLUDES) $(NVCC_LDLIBS) $(CUDA_OUTPUT_FILE)
llm-serve: serve.c
$(CC) $(CFLAGS) $(INCLUDES) $(LDFLAGS) $^ $(LDLIBS) $(OUTPUT_FILE)
clean:
$(REMOVE_FILES) $(TARGETS)
$(REMOVE_FILES) $(TARGETS) llm-serve
$(REMOVE_BUILD_OBJECT_FILES)

104
README.md
View file

@ -113,6 +113,110 @@ make test_gpt2cu USE_CUDNN=1 && ./test_gpt2cu
This tests both the fp32 path and the mixed precision path. The test should pass and print `overall okay: 1`.
## llm-serve (inference server)
`llm-serve` is a lightweight LLaMA inference server that loads GGUF model files and serves an OpenAI-compatible HTTP API. It runs entirely in C with no external dependencies beyond libc and (optionally) OpenMP.
### building
```bash
make llm-serve
```
This produces the `./llm-serve` binary.
### running
```bash
./llm-serve -m /path/to/model.gguf [-p port] [-c context_size] [-t threads]
```
| Flag | Default | Description |
|------|---------|-------------|
| `-m` | *(required)* | Path to a GGUF model file |
| `-p` | 8080 | HTTP port |
| `-c` | model default | Context window size override |
| `-t` | 4 | Number of OpenMP threads |
Example with TinyLlama:
```bash
./llm-serve -m ~/models/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf -p 8080 -t 4
```
### API endpoints
**`GET /health`** — Returns `{"status":"ok"}` when the server is ready.
**`POST /v1/completions`** — OpenAI-compatible text completion.
```bash
curl -X POST http://localhost:8080/v1/completions \
-H "Content-Type: application/json" \
-d '{
"prompt": "The capital of France is",
"max_tokens": 32,
"temperature": 0.7,
"top_p": 0.9,
"stream": false
}'
```
Response:
```json
{
"choices": [{"text": " Paris, which...", "finish_reason": "stop"}],
"usage": {"prompt_tokens": 7, "completion_tokens": 32}
}
```
Set `"stream": true` for Server-Sent Events (SSE) streaming.
### integration test
An integration test script verifies the happy path end-to-end: build, server startup, health check, completions (greedy + sampled), streaming, and error handling.
**Prerequisites:** `curl` and `jq` must be installed.
```bash
# Run with the default model path
./test_serve_integration.sh
# Or specify a model explicitly
./test_serve_integration.sh /path/to/model.gguf
```
The test uses port 18199 to avoid conflicts and automatically cleans up the server process on exit. It takes 1–2 minutes depending on model size (most of that is model loading and generation).
**What it tests:**
| # | Test | What's checked |
|---|------|----------------|
| 1 | `GET /health` | Returns `{"status":"ok"}` |
| 2 | `POST /v1/completions` (greedy) | Valid JSON, non-empty text, finish_reason, usage tokens |
| 3 | `POST /v1/completions` (sampling) | temp=0.8, top_p=0.95 returns text |
| 4 | Missing prompt | Returns HTTP 400 |
| 5 | Unknown route | Returns HTTP 404 |
| 6 | Streaming | SSE `data:` lines + `[DONE]` sentinel |
Output looks like:
```
=== llm-serve integration test ===
Building llm-serve...
Starting server on port 18199...
Server ready (PID 12345).
Test 1: GET /health
✓ /health returns {"status":"ok"}
Test 2: POST /v1/completions (basic)
✓ Response is valid JSON
✓ Response has non-empty choices array
✓ Generated text is non-empty: "Paris, the city of..."
...
=== Results: 12 passed, 0 failed ===
```
## tutorial
I attached a very small tutorial here, in [doc/layernorm/layernorm.md](doc/layernorm/layernorm.md). It's a simple, step-by-step guide to implementing a single layer of the GPT-2 model, the layernorm layer. This is a good starting point to understand how the layers are implemented in C.

BIN
llm-serve Executable file
View file

Binary file not shown.

407
llmc/bpe_tokenizer.h Normal file
View file

@ -0,0 +1,407 @@
/*
BPE Tokenizer loaded from GGUF metadata.
Supports SentencePiece-style (LLaMA 2) and tiktoken-style (LLaMA 3) vocabularies.
*/
#ifndef BPE_TOKENIZER_H
#define BPE_TOKENIZER_H
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdint.h>
#include "gguf.h"
// ----------------------------------------------------------------------------
// BPE Tokenizer
#define BPE_MAX_VOCAB 200000
#define BPE_MAX_MERGES 200000
#define BPE_MAX_TOKEN 512
typedef struct {
char **tokens; // vocab: token strings (null-terminated)
float *scores; // token scores (for SentencePiece)
int *token_type; // token type flags
int vocab_size;
int bos_id;
int eos_id;
// merge rules (for explicit BPE like tiktoken)
char **merge_left;
char **merge_right;
int n_merges;
int has_merges; // 0 = SentencePiece score-based, 1 = explicit merges
// byte-level fallback table
int byte_token[256]; // byte value -> token id, -1 if none
} BPETokenizer;
// ----------------------------------------------------------------------------
// UTF-8 helpers
static int utf8_char_len(unsigned char c) {
if (c < 0x80) return 1;
if (c < 0xE0) return 2;
if (c < 0xF0) return 3;
return 4;
}
// Check if s starts with the UTF-8 encoding of U+2581 (▁, the SentencePiece space)
// U+2581 encodes as E2 96 81
static int is_sp_space(const char *s) {
return (unsigned char)s[0] == 0xE2 &&
(unsigned char)s[1] == 0x96 &&
(unsigned char)s[2] == 0x81;
}
// ----------------------------------------------------------------------------
// Load from GGUF
static void bpe_tokenizer_init(BPETokenizer *tok, GGUFContext *ctx) {
memset(tok, 0, sizeof(*tok));
// bos/eos
tok->bos_id = (int)gguf_get_u64(ctx, "tokenizer.ggml.bos_token_id", 1);
tok->eos_id = (int)gguf_get_u64(ctx, "tokenizer.ggml.eos_token_id", 2);
// tokens array
GGUFValue *tokens_val = gguf_find_kv(ctx, "tokenizer.ggml.tokens");
if (!tokens_val || tokens_val->type != GGUF_TYPE_ARRAY) {
fprintf(stderr, "BPE: no tokenizer.ggml.tokens in GGUF\n");
exit(EXIT_FAILURE);
}
tok->vocab_size = (int)tokens_val->arr.count;
tok->tokens = (char **)calloc(tok->vocab_size, sizeof(char *));
for (int i = 0; i < tok->vocab_size; i++) {
tok->tokens[i] = strdup(tokens_val->arr.items[i].str);
}
// scores (optional)
GGUFValue *scores_val = gguf_find_kv(ctx, "tokenizer.ggml.scores");
if (scores_val && scores_val->type == GGUF_TYPE_ARRAY) {
tok->scores = (float *)calloc(tok->vocab_size, sizeof(float));
for (int i = 0; i < tok->vocab_size && (uint64_t)i < scores_val->arr.count; i++) {
GGUFValue *sv = &scores_val->arr.items[i];
tok->scores[i] = (sv->type == GGUF_TYPE_FLOAT32) ? sv->f32 : 0.0f;
}
}
// token types (optional)
GGUFValue *type_val = gguf_find_kv(ctx, "tokenizer.ggml.token_type");
if (type_val && type_val->type == GGUF_TYPE_ARRAY) {
tok->token_type = (int *)calloc(tok->vocab_size, sizeof(int));
for (int i = 0; i < tok->vocab_size && (uint64_t)i < type_val->arr.count; i++) {
GGUFValue *tv = &type_val->arr.items[i];
tok->token_type[i] = (tv->type == GGUF_TYPE_INT32) ? tv->i32 : 0;
}
}
// explicit merges (optional, tiktoken-style)
GGUFValue *merges_val = gguf_find_kv(ctx, "tokenizer.ggml.merges");
if (merges_val && merges_val->type == GGUF_TYPE_ARRAY && merges_val->arr.count > 0) {
tok->has_merges = 1;
tok->n_merges = (int)merges_val->arr.count;
tok->merge_left = (char **)calloc(tok->n_merges, sizeof(char *));
tok->merge_right = (char **)calloc(tok->n_merges, sizeof(char *));
for (int i = 0; i < tok->n_merges; i++) {
const char *entry = merges_val->arr.items[i].str;
// format: "left right"
const char *sp = strchr(entry, ' ');
if (sp) {
size_t llen = (size_t)(sp - entry);
tok->merge_left[i] = (char *)malloc(llen + 1);
memcpy(tok->merge_left[i], entry, llen);
tok->merge_left[i][llen] = '\0';
tok->merge_right[i] = strdup(sp + 1);
} else {
tok->merge_left[i] = strdup(entry);
tok->merge_right[i] = strdup("");
}
}
}
// build byte fallback table
memset(tok->byte_token, -1, sizeof(tok->byte_token));
// Look for tokens of the form <0xNN>
for (int i = 0; i < tok->vocab_size; i++) {
const char *t = tok->tokens[i];
unsigned int byte_val;
if (sscanf(t, "<0x%02X>", &byte_val) == 1 && byte_val < 256) {
tok->byte_token[byte_val] = i;
}
}
}
// ----------------------------------------------------------------------------
// Decode: token_id -> string
static const char *bpe_decode(BPETokenizer *tok, int token_id) {
if (token_id < 0 || token_id >= tok->vocab_size) return "";
return tok->tokens[token_id];
}
// Decode a token, converting SentencePiece ▁ to space
static void bpe_decode_piece(BPETokenizer *tok, int token_id, char *out, int out_size) {
const char *piece = bpe_decode(tok, token_id);
int j = 0;
const char *p = piece;
while (*p && j < out_size - 1) {
if (is_sp_space(p) && p[3] == '\0' || is_sp_space(p)) {
out[j++] = ' ';
p += 3; // skip 3-byte UTF-8 sequence
} else {
out[j++] = *p++;
}
}
out[j] = '\0';
}
// ----------------------------------------------------------------------------
// Symbol list for BPE encoding
typedef struct BPESym {
int token_id; // -1 if raw byte
char text[BPE_MAX_TOKEN];
int prev, next; // doubly linked list indices
} BPESym;
// Find token id by exact string match
static int bpe_find_token(BPETokenizer *tok, const char *s) {
// linear scan; for large vocabs a hash table would be better
for (int i = 0; i < tok->vocab_size; i++) {
if (strcmp(tok->tokens[i], s) == 0) return i;
}
return -1;
}
// Find best merge pair by score (SentencePiece style)
static int bpe_find_best_pair(BPETokenizer *tok, BPESym *syms, int n, int *out_i) {
float best_score = -1e38f;
int best_idx = -1;
int merged_id = -1;
for (int i = 0; i < n - 1; ) {
if (syms[i].next < 0) { i++; continue; }
int j = syms[i].next;
// try merging syms[i] and syms[j]
char merged[BPE_MAX_TOKEN * 2];
int mlen = snprintf(merged, sizeof(merged), "%s%s", syms[i].text, syms[j].text);
if (mlen < (int)sizeof(merged)) {
int tid = bpe_find_token(tok, merged);
if (tid >= 0) {
float score = tok->scores ? tok->scores[tid] : 0.0f;
if (score > best_score) {
best_score = score;
best_idx = i;
merged_id = tid;
}
}
}
i = j;
}
*out_i = best_idx;
return merged_id;
}
// BPE encode using merge rules (tiktoken-style)
static int bpe_merge_rank(BPETokenizer *tok, const char *left, const char *right) {
for (int i = 0; i < tok->n_merges; i++) {
if (strcmp(tok->merge_left[i], left) == 0 &&
strcmp(tok->merge_right[i], right) == 0) {
return i;
}
}
return INT32_MAX;
}
// Main encode function
// Returns number of tokens. out_tokens must have capacity for len+2 tokens.
static int bpe_encode(BPETokenizer *tok, const char *text, int add_bos,
int *out_tokens, int max_tokens) {
int n_out = 0;
if (add_bos && n_out < max_tokens) {
out_tokens[n_out++] = tok->bos_id;
}
size_t text_len = strlen(text);
if (text_len == 0) return n_out;
// SentencePiece: prepend ▁ (U+2581 = E2 96 81) to represent leading space
// We handle this by checking if first token has ▁ prefix in vocab
// Build initial symbol list from UTF-8 characters
// (or byte-level for unknown chars)
// We use a simple array with a linked list overlay
int max_syms = (int)(text_len + 4);
BPESym *syms = (BPESym *)calloc(max_syms, sizeof(BPESym));
if (!syms) { fprintf(stderr, "BPE: OOM\n"); exit(EXIT_FAILURE); }
int n_syms = 0;
// SentencePiece: treat text as having a space prepended
// Check if vocab uses ▁ convention
int sp_style = (bpe_find_token(tok, "\xE2\x96\x81") >= 0 ||
bpe_find_token(tok, "\xE2\x96\x81 ") >= 0);
if (sp_style) {
// Prepend ▁ to first character
const char *p = text;
int first = 1;
while (*p && n_syms < max_syms) {
int clen = utf8_char_len((unsigned char)*p);
char buf[BPE_MAX_TOKEN];
int blen = 0;
if (first) {
// prepend ▁ (3 bytes)
buf[0] = '\xE2'; buf[1] = '\x96'; buf[2] = '\x81';
blen = 3;
first = 0;
}
for (int k = 0; k < clen && blen < (int)sizeof(buf) - 1; k++) {
buf[blen++] = p[k];
}
buf[blen] = '\0';
int tid = bpe_find_token(tok, buf);
if (tid < 0) {
// try without ▁ prefix
char buf2[BPE_MAX_TOKEN];
strncpy(buf2, buf + (blen > clen ? 3 : 0), sizeof(buf2));
tid = bpe_find_token(tok, buf2);
if (tid < 0 && blen == 1) {
// byte fallback
tid = tok->byte_token[(unsigned char)buf[0]];
}
}
syms[n_syms].token_id = tid;
strncpy(syms[n_syms].text, buf, BPE_MAX_TOKEN - 1);
syms[n_syms].prev = n_syms - 1;
syms[n_syms].next = n_syms + 1;
n_syms++;
p += clen;
}
} else {
// Byte-level BPE (GPT-2 style or tiktoken)
const char *p = text;
while (*p && n_syms < max_syms) {
int clen = utf8_char_len((unsigned char)*p);
char buf[BPE_MAX_TOKEN];
int blen = 0;
for (int k = 0; k < clen && blen < (int)sizeof(buf) - 1; k++) {
buf[blen++] = p[k];
}
buf[blen] = '\0';
int tid = bpe_find_token(tok, buf);
if (tid < 0) {
// byte fallback
for (int k = 0; k < blen && n_syms < max_syms; k++) {
syms[n_syms].token_id = tok->byte_token[(unsigned char)buf[k]];
syms[n_syms].text[0] = buf[k];
syms[n_syms].text[1] = '\0';
syms[n_syms].prev = n_syms - 1;
syms[n_syms].next = n_syms + 1;
n_syms++;
}
p += clen;
continue;
}
syms[n_syms].token_id = tid;
strncpy(syms[n_syms].text, buf, BPE_MAX_TOKEN - 1);
syms[n_syms].prev = n_syms - 1;
syms[n_syms].next = n_syms + 1;
n_syms++;
p += clen;
}
}
if (n_syms > 0) {
syms[0].prev = -1;
syms[n_syms - 1].next = -1;
}
// BPE merge loop
if (!tok->has_merges) {
// SentencePiece score-based merges
while (1) {
int best_i;
int merged_id = bpe_find_best_pair(tok, syms, n_syms, &best_i);
if (merged_id < 0 || best_i < 0) break;
int j = syms[best_i].next;
// merge best_i and j into best_i
char merged[BPE_MAX_TOKEN * 2];
snprintf(merged, sizeof(merged), "%s%s", syms[best_i].text, syms[j].text);
strncpy(syms[best_i].text, merged, BPE_MAX_TOKEN - 1);
syms[best_i].token_id = merged_id;
// unlink j
syms[best_i].next = syms[j].next;
if (syms[j].next >= 0) syms[syms[j].next].prev = best_i;
}
} else {
// Explicit merge rules: find lowest-rank pair, merge, repeat
while (1) {
int best_rank = INT32_MAX;
int best_i = -1;
int best_j = -1;
for (int i = 0; i >= 0 && syms[i].next >= 0; ) {
int j = syms[i].next;
if (j < 0) break;
int rank = bpe_merge_rank(tok, syms[i].text, syms[j].text);
if (rank < best_rank) {
best_rank = rank;
best_i = i;
best_j = j;
}
i = j;
}
if (best_i < 0 || best_rank == INT32_MAX) break;
// merge best_i and best_j
char merged[BPE_MAX_TOKEN * 2];
snprintf(merged, sizeof(merged), "%s%s", syms[best_i].text, syms[best_j].text);
strncpy(syms[best_i].text, merged, BPE_MAX_TOKEN - 1);
int tid = bpe_find_token(tok, merged);
syms[best_i].token_id = (tid >= 0) ? tid : -1;
syms[best_i].next = syms[best_j].next;
if (syms[best_j].next >= 0) syms[syms[best_j].next].prev = best_i;
}
}
// Collect tokens from linked list
for (int i = 0; i >= 0 && n_out < max_tokens; ) {
int tid = syms[i].token_id;
if (tid >= 0) {
out_tokens[n_out++] = tid;
} else {
// byte fallback
const char *t = syms[i].text;
while (*t && n_out < max_tokens) {
int bt = tok->byte_token[(unsigned char)*t];
if (bt >= 0) out_tokens[n_out++] = bt;
t++;
}
}
if (syms[i].next < 0) break;
i = syms[i].next;
}
free(syms);
return n_out;
}
// ----------------------------------------------------------------------------
// Free
static void bpe_tokenizer_free(BPETokenizer *tok) {
for (int i = 0; i < tok->vocab_size; i++) free(tok->tokens[i]);
free(tok->tokens);
free(tok->scores);
free(tok->token_type);
if (tok->has_merges) {
for (int i = 0; i < tok->n_merges; i++) {
free(tok->merge_left[i]);
free(tok->merge_right[i]);
}
free(tok->merge_left);
free(tok->merge_right);
}
}
#endif // BPE_TOKENIZER_H

460
llmc/gguf.h Normal file
View file

@ -0,0 +1,460 @@
/*
GGUF file format parser.
Supports GGUF v3, tensor types F32/F16/Q4_0/Q8_0.
*/
#ifndef GGUF_H
#define GGUF_H
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdint.h>
#include <math.h>
// ----------------------------------------------------------------------------
// GGUF types
#define GGUF_MAGIC 0x46554747 // "GGUF"
#define GGUF_ALIGNMENT 32
typedef enum {
GGUF_TYPE_UINT8 = 0,
GGUF_TYPE_INT8 = 1,
GGUF_TYPE_UINT16 = 2,
GGUF_TYPE_INT16 = 3,
GGUF_TYPE_UINT32 = 4,
GGUF_TYPE_INT32 = 5,
GGUF_TYPE_FLOAT32 = 6,
GGUF_TYPE_BOOL = 7,
GGUF_TYPE_STRING = 8,
GGUF_TYPE_ARRAY = 9,
GGUF_TYPE_UINT64 = 10,
GGUF_TYPE_INT64 = 11,
GGUF_TYPE_FLOAT64 = 12,
} GGUFValueType;
typedef enum {
GGML_TYPE_F32 = 0,
GGML_TYPE_F16 = 1,
GGML_TYPE_Q4_0 = 2,
GGML_TYPE_Q4_1 = 3,
GGML_TYPE_Q8_0 = 8,
} GGMLType;
#define GGML_TYPE_COUNT 16
static const int ggml_blk_size[GGML_TYPE_COUNT] = {
[GGML_TYPE_F32] = 1,
[GGML_TYPE_F16] = 1,
[GGML_TYPE_Q4_0] = 32,
[GGML_TYPE_Q4_1] = 32,
[GGML_TYPE_Q8_0] = 32,
};
static const int ggml_type_size[GGML_TYPE_COUNT] = {
[GGML_TYPE_F32] = 4,
[GGML_TYPE_F16] = 2,
[GGML_TYPE_Q4_0] = 18, // 2 (f16 scale) + 16 (32 x 4bit)
[GGML_TYPE_Q4_1] = 20, // 2 + 2 + 16
[GGML_TYPE_Q8_0] = 34, // 2 (f16 scale) + 32 (int8 quants)
};
// ----------------------------------------------------------------------------
// GGUF value (tagged union)
typedef struct GGUFValue GGUFValue;
typedef struct {
GGUFValueType type;
uint64_t count;
GGUFValue *items;
} GGUFArray;
struct GGUFValue {
GGUFValueType type;
union {
uint8_t u8;
int8_t i8;
uint16_t u16;
int16_t i16;
uint32_t u32;
int32_t i32;
float f32;
uint8_t bool_;
uint64_t u64;
int64_t i64;
double f64;
char *str; // null-terminated, heap allocated
GGUFArray arr;
};
};
// ----------------------------------------------------------------------------
// GGUF metadata key-value pair
typedef struct {
char *key; // null-terminated
GGUFValue value;
} GGUFMetaKV;
// ----------------------------------------------------------------------------
// GGUF tensor info
#define GGUF_MAX_DIMS 4
typedef struct {
char *name;
uint32_t n_dims;
uint64_t dims[GGUF_MAX_DIMS];
GGMLType type;
uint64_t offset; // offset from start of tensor data section
void *data; // pointer into mmap'd or malloc'd buffer
size_t data_size;
} GGUFTensor;
// ----------------------------------------------------------------------------
// GGUF context
typedef struct {
uint32_t version;
uint64_t n_tensors;
uint64_t n_kv;
GGUFMetaKV *kv;
GGUFTensor *tensors;
// raw file data (malloc'd)
void *data;
size_t data_size;
size_t data_offset; // offset in file where tensor data begins
} GGUFContext;
// ----------------------------------------------------------------------------
// Internal read helpers
static void gguf_read_check(FILE *f, void *dst, size_t n, const char *label) {
if (fread(dst, 1, n, f) != n) {
fprintf(stderr, "GGUF: failed to read %s\n", label);
exit(EXIT_FAILURE);
}
}
static char *gguf_read_string(FILE *f) {
uint64_t len;
gguf_read_check(f, &len, sizeof(len), "string length");
char *s = (char *)malloc(len + 1);
if (!s) { fprintf(stderr, "GGUF: OOM\n"); exit(EXIT_FAILURE); }
gguf_read_check(f, s, len, "string data");
s[len] = '\0';
return s;
}
static void gguf_read_value(FILE *f, GGUFValue *v, GGUFValueType type);
static void gguf_read_array(FILE *f, GGUFArray *arr) {
uint32_t elem_type;
gguf_read_check(f, &elem_type, sizeof(elem_type), "array elem type");
arr->type = (GGUFValueType)elem_type;
gguf_read_check(f, &arr->count, sizeof(arr->count), "array count");
arr->items = (GGUFValue *)calloc(arr->count, sizeof(GGUFValue));
if (!arr->items) { fprintf(stderr, "GGUF: OOM\n"); exit(EXIT_FAILURE); }
for (uint64_t i = 0; i < arr->count; i++) {
gguf_read_value(f, &arr->items[i], arr->type);
}
}
static void gguf_read_value(FILE *f, GGUFValue *v, GGUFValueType type) {
v->type = type;
switch (type) {
case GGUF_TYPE_UINT8: gguf_read_check(f, &v->u8, 1, "u8"); break;
case GGUF_TYPE_INT8: gguf_read_check(f, &v->i8, 1, "i8"); break;
case GGUF_TYPE_UINT16: gguf_read_check(f, &v->u16, 2, "u16"); break;
case GGUF_TYPE_INT16: gguf_read_check(f, &v->i16, 2, "i16"); break;
case GGUF_TYPE_UINT32: gguf_read_check(f, &v->u32, 4, "u32"); break;
case GGUF_TYPE_INT32: gguf_read_check(f, &v->i32, 4, "i32"); break;
case GGUF_TYPE_FLOAT32: gguf_read_check(f, &v->f32, 4, "f32"); break;
case GGUF_TYPE_BOOL: gguf_read_check(f, &v->bool_,1, "bool"); break;
case GGUF_TYPE_UINT64: gguf_read_check(f, &v->u64, 8, "u64"); break;
case GGUF_TYPE_INT64: gguf_read_check(f, &v->i64, 8, "i64"); break;
case GGUF_TYPE_FLOAT64: gguf_read_check(f, &v->f64, 8, "f64"); break;
case GGUF_TYPE_STRING: v->str = gguf_read_string(f); break;
case GGUF_TYPE_ARRAY: gguf_read_array(f, &v->arr); break;
default:
fprintf(stderr, "GGUF: unknown value type %d\n", type);
exit(EXIT_FAILURE);
}
}
// ----------------------------------------------------------------------------
// Main load function
static GGUFContext *gguf_load(const char *path) {
FILE *f = fopen(path, "rb");
if (!f) {
fprintf(stderr, "GGUF: cannot open '%s'\n", path);
exit(EXIT_FAILURE);
}
// magic
uint32_t magic;
gguf_read_check(f, &magic, 4, "magic");
if (magic != GGUF_MAGIC) {
fprintf(stderr, "GGUF: bad magic 0x%08X (expected 0x%08X)\n", magic, GGUF_MAGIC);
exit(EXIT_FAILURE);
}
GGUFContext *ctx = (GGUFContext *)calloc(1, sizeof(GGUFContext));
if (!ctx) { fprintf(stderr, "GGUF: OOM\n"); exit(EXIT_FAILURE); }
gguf_read_check(f, &ctx->version, 4, "version");
gguf_read_check(f, &ctx->n_tensors, 8, "n_tensors");
gguf_read_check(f, &ctx->n_kv, 8, "n_kv");
// metadata key-value pairs
ctx->kv = (GGUFMetaKV *)calloc(ctx->n_kv, sizeof(GGUFMetaKV));
if (!ctx->kv) { fprintf(stderr, "GGUF: OOM\n"); exit(EXIT_FAILURE); }
for (uint64_t i = 0; i < ctx->n_kv; i++) {
ctx->kv[i].key = gguf_read_string(f);
uint32_t vtype;
gguf_read_check(f, &vtype, 4, "kv value type");
gguf_read_value(f, &ctx->kv[i].value, (GGUFValueType)vtype);
}
// tensor info
ctx->tensors = (GGUFTensor *)calloc(ctx->n_tensors, sizeof(GGUFTensor));
if (!ctx->tensors) { fprintf(stderr, "GGUF: OOM\n"); exit(EXIT_FAILURE); }
for (uint64_t i = 0; i < ctx->n_tensors; i++) {
GGUFTensor *t = &ctx->tensors[i];
t->name = gguf_read_string(f);
gguf_read_check(f, &t->n_dims, 4, "n_dims");
for (uint32_t d = 0; d < t->n_dims; d++) {
gguf_read_check(f, &t->dims[d], 8, "dim");
}
uint32_t ttype;
gguf_read_check(f, &ttype, 4, "tensor type");
t->type = (GGMLType)ttype;
gguf_read_check(f, &t->offset, 8, "offset");
}
// tensor data starts at next alignment boundary
long header_end = ftell(f);
long align_end = ((header_end + GGUF_ALIGNMENT - 1) / GGUF_ALIGNMENT) * GGUF_ALIGNMENT;
ctx->data_offset = (size_t)align_end;
// compute total tensor data size
uint64_t max_end = 0;
for (uint64_t i = 0; i < ctx->n_tensors; i++) {
GGUFTensor *t = &ctx->tensors[i];
// compute number of elements
uint64_t ne = 1;
for (uint32_t d = 0; d < t->n_dims; d++) ne *= t->dims[d];
// compute bytes
int bsz = ggml_blk_size[t->type];
int tsz = ggml_type_size[t->type];
uint64_t n_blocks = (ne + bsz - 1) / bsz;
t->data_size = (size_t)(n_blocks * tsz);
uint64_t end = t->offset + t->data_size;
if (end > max_end) max_end = end;
}
ctx->data_size = (size_t)max_end;
// load tensor data
ctx->data = malloc(ctx->data_size);
if (!ctx->data && ctx->data_size > 0) {
fprintf(stderr, "GGUF: OOM for tensor data (%zu bytes)\n", ctx->data_size);
exit(EXIT_FAILURE);
}
fseek(f, (long)ctx->data_offset, SEEK_SET);
size_t got = fread(ctx->data, 1, ctx->data_size, f);
if (got != ctx->data_size) {
fprintf(stderr, "GGUF: partial tensor data read (%zu / %zu)\n", got, ctx->data_size);
exit(EXIT_FAILURE);
}
fclose(f);
// set data pointers
for (uint64_t i = 0; i < ctx->n_tensors; i++) {
GGUFTensor *t = &ctx->tensors[i];
t->data = (char *)ctx->data + t->offset;
}
return ctx;
}
// ----------------------------------------------------------------------------
// Metadata lookup helpers
static GGUFValue *gguf_find_kv(GGUFContext *ctx, const char *key) {
for (uint64_t i = 0; i < ctx->n_kv; i++) {
if (strcmp(ctx->kv[i].key, key) == 0) {
return &ctx->kv[i].value;
}
}
return NULL;
}
static const char *gguf_get_str(GGUFContext *ctx, const char *key, const char *def) {
GGUFValue *v = gguf_find_kv(ctx, key);
if (!v || v->type != GGUF_TYPE_STRING) return def;
return v->str;
}
static uint64_t gguf_get_u64(GGUFContext *ctx, const char *key, uint64_t def) {
GGUFValue *v = gguf_find_kv(ctx, key);
if (!v) return def;
switch (v->type) {
case GGUF_TYPE_UINT8: return v->u8;
case GGUF_TYPE_INT8: return (uint64_t)v->i8;
case GGUF_TYPE_UINT16: return v->u16;
case GGUF_TYPE_INT16: return (uint64_t)v->i16;
case GGUF_TYPE_UINT32: return v->u32;
case GGUF_TYPE_INT32: return (uint64_t)v->i32;
case GGUF_TYPE_UINT64: return v->u64;
case GGUF_TYPE_INT64: return (uint64_t)v->i64;
default: return def;
}
}
static float gguf_get_f32(GGUFContext *ctx, const char *key, float def) {
GGUFValue *v = gguf_find_kv(ctx, key);
if (!v) return def;
if (v->type == GGUF_TYPE_FLOAT32) return v->f32;
if (v->type == GGUF_TYPE_FLOAT64) return (float)v->f64;
return def;
}
static GGUFTensor *gguf_find_tensor(GGUFContext *ctx, const char *name) {
for (uint64_t i = 0; i < ctx->n_tensors; i++) {
if (strcmp(ctx->tensors[i].name, name) == 0) {
return &ctx->tensors[i];
}
}
return NULL;
}
// ----------------------------------------------------------------------------
// F16 -> F32 conversion
static inline float f16_to_f32(uint16_t h) {
uint32_t sign = (uint32_t)(h & 0x8000) << 16;
uint32_t exponent = (h >> 10) & 0x1F;
uint32_t mantissa = h & 0x3FF;
uint32_t result;
if (exponent == 0) {
if (mantissa == 0) {
result = sign;
} else {
// denormal
exponent = 1;
while (!(mantissa & 0x400)) { mantissa <<= 1; exponent--; }
mantissa &= 0x3FF;
result = sign | ((exponent + 127 - 15) << 23) | (mantissa << 13);
}
} else if (exponent == 31) {
// inf or nan
result = sign | 0x7F800000 | (mantissa << 13);
} else {
result = sign | ((exponent + 127 - 15) << 23) | (mantissa << 13);
}
float f;
memcpy(&f, &result, 4);
return f;
}
// ----------------------------------------------------------------------------
// Dequantize a tensor row to float
// Returns number of elements written to out.
// out must have space for `ne` floats.
static void gguf_dequantize_row(GGUFTensor *t, int64_t row, float *out) {
uint64_t row_size = t->dims[0]; // elements per row (first dim is fastest)
uint8_t *src = (uint8_t *)t->data;
if (t->type == GGML_TYPE_F32) {
float *p = (float *)src + row * row_size;
memcpy(out, p, row_size * sizeof(float));
return;
}
if (t->type == GGML_TYPE_F16) {
uint16_t *p = (uint16_t *)src + row * row_size;
for (uint64_t i = 0; i < row_size; i++) {
out[i] = f16_to_f32(p[i]);
}
return;
}
int blk_size = ggml_blk_size[t->type];
int blk_bytes = ggml_type_size[t->type];
uint64_t n_blocks_per_row = (row_size + blk_size - 1) / blk_size;
uint8_t *row_data = src + row * n_blocks_per_row * blk_bytes;
if (t->type == GGML_TYPE_Q8_0) {
// block = [f16 scale (2 bytes)] [32 x int8 (32 bytes)]
for (uint64_t b = 0; b < n_blocks_per_row; b++) {
uint8_t *blk = row_data + b * 34;
uint16_t scale_bits;
memcpy(&scale_bits, blk, 2);
float scale = f16_to_f32(scale_bits);
int8_t *quants = (int8_t *)(blk + 2);
uint64_t base = b * 32;
for (int k = 0; k < 32 && (base + k) < row_size; k++) {
out[base + k] = scale * (float)quants[k];
}
}
return;
}
if (t->type == GGML_TYPE_Q4_0) {
// block = [f16 scale (2 bytes)] [16 bytes = 32 x 4-bit quants]
for (uint64_t b = 0; b < n_blocks_per_row; b++) {
uint8_t *blk = row_data + b * 18;
uint16_t scale_bits;
memcpy(&scale_bits, blk, 2);
float scale = f16_to_f32(scale_bits);
uint8_t *quants = blk + 2;
uint64_t base = b * 32;
for (int k = 0; k < 16 && (base + k * 2) < row_size; k++) {
uint8_t byte = quants[k];
int lo = (int)(byte & 0xF) - 8;
int hi = (int)(byte >> 4) - 8;
out[base + k * 2] = scale * (float)lo;
out[base + k * 2 + 1] = scale * (float)hi;
}
}
return;
}
fprintf(stderr, "GGUF: unsupported tensor type %d for dequantization\n", t->type);
exit(EXIT_FAILURE);
}
// ----------------------------------------------------------------------------
// Free
static void gguf_value_free(GGUFValue *v) {
if (v->type == GGUF_TYPE_STRING) { free(v->str); }
else if (v->type == GGUF_TYPE_ARRAY) {
for (uint64_t i = 0; i < v->arr.count; i++) {
gguf_value_free(&v->arr.items[i]);
}
free(v->arr.items);
}
}
static void gguf_free(GGUFContext *ctx) {
for (uint64_t i = 0; i < ctx->n_kv; i++) {
free(ctx->kv[i].key);
gguf_value_free(&ctx->kv[i].value);
}
free(ctx->kv);
for (uint64_t i = 0; i < ctx->n_tensors; i++) {
free(ctx->tensors[i].name);
}
free(ctx->tensors);
free(ctx->data);
free(ctx);
}
#endif // GGUF_H

916
serve.c Normal file
View file

@ -0,0 +1,916 @@
/*
llm-serve: LLaMA inference server with HTTP API.
Loads GGUF model files, serves OpenAI-compatible /v1/completions endpoint.
Usage:
./llm-serve -m model.gguf [-p port] [-c context_size] [-t threads]
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdint.h>
#include <math.h>
#include <time.h>
#include <signal.h>
#include <unistd.h>
#include <fcntl.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#ifdef _OPENMP
#include <omp.h>
#endif
#include "llmc/gguf.h"
#include "llmc/bpe_tokenizer.h"
#include "llmc/sampler.h"
// ============================================================================
// LLaMA Model Config
typedef struct {
int vocab_size;
int embed_dim; // embedding_length / n_embd
int n_layers;
int n_heads; // Q heads
int n_kv_heads; // KV heads (GQA)
int ffn_dim; // feed_forward_length
int context_len;
float rope_theta; // rope.freq_base
float rms_eps; // attention.layer_norm_rms_epsilon
int head_dim; // embed_dim / n_heads
} LlamaConfig;
// ============================================================================
// LLaMA Model Weights (pointers into GGUF data, dequantized lazily)
typedef struct {
// token embedding table [vocab_size, embed_dim]
GGUFTensor *token_embd;
// per-layer
GGUFTensor **attn_norm; // [n_layers] RMSNorm weight [embed_dim]
GGUFTensor **attn_q; // [n_layers] [n_heads * head_dim, embed_dim]
GGUFTensor **attn_k; // [n_layers] [n_kv_heads * head_dim, embed_dim]
GGUFTensor **attn_v; // [n_layers] [n_kv_heads * head_dim, embed_dim]
GGUFTensor **attn_out; // [n_layers] [embed_dim, n_heads * head_dim]
GGUFTensor **ffn_norm; // [n_layers] RMSNorm weight [embed_dim]
GGUFTensor **ffn_gate; // [n_layers] [ffn_dim, embed_dim]
GGUFTensor **ffn_up; // [n_layers] [ffn_dim, embed_dim]
GGUFTensor **ffn_down; // [n_layers] [embed_dim, ffn_dim]
// output
GGUFTensor *output_norm; // [embed_dim]
GGUFTensor *output; // [vocab_size, embed_dim] (may be NULL if tied)
} LlamaWeights;
// ============================================================================
// KV Cache
typedef struct {
float *k; // [n_layers, context_len, n_kv_heads, head_dim]
float *v; // [n_layers, context_len, n_kv_heads, head_dim]
int n_layers, context_len, n_kv_heads, head_dim;
} KVCache;
static KVCache kvcache_alloc(int n_layers, int ctx_len, int n_kv_heads, int head_dim) {
KVCache c;
c.n_layers = n_layers;
c.context_len = ctx_len;
c.n_kv_heads = n_kv_heads;
c.head_dim = head_dim;
size_t sz = (size_t)n_layers * ctx_len * n_kv_heads * head_dim;
c.k = (float *)calloc(sz, sizeof(float));
c.v = (float *)calloc(sz, sizeof(float));
if (!c.k || !c.v) { fprintf(stderr, "OOM for KV cache\n"); exit(EXIT_FAILURE); }
return c;
}
static float *kvcache_k(KVCache *c, int layer, int pos) {
return c->k + ((size_t)layer * c->context_len + pos) * c->n_kv_heads * c->head_dim;
}
static float *kvcache_v(KVCache *c, int layer, int pos) {
return c->v + ((size_t)layer * c->context_len + pos) * c->n_kv_heads * c->head_dim;
}
// ============================================================================
// Run state (activations)
typedef struct {
float *x; // [embed_dim] current residual
float *xb; // [embed_dim] scratch
float *xb2; // [embed_dim] scratch
float *hb; // [ffn_dim]
float *hb2; // [ffn_dim]
float *q; // [n_heads * head_dim]
float *k; // [n_kv_heads * head_dim]
float *v; // [n_kv_heads * head_dim]
float *att; // [n_heads * context_len]
float *logits; // [vocab_size]
} RunState;
static RunState runstate_alloc(LlamaConfig *cfg) {
RunState s;
int d = cfg->embed_dim;
int ff = cfg->ffn_dim;
int qd = cfg->n_heads * cfg->head_dim;
int kvd = cfg->n_kv_heads * cfg->head_dim;
s.x = (float *)calloc(d, sizeof(float));
s.xb = (float *)calloc(d, sizeof(float));
s.xb2 = (float *)calloc(d, sizeof(float));
s.hb = (float *)calloc(ff, sizeof(float));
s.hb2 = (float *)calloc(ff, sizeof(float));
s.q = (float *)calloc(qd, sizeof(float));
s.k = (float *)calloc(kvd, sizeof(float));
s.v = (float *)calloc(kvd, sizeof(float));
s.att = (float *)calloc(cfg->n_heads * cfg->context_len, sizeof(float));
s.logits = (float *)calloc(cfg->vocab_size, sizeof(float));
if (!s.x || !s.xb || !s.xb2 || !s.hb || !s.hb2 ||
!s.q || !s.k || !s.v || !s.att || !s.logits) {
fprintf(stderr, "OOM for run state\n"); exit(EXIT_FAILURE);
}
return s;
}
// ============================================================================
// Math primitives
static void rmsnorm(float *out, const float *x, const float *weight, int n, float eps) {
float ss = 0.0f;
for (int i = 0; i < n; i++) ss += x[i] * x[i];
ss = 1.0f / sqrtf(ss / (float)n + eps);
for (int i = 0; i < n; i++) out[i] = weight[i] * (x[i] * ss);
}
static void softmax(float *x, int n) {
float max_val = x[0];
for (int i = 1; i < n; i++) if (x[i] > max_val) max_val = x[i];
float sum = 0.0f;
for (int i = 0; i < n; i++) { x[i] = expf(x[i] - max_val); sum += x[i]; }
for (int i = 0; i < n; i++) x[i] /= sum;
}
static inline float silu(float x) { return x / (1.0f + expf(-x)); }
// Matrix-vector multiply: out[m] = W[m,n] @ in[n]
// W is stored as a GGUFTensor (rows=m, cols=n), dequantized row by row.
static void matvec(float *out, GGUFTensor *W, const float *in, int m, int n) {
float *row = (float *)malloc(n * sizeof(float));
if (!row) { fprintf(stderr, "OOM matvec\n"); exit(EXIT_FAILURE); }
#pragma omp parallel for schedule(dynamic,16)
for (int i = 0; i < m; i++) {
// dequantize row i
float *row_local = (float *)malloc(n * sizeof(float));
gguf_dequantize_row(W, i, row_local);
float dot = 0.0f;
for (int j = 0; j < n; j++) dot += row_local[j] * in[j];
out[i] = dot;
free(row_local);
}
free(row);
}
// ============================================================================
// RoPE
static void rope_apply(float *q, float *k, int pos, LlamaConfig *cfg) {
int hd = cfg->head_dim;
float theta = cfg->rope_theta;
// Q
for (int h = 0; h < cfg->n_heads; h++) {
float *qh = q + h * hd;
for (int i = 0; i < hd / 2; i++) {
float freq = 1.0f / powf(theta, (float)(2 * i) / (float)hd);
float val = (float)pos * freq;
float cos_val = cosf(val), sin_val = sinf(val);
float q0 = qh[2*i], q1 = qh[2*i+1];
qh[2*i] = q0 * cos_val - q1 * sin_val;
qh[2*i+1] = q0 * sin_val + q1 * cos_val;
}
}
// K
for (int h = 0; h < cfg->n_kv_heads; h++) {
float *kh = k + h * hd;
for (int i = 0; i < hd / 2; i++) {
float freq = 1.0f / powf(theta, (float)(2 * i) / (float)hd);
float val = (float)pos * freq;
float cos_val = cosf(val), sin_val = sinf(val);
float k0 = kh[2*i], k1 = kh[2*i+1];
kh[2*i] = k0 * cos_val - k1 * sin_val;
kh[2*i+1] = k0 * sin_val + k1 * cos_val;
}
}
}
// ============================================================================
// Forward pass (single token at position `pos`)
static void llama_forward(LlamaConfig *cfg, LlamaWeights *w, RunState *s,
KVCache *kv, int token_id, int pos) {
int d = cfg->embed_dim;
int hd = cfg->head_dim;
int nh = cfg->n_heads;
int nkv = cfg->n_kv_heads;
int kv_mul = nh / nkv; // GQA expansion factor
int ff = cfg->ffn_dim;
float eps = cfg->rms_eps;
// --- Token embedding ---
// token_embd: [vocab_size, embed_dim] stored as (vocab_size rows, embed_dim cols)
gguf_dequantize_row(w->token_embd, token_id, s->x);
// --- Transformer layers ---
for (int l = 0; l < cfg->n_layers; l++) {
// RMSNorm before attention
float *norm_w = (float *)malloc(d * sizeof(float));
gguf_dequantize_row(w->attn_norm[l], 0, norm_w);
rmsnorm(s->xb, s->x, norm_w, d, eps);
free(norm_w);
// Q, K, V projections
matvec(s->q, w->attn_q[l], s->xb, nh * hd, d);
matvec(s->k, w->attn_k[l], s->xb, nkv * hd, d);
matvec(s->v, w->attn_v[l], s->xb, nkv * hd, d);
// RoPE
rope_apply(s->q, s->k, pos, cfg);
// Store K, V in cache
memcpy(kvcache_k(kv, l, pos), s->k, nkv * hd * sizeof(float));
memcpy(kvcache_v(kv, l, pos), s->v, nkv * hd * sizeof(float));
// Attention: for each Q head
memset(s->xb, 0, d * sizeof(float));
float scale = 1.0f / sqrtf((float)hd);
for (int h = 0; h < nh; h++) {
float *qh = s->q + h * hd;
int kvh = h / kv_mul; // which KV head to use (GQA)
float *att = s->att + h * cfg->context_len;
// Dot product Q with all K in cache [0..pos]
for (int t = 0; t <= pos; t++) {
float *kh = kvcache_k(kv, l, t) + kvh * hd;
float dot = 0.0f;
for (int i = 0; i < hd; i++) dot += qh[i] * kh[i];
att[t] = dot * scale;
}
softmax(att, pos + 1);
// Weighted sum of V
float *xbh = s->xb + h * hd;
memset(xbh, 0, hd * sizeof(float));
for (int t = 0; t <= pos; t++) {
float *vh = kvcache_v(kv, l, t) + kvh * hd;
float a = att[t];
for (int i = 0; i < hd; i++) xbh[i] += a * vh[i];
}
}
// Output projection + residual
matvec(s->xb2, w->attn_out[l], s->xb, d, nh * hd);
for (int i = 0; i < d; i++) s->x[i] += s->xb2[i];
// RMSNorm before FFN
float *fnorm_w = (float *)malloc(d * sizeof(float));
gguf_dequantize_row(w->ffn_norm[l], 0, fnorm_w);
rmsnorm(s->xb, s->x, fnorm_w, d, eps);
free(fnorm_w);
// SwiGLU FFN
matvec(s->hb, w->ffn_gate[l], s->xb, ff, d);
matvec(s->hb2, w->ffn_up[l], s->xb, ff, d);
for (int i = 0; i < ff; i++) s->hb[i] = silu(s->hb[i]) * s->hb2[i];
matvec(s->xb, w->ffn_down[l], s->hb, d, ff);
// Residual
for (int i = 0; i < d; i++) s->x[i] += s->xb[i];
}
// --- Output norm + logits ---
float *onorm_w = (float *)malloc(d * sizeof(float));
gguf_dequantize_row(w->output_norm, 0, onorm_w);
rmsnorm(s->xb, s->x, onorm_w, d, eps);
free(onorm_w);
GGUFTensor *out_W = w->output ? w->output : w->token_embd;
matvec(s->logits, out_W, s->xb, cfg->vocab_size, d);
}
// ============================================================================
// Sampling helpers
static int sample_argmax(const float *logits, int n) {
int best = 0;
for (int i = 1; i < n; i++) if (logits[i] > logits[best]) best = i;
return best;
}
static int sample_topp(const float *logits, int n, float temperature, float top_p,
unsigned long long *rng_state) {
if (temperature <= 0.0f) return sample_argmax(logits, n);
// Temperature scaling + softmax
float *probs = (float *)malloc(n * sizeof(float));
float max_l = logits[0];
for (int i = 1; i < n; i++) if (logits[i] > max_l) max_l = logits[i];
float sum = 0.0f;
for (int i = 0; i < n; i++) { probs[i] = expf((logits[i] - max_l) / temperature); sum += probs[i]; }
for (int i = 0; i < n; i++) probs[i] /= sum;
// Top-p nucleus sampling
// Sort by prob descending (simple insertion sort for small n, but vocab can be large)
// For efficiency: find cumulative sum threshold
float coin = random_f32(rng_state);
if (top_p >= 1.0f) {
// just sample from full distribution
float cdf = 0.0f;
for (int i = 0; i < n; i++) {
cdf += probs[i];
if (coin < cdf) { free(probs); return i; }
}
free(probs);
return n - 1;
}
// Build sorted index array (selection sort subset up to top_p)
int *idx = (int *)malloc(n * sizeof(int));
for (int i = 0; i < n; i++) idx[i] = i;
float cumsum = 0.0f;
int nucleus_size = 0;
for (int i = 0; i < n - 1; i++) {
// find max in [i, n)
int max_j = i;
for (int j = i + 1; j < n; j++) {
if (probs[idx[j]] > probs[idx[max_j]]) max_j = j;
}
int tmp = idx[i]; idx[i] = idx[max_j]; idx[max_j] = tmp;
cumsum += probs[idx[i]];
nucleus_size = i + 1;
if (cumsum >= top_p) break;
}
if (nucleus_size == 0) nucleus_size = 1;
// Re-normalize nucleus
float nsum = 0.0f;
for (int i = 0; i < nucleus_size; i++) nsum += probs[idx[i]];
float cdf = 0.0f;
int result = idx[nucleus_size - 1];
for (int i = 0; i < nucleus_size; i++) {
cdf += probs[idx[i]] / nsum;
if (coin < cdf) { result = idx[i]; break; }
}
free(probs);
free(idx);
return result;
}
// ============================================================================
// Model loading
typedef struct {
LlamaConfig cfg;
LlamaWeights weights;
KVCache kv;
RunState state;
BPETokenizer tokenizer;
GGUFContext *gguf;
unsigned long long rng_state;
} LlamaModel;
static GGUFTensor *require_tensor(GGUFContext *ctx, const char *name) {
GGUFTensor *t = gguf_find_tensor(ctx, name);
if (!t) {
fprintf(stderr, "WARN: tensor '%s' not found\n", name);
}
return t;
}
static LlamaModel *llama_load(const char *path, int context_override) {
fprintf(stderr, "Loading GGUF model: %s\n", path);
GGUFContext *ctx = gguf_load(path);
const char *arch = gguf_get_str(ctx, "general.architecture", "llama");
char key[256];
LlamaConfig cfg;
memset(&cfg, 0, sizeof(cfg));
#define ARCH_KEY(k) (snprintf(key, sizeof(key), "%s.%s", arch, k), key)
cfg.vocab_size = (int)gguf_get_u64(ctx, ARCH_KEY("vocab_size"), 32000);
cfg.embed_dim = (int)gguf_get_u64(ctx, ARCH_KEY("embedding_length"), 4096);
cfg.n_layers = (int)gguf_get_u64(ctx, ARCH_KEY("block_count"), 32);
cfg.n_heads = (int)gguf_get_u64(ctx, ARCH_KEY("attention.head_count"), 32);
cfg.n_kv_heads = (int)gguf_get_u64(ctx, ARCH_KEY("attention.head_count_kv"), cfg.n_heads);
cfg.ffn_dim = (int)gguf_get_u64(ctx, ARCH_KEY("feed_forward_length"), 11008);
cfg.context_len = (int)gguf_get_u64(ctx, ARCH_KEY("context_length"), 2048);
cfg.rope_theta = gguf_get_f32(ctx, ARCH_KEY("rope.freq_base"), 10000.0f);
cfg.rms_eps = gguf_get_f32(ctx, ARCH_KEY("attention.layer_norm_rms_epsilon"), 1e-5f);
cfg.head_dim = cfg.embed_dim / cfg.n_heads;
if (context_override > 0 && context_override < cfg.context_len)
cfg.context_len = context_override;
// Also try vocab size from tokenizer
{
GGUFValue *tv = gguf_find_kv(ctx, "tokenizer.ggml.tokens");
if (tv && tv->type == GGUF_TYPE_ARRAY && (int)tv->arr.count > cfg.vocab_size)
cfg.vocab_size = (int)tv->arr.count;
}
fprintf(stderr, " arch=%s layers=%d heads=%d kv_heads=%d embed=%d ffn=%d ctx=%d\n",
arch, cfg.n_layers, cfg.n_heads, cfg.n_kv_heads,
cfg.embed_dim, cfg.ffn_dim, cfg.context_len);
// Load weights
LlamaWeights w;
memset(&w, 0, sizeof(w));
w.token_embd = require_tensor(ctx, "token_embd.weight");
w.output_norm = require_tensor(ctx, "output_norm.weight");
w.output = gguf_find_tensor(ctx, "output.weight"); // may be NULL (tied)
w.attn_norm = (GGUFTensor **)calloc(cfg.n_layers, sizeof(GGUFTensor *));
w.attn_q = (GGUFTensor **)calloc(cfg.n_layers, sizeof(GGUFTensor *));
w.attn_k = (GGUFTensor **)calloc(cfg.n_layers, sizeof(GGUFTensor *));
w.attn_v = (GGUFTensor **)calloc(cfg.n_layers, sizeof(GGUFTensor *));
w.attn_out = (GGUFTensor **)calloc(cfg.n_layers, sizeof(GGUFTensor *));
w.ffn_norm = (GGUFTensor **)calloc(cfg.n_layers, sizeof(GGUFTensor *));
w.ffn_gate = (GGUFTensor **)calloc(cfg.n_layers, sizeof(GGUFTensor *));
w.ffn_up = (GGUFTensor **)calloc(cfg.n_layers, sizeof(GGUFTensor *));
w.ffn_down = (GGUFTensor **)calloc(cfg.n_layers, sizeof(GGUFTensor *));
for (int l = 0; l < cfg.n_layers; l++) {
snprintf(key, sizeof(key), "blk.%d.attn_norm.weight", l);
w.attn_norm[l] = require_tensor(ctx, key);
snprintf(key, sizeof(key), "blk.%d.attn_q.weight", l);
w.attn_q[l] = require_tensor(ctx, key);
snprintf(key, sizeof(key), "blk.%d.attn_k.weight", l);
w.attn_k[l] = require_tensor(ctx, key);
snprintf(key, sizeof(key), "blk.%d.attn_v.weight", l);
w.attn_v[l] = require_tensor(ctx, key);
snprintf(key, sizeof(key), "blk.%d.attn_output.weight", l);
w.attn_out[l] = require_tensor(ctx, key);
snprintf(key, sizeof(key), "blk.%d.ffn_norm.weight", l);
w.ffn_norm[l] = require_tensor(ctx, key);
snprintf(key, sizeof(key), "blk.%d.ffn_gate.weight", l);
w.ffn_gate[l] = require_tensor(ctx, key);
snprintf(key, sizeof(key), "blk.%d.ffn_up.weight", l);
w.ffn_up[l] = require_tensor(ctx, key);
snprintf(key, sizeof(key), "blk.%d.ffn_down.weight", l);
w.ffn_down[l] = require_tensor(ctx, key);
}
LlamaModel *model = (LlamaModel *)calloc(1, sizeof(LlamaModel));
model->cfg = cfg;
model->weights = w;
model->gguf = ctx;
model->kv = kvcache_alloc(cfg.n_layers, cfg.context_len, cfg.n_kv_heads, cfg.head_dim);
model->state = runstate_alloc(&cfg);
model->rng_state = (unsigned long long)time(NULL);
bpe_tokenizer_init(&model->tokenizer, ctx);
fprintf(stderr, " vocab_size=%d bos=%d eos=%d\n",
model->tokenizer.vocab_size,
model->tokenizer.bos_id,
model->tokenizer.eos_id);
fprintf(stderr, "Model loaded.\n");
return model;
}
// ============================================================================
// Generate tokens
typedef struct {
int *tokens;
int n_tokens;
float temperature;
float top_p;
int max_new_tokens;
int stream;
int client_fd;
} GenParams;
// Returns generated text (malloc'd). Caller must free.
static char *generate(LlamaModel *model, GenParams *p) {
LlamaConfig *cfg = &model->cfg;
BPETokenizer *tok = &model->tokenizer;
// Reset KV cache
int kv_stride = cfg->context_len * cfg->n_kv_heads * cfg->head_dim;
memset(model->kv.k, 0, (size_t)cfg->n_layers * kv_stride * sizeof(float));
memset(model->kv.v, 0, (size_t)cfg->n_layers * kv_stride * sizeof(float));
size_t out_cap = 4096;
char *out_text = (char *)malloc(out_cap);
if (!out_text) { fprintf(stderr, "OOM\n"); exit(EXIT_FAILURE); }
out_text[0] = '\0';
size_t out_len = 0;
int pos = 0;
int n_prompt = p->n_tokens;
// Prefill prompt tokens
for (int i = 0; i < n_prompt && pos < cfg->context_len; i++, pos++) {
llama_forward(cfg, &model->weights, &model->state, &model->kv, p->tokens[i], pos);
}
int n_gen = 0;
char piece_buf[BPE_MAX_TOKEN];
char sse_buf[BPE_MAX_TOKEN + 128];
while (n_gen < p->max_new_tokens && pos < cfg->context_len) {
// Sample next token
int next;
if (p->temperature <= 0.0f) {
next = sample_argmax(model->state.logits, cfg->vocab_size);
} else {
next = sample_topp(model->state.logits, cfg->vocab_size,
p->temperature, p->top_p, &model->rng_state);
}
// EOS check
if (next == tok->eos_id) break;
// Decode token to text
bpe_decode_piece(tok, next, piece_buf, sizeof(piece_buf));
// Append to output
size_t plen = strlen(piece_buf);
if (out_len + plen + 1 > out_cap) {
out_cap *= 2;
out_text = (char *)realloc(out_text, out_cap);
if (!out_text) { fprintf(stderr, "OOM\n"); exit(EXIT_FAILURE); }
}
memcpy(out_text + out_len, piece_buf, plen);
out_len += plen;
out_text[out_len] = '\0';
// Stream SSE if requested
if (p->stream && p->client_fd >= 0) {
int slen = snprintf(sse_buf, sizeof(sse_buf),
"data: {\"choices\":[{\"text\":\"%s\",\"finish_reason\":null}]}\n\n",
piece_buf);
send(p->client_fd, sse_buf, slen, MSG_NOSIGNAL);
}
// Forward for next token
llama_forward(cfg, &model->weights, &model->state, &model->kv, next, pos);
pos++;
n_gen++;
}
if (p->stream && p->client_fd >= 0) {
const char *done = "data: [DONE]\n\n";
send(p->client_fd, done, strlen(done), MSG_NOSIGNAL);
}
return out_text;
}
// ============================================================================
// JSON helpers
// Simple JSON string escape into dst (returns chars written, not including \0)
static int json_escape(const char *src, char *dst, int dst_cap) {
int j = 0;
for (int i = 0; src[i] && j < dst_cap - 4; i++) {
unsigned char c = (unsigned char)src[i];
if (c == '"') { dst[j++] = '\\'; dst[j++] = '"'; }
else if (c == '\\') { dst[j++] = '\\'; dst[j++] = '\\'; }
else if (c == '\n') { dst[j++] = '\\'; dst[j++] = 'n'; }
else if (c == '\r') { dst[j++] = '\\'; dst[j++] = 'r'; }
else if (c == '\t') { dst[j++] = '\\'; dst[j++] = 't'; }
else if (c < 0x20) { j += snprintf(dst+j, dst_cap-j, "\\u%04X", c); }
else { dst[j++] = (char)c; }
}
dst[j] = '\0';
return j;
}
// Extract string value for a key from JSON (very naive, no nesting)
// Returns pointer into buf (null-terminated), or NULL.
static char *json_get_str(const char *json, const char *key, char *buf, int buf_size) {
char search[128];
snprintf(search, sizeof(search), "\"%s\"", key);
const char *p = strstr(json, search);
if (!p) return NULL;
p += strlen(search);
while (*p == ' ' || *p == ':' || *p == ' ') p++;
if (*p != '"') return NULL;
p++;
int i = 0;
while (*p && *p != '"' && i < buf_size - 1) {
if (*p == '\\' && *(p+1)) {
p++;
if (*p == 'n') buf[i++] = '\n';
else if (*p == 't') buf[i++] = '\t';
else if (*p == 'r') buf[i++] = '\r';
else buf[i++] = *p;
} else {
buf[i++] = *p;
}
p++;
}
buf[i] = '\0';
return buf;
}
static float json_get_float(const char *json, const char *key, float def) {
char search[128];
snprintf(search, sizeof(search), "\"%s\"", key);
const char *p = strstr(json, search);
if (!p) return def;
p += strlen(search);
while (*p == ' ' || *p == ':') p++;
if (*p == '"') return def;
return (float)atof(p);
}
static int json_get_int(const char *json, const char *key, int def) {
char search[128];
snprintf(search, sizeof(search), "\"%s\"", key);
const char *p = strstr(json, search);
if (!p) return def;
p += strlen(search);
while (*p == ' ' || *p == ':') p++;
if (*p == '"') return def;
return atoi(p);
}
static int json_get_bool(const char *json, const char *key, int def) {
char search[128];
snprintf(search, sizeof(search), "\"%s\"", key);
const char *p = strstr(json, search);
if (!p) return def;
p += strlen(search);
while (*p == ' ' || *p == ':') p++;
if (strncmp(p, "true", 4) == 0) return 1;
if (strncmp(p, "false", 5) == 0) return 0;
return def;
}
// ============================================================================
// HTTP server
#define HTTP_MAX_REQUEST (1 << 20) // 1 MB
static void send_response(int fd, int status, const char *content_type,
const char *body, int body_len, int is_stream) {
char header[512];
int hlen;
if (is_stream) {
hlen = snprintf(header, sizeof(header),
"HTTP/1.1 %d OK\r\n"
"Content-Type: text/event-stream\r\n"
"Cache-Control: no-cache\r\n"
"Connection: keep-alive\r\n"
"\r\n", status);
} else {
hlen = snprintf(header, sizeof(header),
"HTTP/1.1 %d OK\r\n"
"Content-Type: %s\r\n"
"Content-Length: %d\r\n"
"Connection: close\r\n"
"\r\n", status, content_type, body_len);
}
send(fd, header, hlen, MSG_NOSIGNAL);
if (body && body_len > 0 && !is_stream)
send(fd, body, body_len, MSG_NOSIGNAL);
}
static void send_error(int fd, int status, const char *msg) {
char body[256];
int blen = snprintf(body, sizeof(body),
"{\"error\":{\"message\":\"%s\",\"code\":%d}}", msg, status);
char header[256];
int hlen = snprintf(header, sizeof(header),
"HTTP/1.1 %d Error\r\n"
"Content-Type: application/json\r\n"
"Content-Length: %d\r\n"
"Connection: close\r\n"
"\r\n", status, blen);
send(fd, header, hlen, MSG_NOSIGNAL);
send(fd, body, blen, MSG_NOSIGNAL);
}
static void handle_request(int fd, LlamaModel *model, const char *method,
const char *path, const char *body) {
// GET /health
if (strcmp(method, "GET") == 0 && strcmp(path, "/health") == 0) {
const char *resp = "{\"status\":\"ok\"}";
send_response(fd, 200, "application/json", resp, (int)strlen(resp), 0);
return;
}
// POST /v1/completions
if (strcmp(method, "POST") == 0 &&
(strcmp(path, "/v1/completions") == 0 || strcmp(path, "/completions") == 0)) {
if (!body || body[0] == '\0') {
send_error(fd, 400, "empty body"); return;
}
char prompt_buf[65536];
if (!json_get_str(body, "prompt", prompt_buf, sizeof(prompt_buf))) {
send_error(fd, 400, "missing prompt"); return;
}
float temperature = json_get_float(body, "temperature", 0.7f);
float top_p = json_get_float(body, "top_p", 0.9f);
int max_tokens = json_get_int(body, "max_tokens", 128);
int stream = json_get_bool(body, "stream", 0);
if (max_tokens <= 0) max_tokens = 128;
if (max_tokens > model->cfg.context_len) max_tokens = model->cfg.context_len;
// Tokenize prompt
int *prompt_tokens = (int *)malloc((strlen(prompt_buf) + 4) * sizeof(int));
int n_prompt = bpe_encode(&model->tokenizer, prompt_buf, 1,
prompt_tokens, (int)strlen(prompt_buf) + 4);
// Limit prompt to context
int ctx = model->cfg.context_len;
if (n_prompt >= ctx) n_prompt = ctx - 1;
fprintf(stderr, "Generating: prompt_tokens=%d max_new=%d temp=%.2f\n",
n_prompt, max_tokens, temperature);
GenParams gp;
gp.tokens = prompt_tokens;
gp.n_tokens = n_prompt;
gp.temperature = temperature;
gp.top_p = top_p;
gp.max_new_tokens = max_tokens;
gp.stream = stream;
gp.client_fd = stream ? fd : -1;
if (stream) {
send_response(fd, 200, NULL, NULL, 0, 1);
}
char *gen_text = generate(model, &gp);
free(prompt_tokens);
if (!stream) {
// Escape generated text for JSON
size_t esc_size = strlen(gen_text) * 6 + 4;
char *esc_text = (char *)malloc(esc_size);
json_escape(gen_text, esc_text, (int)esc_size);
size_t resp_size = esc_size + 256;
char *resp = (char *)malloc(resp_size);
int rlen = snprintf(resp, resp_size,
"{\"choices\":[{\"text\":\"%s\",\"finish_reason\":\"stop\"}],"
"\"usage\":{\"prompt_tokens\":%d,\"completion_tokens\":%d}}",
esc_text, n_prompt, (int)strlen(gen_text));
send_response(fd, 200, "application/json", resp, rlen, 0);
free(resp);
free(esc_text);
}
free(gen_text);
return;
}
send_error(fd, 404, "not found");
}
static void serve_loop(LlamaModel *model, int port) {
int server_fd = socket(AF_INET, SOCK_STREAM, 0);
if (server_fd < 0) { perror("socket"); exit(EXIT_FAILURE); }
int opt = 1;
setsockopt(server_fd, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt));
struct sockaddr_in addr;
memset(&addr, 0, sizeof(addr));
addr.sin_family = AF_INET;
addr.sin_addr.s_addr = INADDR_ANY;
addr.sin_port = htons((uint16_t)port);
if (bind(server_fd, (struct sockaddr *)&addr, sizeof(addr)) < 0) {
perror("bind"); exit(EXIT_FAILURE);
}
if (listen(server_fd, 16) < 0) {
perror("listen"); exit(EXIT_FAILURE);
}
fprintf(stderr, "llm-serve listening on http://0.0.0.0:%d\n", port);
fprintf(stderr, " POST /v1/completions\n");
fprintf(stderr, " GET /health\n");
char *req_buf = (char *)malloc(HTTP_MAX_REQUEST);
if (!req_buf) { fprintf(stderr, "OOM\n"); exit(EXIT_FAILURE); }
while (1) {
struct sockaddr_in client_addr;
socklen_t clen = sizeof(client_addr);
int client_fd = accept(server_fd, (struct sockaddr *)&client_addr, &clen);
if (client_fd < 0) { perror("accept"); continue; }
// Read request
int total = 0;
while (total < HTTP_MAX_REQUEST - 1) {
int n = (int)recv(client_fd, req_buf + total, HTTP_MAX_REQUEST - 1 - total, 0);
if (n <= 0) break;
total += n;
req_buf[total] = '\0';
// Check if we have a complete HTTP request
if (strstr(req_buf, "\r\n\r\n")) {
// Check Content-Length
const char *cl_hdr = strstr(req_buf, "Content-Length:");
if (!cl_hdr) cl_hdr = strstr(req_buf, "content-length:");
if (cl_hdr) {
int cl = atoi(cl_hdr + 15);
const char *body_start = strstr(req_buf, "\r\n\r\n");
if (body_start) {
int body_received = (int)(total - (body_start + 4 - req_buf));
if (body_received >= cl) break;
}
} else {
break;
}
}
}
req_buf[total] = '\0';
// Parse request line
char method[16] = {0}, path[256] = {0};
sscanf(req_buf, "%15s %255s", method, path);
// Find body
const char *body_sep = strstr(req_buf, "\r\n\r\n");
const char *body = body_sep ? body_sep + 4 : "";
handle_request(client_fd, model, method, path, body);
close(client_fd);
}
free(req_buf);
close(server_fd);
}
// ============================================================================
// CLI
static void print_usage(const char *prog) {
fprintf(stderr,
"Usage: %s [options]\n"
" -m, --model <path> Path to GGUF model file (required)\n"
" -p, --port <port> Server port (default: 8080)\n"
" -c, --context <size> Context size override\n"
" -t, --threads <n> Number of OpenMP threads (default: 4)\n"
" -h, --help Show help\n",
prog);
}
int main(int argc, char *argv[]) {
const char *model_path = NULL;
int port = 8080;
int context_size = 0;
int n_threads = 4;
for (int i = 1; i < argc; i++) {
if ((strcmp(argv[i], "-m") == 0 || strcmp(argv[i], "--model") == 0) && i+1 < argc) {
model_path = argv[++i];
} else if ((strcmp(argv[i], "-p") == 0 || strcmp(argv[i], "--port") == 0) && i+1 < argc) {
port = atoi(argv[++i]);
} else if ((strcmp(argv[i], "-c") == 0 || strcmp(argv[i], "--context") == 0) && i+1 < argc) {
context_size = atoi(argv[++i]);
} else if ((strcmp(argv[i], "-t") == 0 || strcmp(argv[i], "--threads") == 0) && i+1 < argc) {
n_threads = atoi(argv[++i]);
} else if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) {
print_usage(argv[0]);
return 0;
} else {
fprintf(stderr, "Unknown argument: %s\n", argv[i]);
print_usage(argv[0]);
return 1;
}
}
if (!model_path) {
fprintf(stderr, "Error: -m <model.gguf> is required\n");
print_usage(argv[0]);
return 1;
}
#ifdef _OPENMP
omp_set_num_threads(n_threads);
fprintf(stderr, "OpenMP threads: %d\n", n_threads);
#else
(void)n_threads;
#endif
signal(SIGPIPE, SIG_IGN);
LlamaModel *model = llama_load(model_path, context_size);
serve_loop(model, port);
return 0;
}

242
test_serve_integration.sh Executable file
View file

@ -0,0 +1,242 @@
#!/usr/bin/env bash
# Integration test for llm-serve: happy-path smoke test.
# Starts the server, checks /health, sends a completion request,
# validates response structure, and tears down.
#
# Usage: ./test_serve_integration.sh [path/to/model.gguf]
# Requires: curl, jq
set -euo pipefail
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[0;33m'
NC='\033[0m'
PASS=0
FAIL=0
pass() { ((PASS++)); echo -e " ${GREEN}✓${NC} $1"; }
fail() { ((FAIL++)); echo -e " ${RED}✗${NC} $1"; }
# ---------------------------------------------------------------------------
# Config
MODEL="${1:-$HOME/Projects/TestInference/models/tinyllama-1.1b-chat-v1.0.Q8_0.gguf}"
PORT=18199 # high port to avoid conflicts
SERVER_PID=""
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
cleanup() {
if [[ -n "$SERVER_PID" ]] && kill -0 "$SERVER_PID" 2>/dev/null; then
kill "$SERVER_PID" 2>/dev/null || true
wait "$SERVER_PID" 2>/dev/null || true
fi
}
trap cleanup EXIT
# ---------------------------------------------------------------------------
# Preflight checks
echo "=== llm-serve integration test ==="
echo ""
if [[ ! -f "$MODEL" ]]; then
echo -e "${RED}ERROR: Model not found: $MODEL${NC}"
echo "Pass a GGUF model path as the first argument."
exit 1
fi
for cmd in curl jq; do
if ! command -v "$cmd" &>/dev/null; then
echo -e "${RED}ERROR: $cmd not found${NC}"
exit 1
fi
done
# ---------------------------------------------------------------------------
# Build
echo "Building llm-serve..."
cd "$SCRIPT_DIR"
make llm-serve 2>&1 | tail -3
if [[ ! -x "./llm-serve" ]]; then
echo -e "${RED}ERROR: Build failed — ./llm-serve not found${NC}"
exit 1
fi
echo ""
# ---------------------------------------------------------------------------
# Start server
echo "Starting server on port $PORT..."
./llm-serve -m "$MODEL" -p "$PORT" -c 256 -t 2 &
SERVER_PID=$!
# Wait for the server to become ready (up to 30s for model load)
READY=0
for i in $(seq 1 60); do
if curl -sf "http://127.0.0.1:$PORT/health" >/dev/null 2>&1; then
READY=1
break
fi
# Check if process died
if ! kill -0 "$SERVER_PID" 2>/dev/null; then
echo -e "${RED}ERROR: Server exited prematurely${NC}"
exit 1
fi
sleep 0.5
done
if [[ "$READY" -ne 1 ]]; then
echo -e "${RED}ERROR: Server didn't become ready in 30s${NC}"
exit 1
fi
echo "Server ready (PID $SERVER_PID)."
echo ""
# ---------------------------------------------------------------------------
# Test 1: GET /health
echo "Test 1: GET /health"
HEALTH=$(curl -sf "http://127.0.0.1:$PORT/health")
if echo "$HEALTH" | jq -e '.status == "ok"' >/dev/null 2>&1; then
pass "/health returns {\"status\":\"ok\"}"
else
fail "/health unexpected response: $HEALTH"
fi
# ---------------------------------------------------------------------------
# Test 2: POST /v1/completions — basic generation
echo "Test 2: POST /v1/completions (basic)"
RESP=$(curl -sf -X POST "http://127.0.0.1:$PORT/v1/completions" \
-H "Content-Type: application/json" \
-d '{
"prompt": "The capital of France is",
"max_tokens": 16,
"temperature": 0.0
}')
# 2a: Response is valid JSON
if echo "$RESP" | jq . >/dev/null 2>&1; then
pass "Response is valid JSON"
else
fail "Response is not valid JSON: $RESP"
fi
# 2b: Has choices array
if echo "$RESP" | jq -e '.choices | length > 0' >/dev/null 2>&1; then
pass "Response has non-empty choices array"
else
fail "Missing or empty choices array"
fi
# 2c: choices[0].text is a non-empty string
GEN_TEXT=$(echo "$RESP" | jq -r '.choices[0].text // ""')
if [[ -n "$GEN_TEXT" ]]; then
pass "Generated text is non-empty: \"$(echo "$GEN_TEXT" | head -c 80)...\""
else
fail "Generated text is empty"
fi
# 2d: Has finish_reason
if echo "$RESP" | jq -e '.choices[0].finish_reason' >/dev/null 2>&1; then
pass "Response includes finish_reason"
else
fail "Missing finish_reason"
fi
# 2e: Has usage info
if echo "$RESP" | jq -e '.usage.prompt_tokens > 0' >/dev/null 2>&1; then
pass "Response includes usage.prompt_tokens > 0"
else
fail "Missing or zero usage.prompt_tokens"
fi
if echo "$RESP" | jq -e '.usage.completion_tokens >= 0' >/dev/null 2>&1; then
pass "Response includes usage.completion_tokens"
else
fail "Missing usage.completion_tokens"
fi
# ---------------------------------------------------------------------------
# Test 3: POST /v1/completions with temperature/top_p
echo "Test 3: POST /v1/completions (with sampling params)"
RESP2=$(curl -sf -X POST "http://127.0.0.1:$PORT/v1/completions" \
-H "Content-Type: application/json" \
-d '{
"prompt": "Once upon a time",
"max_tokens": 8,
"temperature": 0.8,
"top_p": 0.95
}')
GEN_TEXT2=$(echo "$RESP2" | jq -r '.choices[0].text // ""')
if [[ -n "$GEN_TEXT2" ]]; then
pass "Sampling generation returned text: \"$(echo "$GEN_TEXT2" | head -c 80)...\""
else
fail "Sampling generation returned empty text"
fi
# ---------------------------------------------------------------------------
# Test 4: Error handling — missing prompt
echo "Test 4: Error handling"
ERR_RESP=$(curl -s -o /dev/null -w "%{http_code}" -X POST "http://127.0.0.1:$PORT/v1/completions" \
-H "Content-Type: application/json" \
-d '{}')
if [[ "$ERR_RESP" == "400" ]]; then
pass "Empty body returns 400"
else
fail "Expected 400 for missing prompt, got $ERR_RESP"
fi
# ---------------------------------------------------------------------------
# Test 5: 404 on unknown route
echo "Test 5: Unknown route"
NOT_FOUND=$(curl -s -o /dev/null -w "%{http_code}" "http://127.0.0.1:$PORT/v1/nonexistent")
if [[ "$NOT_FOUND" == "404" ]]; then
pass "Unknown route returns 404"
else
fail "Expected 404, got $NOT_FOUND"
fi
# ---------------------------------------------------------------------------
# Test 6: Streaming
echo "Test 6: POST /v1/completions (streaming)"
STREAM_RESP=$(curl -sf -N -X POST "http://127.0.0.1:$PORT/v1/completions" \
-H "Content-Type: application/json" \
--max-time 60 \
-d '{
"prompt": "Hello",
"max_tokens": 4,
"temperature": 0.0,
"stream": true
}' 2>&1 || true)
if echo "$STREAM_RESP" | grep -q "data:"; then
pass "Streaming response contains SSE data lines"
else
fail "Streaming response missing SSE data lines"
fi
if echo "$STREAM_RESP" | grep -q '\[DONE\]'; then
pass "Streaming response ends with [DONE]"
else
fail "Streaming response missing [DONE] sentinel"
fi
# ---------------------------------------------------------------------------
# Summary
echo ""
echo "=== Results: ${GREEN}$PASS passed${NC}, ${RED}$FAIL failed${NC} ==="
if [[ "$FAIL" -gt 0 ]]; then
exit 1
fi