From 17562bf01146cf49fa68f3f0bce4a8fde7365252 Mon Sep 17 00:00:00 2001 From: Cruthaifios Date: Sat, 21 Mar 2026 22:33:41 -0500 Subject: [PATCH] Got serving ggufs working --- Makefile | 5 +- README.md | 104 +++++ llm-serve | Bin 0 -> 61760 bytes llmc/bpe_tokenizer.h | 407 +++++++++++++++++ llmc/gguf.h | 460 +++++++++++++++++++ serve.c | 916 ++++++++++++++++++++++++++++++++++++++ test_serve_integration.sh | 242 ++++++++++ 7 files changed, 2133 insertions(+), 1 deletion(-) create mode 100755 llm-serve create mode 100644 llmc/bpe_tokenizer.h create mode 100644 llmc/gguf.h create mode 100644 serve.c create mode 100755 test_serve_integration.sh diff --git a/Makefile b/Makefile index 73b8372..32ecc14 100644 --- a/Makefile +++ b/Makefile @@ -285,6 +285,9 @@ test_gpt2fp32cu: test_gpt2_fp32.cu profile_gpt2cu: profile_gpt2.cu $(NVCC_CUDNN) $(NVCC) $(NVCC_FLAGS) $(PFLAGS) -lineinfo $^ $(NVCC_LDFLAGS) $(NVCC_INCLUDES) $(NVCC_LDLIBS) $(CUDA_OUTPUT_FILE) +llm-serve: serve.c + $(CC) $(CFLAGS) $(INCLUDES) $(LDFLAGS) $^ $(LDLIBS) $(OUTPUT_FILE) + clean: - $(REMOVE_FILES) $(TARGETS) + $(REMOVE_FILES) $(TARGETS) llm-serve $(REMOVE_BUILD_OBJECT_FILES) diff --git a/README.md b/README.md index d2d1078..6b0ce89 100644 --- a/README.md +++ b/README.md @@ -113,6 +113,110 @@ make test_gpt2cu USE_CUDNN=1 && ./test_gpt2cu This tests both the fp32 path and the mixed precision path. The test should pass and print `overall okay: 1`. +## llm-serve (inference server) + +`llm-serve` is a lightweight LLaMA inference server that loads GGUF model files and serves an OpenAI-compatible HTTP API. It runs entirely in C with no external dependencies beyond libc and (optionally) OpenMP. + +### building + +```bash +make llm-serve +``` + +This produces the `./llm-serve` binary. + +### running + +```bash +./llm-serve -m /path/to/model.gguf [-p port] [-c context_size] [-t threads] +``` + +| Flag | Default | Description | +|------|---------|-------------| +| `-m` | *(required)* | Path to a GGUF model file | +| `-p` | 8080 | HTTP port | +| `-c` | model default | Context window size override | +| `-t` | 4 | Number of OpenMP threads | + +Example with TinyLlama: + +```bash +./llm-serve -m ~/models/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf -p 8080 -t 4 +``` + +### API endpoints + +**`GET /health`** — Returns `{"status":"ok"}` when the server is ready. + +**`POST /v1/completions`** — OpenAI-compatible text completion. + +```bash +curl -X POST http://localhost:8080/v1/completions \ + -H "Content-Type: application/json" \ + -d '{ + "prompt": "The capital of France is", + "max_tokens": 32, + "temperature": 0.7, + "top_p": 0.9, + "stream": false + }' +``` + +Response: +```json +{ + "choices": [{"text": " Paris, which...", "finish_reason": "stop"}], + "usage": {"prompt_tokens": 7, "completion_tokens": 32} +} +``` + +Set `"stream": true` for Server-Sent Events (SSE) streaming. + +### integration test + +An integration test script verifies the happy path end-to-end: build, server startup, health check, completions (greedy + sampled), streaming, and error handling. + +**Prerequisites:** `curl` and `jq` must be installed. + +```bash +# Run with the default model path +./test_serve_integration.sh + +# Or specify a model explicitly +./test_serve_integration.sh /path/to/model.gguf +``` + +The test uses port 18199 to avoid conflicts and automatically cleans up the server process on exit. It takes 1–2 minutes depending on model size (most of that is model loading and generation). + +**What it tests:** + +| # | Test | What's checked | +|---|------|----------------| +| 1 | `GET /health` | Returns `{"status":"ok"}` | +| 2 | `POST /v1/completions` (greedy) | Valid JSON, non-empty text, finish_reason, usage tokens | +| 3 | `POST /v1/completions` (sampling) | temp=0.8, top_p=0.95 returns text | +| 4 | Missing prompt | Returns HTTP 400 | +| 5 | Unknown route | Returns HTTP 404 | +| 6 | Streaming | SSE `data:` lines + `[DONE]` sentinel | + +Output looks like: + +``` +=== llm-serve integration test === +Building llm-serve... +Starting server on port 18199... +Server ready (PID 12345). + +Test 1: GET /health + ✓ /health returns {"status":"ok"} +Test 2: POST /v1/completions (basic) + ✓ Response is valid JSON + ✓ Response has non-empty choices array + ✓ Generated text is non-empty: "Paris, the city of..." + ... +=== Results: 12 passed, 0 failed === +``` + ## tutorial I attached a very small tutorial here, in [doc/layernorm/layernorm.md](doc/layernorm/layernorm.md). It's a simple, step-by-step guide to implementing a single layer of the GPT-2 model, the layernorm layer. This is a good starting point to understand how the layers are implemented in C. diff --git a/llm-serve b/llm-serve new file mode 100755 index 0000000000000000000000000000000000000000..e93fbdb82d19fed2a54508411a2856a139efec8a GIT binary patch literal 61760 zcmeFadwdi{)<4`6CJ->O2LX*P%4{}hf?N^=We_xj6X?+iMg$Fr#4wpaG$b*Z;i7V# z4AQpEMjvqTQCxSQ#btLz*86Y8Kn=_VuR#v!O69uyWcSl@f>+L@ zrKwyj-J0nSK4$hebiJ@lO9!u!-#c#SMavT6TF%Wwa(^EFX##F$`X zbUsO3RL=CVhHv_+^VO`&Gk6WklgQ9_j4sdUYnn!e3tpqk&DG0AUu}B1;Fap2&!gYD z^Z%dJm#vpm{;AQ4zUs~9g4bYukDwgM<$wFp#mP<4>+7x`;`9oFSJD&uj3_FcefOvl zMft;u3X44phb16GTf)?gri)6?TRHBN*rkKEE4yk8bGvrW^cQ!H-#Et<&F! zx}kGZH}JvT(0`;G{J(aCZ|eqL0`9t!!;{_MpX>%7=$skf=+784#YJ#^JCKXpSVrW<}}3h#4whT-SBgFH*_B927UP|n>~5d{9M<< zQZ1D~d5oT#^l(;oZc#}|X>M^zF=&^#ONtAfx%rEV^XB2oYT`=fmFAYa+_}Y`dAaVn zWv;yZa*h=&%`3|*DsmNRDMrRsoDWh9@(SG%z%FSQyB4}RkS#B9K2uWa=1N^qMdi+Ca-DOZ$t}n$EaD2vTrT>Pm(RHtf;@Ls|7$59@UJwBjx-PKXd3lAX-;G`@z5@#S76MizTl%(Vd%)=VG1M3zlec1V#L2#_BG&dPnITFF9s!;5xz+e zE)Lh;EM?^S=vO0yQkm<3Pnd*5)U8jPHt9UW41yn|l#3 z`cRFfdVV(2%_+CTjY{BN)nOUQl)jGpFFoD-D@ym_-q6#Zvve=+4@lF;fHp1G^GcCk z!|J=9+oZ!jVCi^ntDf#(SeU?}U3Gf;CYJ8WeWIt`Hy8FMFY%zB_S`~gBX<;O`WPV9 z>3SZeUu5)qa3?$A$As{M>xiH|I;F4EIDMTEf~OKSa25sMtmDVEuxxOyU-k5%BU+k# z%U~WsS|Z?)^NbP!AD|Uwe@{igNq2&u_6T@-5QuBg&XPY(er)g~e68K}!UZNd^vRBZ z>!M_NIT7%nFHRXxMZn3=4u0lFz}fmx&nS(6Ge290FN}Z_Pr=W!2sk=8^jQ@F?-PP@ zTxA6Ox(N8X2zY!1{Ot%h6fX4H7y-XN1m(ER5pX^NzBK|)V=DNmj)3b+T9(xm0oRw- z4Bi|8r*%#6b0`9SV-SdIO9cF;2)GgfPmF+{ih$o70dJ3h-x2}mbiJZJ`9%agE&?98 zC^ALBX{{0bBu2n*3j%R9N5BU~z*8dN<_LIN1pJo~@G%kaqzHIM1pHSKa3KPo90AXc zfDev<=S0A7kAOcF0ly;xJ~slM5&$(W|@Cf+Z5%3Wa@Qo4hv8ed_|^#cT@mo=2>7T7cvA#?bOgLP0)BS{ z{Mzj`30#xFH3?jkz%>b6lfX3zT$8{x30#xFH3|HGRRU-E+d2gA4{?GNyX!-a6MWU~ zm_W1Ot&gi?EC)v4u#@8ggZtwzKPZDE{JE5)wgv)$f=ZUAz>K;toTh+`S{qJNAV&RL zI86Z<_4na41zyxw!)Xe*s4K#0TGXp$;WP!N)Y;)Q1zglA;WPzW)Cu7<1z6OP;WPzS z)TD5l0xIea;WPzO)R=IZ0x0U4E1~vM;6!Z=rzv2f?hB_WK&IA)(-b&S{}xVDz(oCh zI8A{w_0@2i0w?N)K|l4 z3S6iw!f6UvsAb_a1uE3p;WPy()G6UK1t!!9;WPy#)REyd1tQd>aGC-T>J8yE1s>Fx zaGC-R>X}QS_EVriZ4IX>kg4tqr2~Us`xnO*@Plj|8!v&uUvKZ6J`k1O6P0d=O7DzH zZ;4937nOb|D*aYe`n9O^>ZtU}sPvMk^!%uFaa6h>Dm^PI{a93ba#VU^RQkTC^xaYE z;Zf<^qtdrUrEiQ%Umun38I``Wt#kjKi%S0(mHs{|{dH9OKva59RJtK5y)!DkB`W=1 zRQjE$^jlHs*P_y^qtYv*(o3S!^P|$mQR#xH^sK1#W1+MlkN=azampH4+Yf0TQD$J= z0lJ!?OF~ea27Q*Gb3@SK8uT@SlKodbkE5f8^Mfiti6$&I{;ZE%AG1!gPMs!*KR=4e zF2x%#@T-Slma@rnu2{v(f;=m((U#nw<|V85+bhjA=jL>tkSdF}|2wVwL777g*`zmh5Bk9#l z2+A851A+8vbsP|A{zuu*WsTduTG@)HA$VfO^8|70^i0)+kwu)!^N|UGY^NzX{hT1Z z;OCSaSW3udpn3#T3_oZYvbboRQ5sI7`cH;K_2%pKzK$$3<87+b{V+17=^4vd2JfSp zJE#JTiPNAFUBPE)7nta;&Ol~ZWvB^m&!1Tn+_i1L6TB~=uRTu-Qo?DRMc7VL*-p|^ zsHjp2wfiS3Ws;M|Hv<6zyZ~@edQWx)0(9hogGZI}#FN&U)+epAthsd#@d7Y6zl@Bq zsMCPl_IGJ!a)0rNauoat()brZQ#_nr4eWKm*7~Fu5E&C6CV;e(2vo_PJP^`Nj#HkV z!Es0>3Sy&jJ&x;y+ZOcSFb=-s-+S`X@~lf4%h0XwO}q4Do!$@R&!wNRi+g7YVyoc& z_Eh%N%=GH?U1;uK(bB+xx+;$QDNGaaU4RI3Y%N_Z?d}0^C0Eit`lxrHMW-925B%2| z{^(cfl6pnJXsP+}exatlhhS(B_Fi%K2Z1b|KwO|jn@5A?iN9sz=OLT(+&N9~jvqp| zXplP|(z+W7Wl?(|pyY$shS*G`4b(O+vb?&0DF~X5QABouj6$44DlujGPPm5)Qf{JM z{3-oJ+w}#NLcouqo8qvXpoZP+5WiP$hV*^ap6>+tfw;C9bPb$kt!wKFDjyZZ&y+uc zJ~cnlJLb9t{UnnhCwu^lBslxOA{chx7ibm4Lr>HRQbybpb^I3kEATs6;vZV#L%OhR z!AZAad3WMBj`+PB{F0zXb8Lv>?qPJ#4az`yA4w4Bt7vVjf1A_lkk9 z@W5_cr$HHGA@0odFR5?Evy5lwxOH~f8W0TaR{qtx^nJql^Jv+6FYA{77Wnt~;@fOD zG<&ba@fCM5iZQ&8_VKfPU-G{F;Lh%??wkH4^XshqKeA7Xd&L9d!R4(1EO2VRF%{JO z*r#Cmu|SXI-vD`8f%p4Cyzei#E${{0x3i{Iik}vOBPKSEw7`V?4~e;XFqcMuq+ zAFd$N_pSS`9b`Cz@rS@CB)QX8d1`Wrm4w%NlU0+Rt@rWXd(r$|g7^i*Rz8^Z z6Tj1OQ;h%kkQ#pniQ>7w##aT)8F!ozsC8I&m$g775JwV;g^AYNU}mCmpp|M%cz%kK z%jH4tHC}&d&38A6YVW;mtFnB%_=>l|E=eZ+T$XGUG`od%8%xC+u~uvx98#0xozAYuY}?Jh5!4Skv|~tj_q9&6rb8L;G{#SLowf`TN=6 zNIr}^Fo6+-582yk+f2A)%)!tGy(t*YZ(A_5o)Vl35^Yk+z`2abz=;wZq9YmB!foA1ykfXaaLHt?}Ywco# zQin;>f9mmFnZ6VJ>Vx1C6Z@E+{K_=cn2v$+cs0|P`kEI^$vx|9pWJHfC3vf$4E3J2 z`lE|)t~1aS>u{P@39{K_8Xc|5Op^+5rroA1A_AkbksBoPN8uJ zNOASG_HFSW%1aDZ%jG4>@FOYvgrZm}DU$dzALWF*0HlFWx)=U^uBQZ{;wbVBL{jd4=8wBS$`9TB3un>6uFJ>vk z;Wwfj2MqACDFxvAqtlZV#^E9G3p1w(>!|+vx<`^uqpYtTV7s%HU}#5Gvxaq6aDIu* zL{PBO>{oLXCG1Jx7)0MVz-(euje;Szot||poJb>~U8ZO2i zXm5xku+Kq%n~~Xss^ka2>69ihbB%Zi6#ZwWR}-E3)@T0|yZ(8MX|X{(1Zctv1&8j ziN1BXSV2tm{Q!1sVl%Z12*^660mx|h%!~}W7ga+?K0p$>F_dUXJKn}cx%ea9SG5pG ztiw3`nR*U$3MgeWJl`^Q$-pogcPjmyQV1UiB}j8^(t08(NIf0UluY;ytEa-gcz2Mn zVQf|*vIf))!Vzm}7zuKGPyZ#;5W!n(u5X>T)h}3fdw$dnP8G>GXmYMHIF>4^i?8_M zR+#hi;u(phQ?JOA1Kta;I^qR?yD5H^c-s5TC3M8r8pCey_gz?>`vvE2X$R4Q;u=>8 z@+da1LnZ557j0{@e7fXTWZ2S--r5)qa&XCY3=+qZxZN}crKKurWqs=-ugj#d{xef( z(%l7|KP9}r2t%V|dhA-dLOHxwV&?Z3WWngbDkF8b;O$7{UmT7&r69@F4e9$pHNGEE z<4yR}gyGsZzCVOyB>5bLi6uYswzU1v^lo9j=W^MXGA3g9 zEM>!3xe*2Z7fM(HjygJ;uV`a^NbH6c>4&RXc}io2##xvZsfsEEM}-j_DMNK0)__d6q8Ov| zVD5&8US06;6|_o_e2j;E>AN&>gn0N4Iv>%SXmkaCX4Vr0)`|PN6P`?0Qpd66Aqw9u?O% z0|wHApI?D?i94!^D{*~0UF0X%Im8`JxP@5)d8QWr_S6j-po;XSjmXmdP;@+4$!|5@ihK7BuKUjNA7-oK`PIo$a3_CQ0X<>#fM#Mpo-p#}u1 z?S}IfAGMnIJ%O~8St+pwJ0#XD*fwe}sI#W{s`=F?VAPNv{N{2PkLJDYn08KJYlZ zQ65+9fyPYBSMA!TV}$v+VE7_923di}F!0OA z1WSDbEfA#V0bBbvDosC^{-tvH7{S;SDpkdUqsB}`j}Q)%Wb$Wh(gx;21Yn|ka{-W; z!;^Xnvl>!?CecUTl8M#F>N(gYE2u=*!-Q4>=#WRPCsLuQi478mSXGV8*U#@*g^M(; z(jiTTrJSFEj;6lh-+v@gaP9-oYgu7&E$O=;_rEU_YP0Gnw!%P=Bx&W}ug5?lrMHtG zu$I)qCT*ft2oR{lISVnUwVL+aD6Dhv?;i#n(H(tq^=$+di0vatch5uesC2t5TX>;8xU z-%B1B2W6?RdCru)y}ow(w)o}nU96V+C3gwyFnJl^1V9L6%$=&)!q@tq(cNlduj9pon(Fvh7X91QmgPg0L+u(?B{+5P1Zhis8Or4Fc$U#

*7RsI<_%J@H)_sa$*;4o~y9b<0M%vbb-QaPNbHD2trdeH2^gR*fXrV3U8=wZgy zW=Al@WY96QPBDN9^bu7Rs+bk9uyr4bQ#S{BIFkcoN4Wgzx1azI%8rpn;#BKPHWHA; zN=Tb}Lv^E~bw`<;OeCiclh$ZPN>f=sNC&&BqpxAfgvqjdWLyMos+C4&Pim1y1ym=1 zDv=`qit9*ydlEUo2Typ8IsVA)iOF>rzf!>-h^~{Qc{J+w3SzT9-hPBqlE_D zk0Eu~h;l)7K<8-#BDGOoV4zWja5WoM7>6|+qMQ1_f(3mM(LV$u`f}m%<-SjF*61S& z3q_2k7?>GqS5Ns_8d0|ivLnU_81%TW!E$~{0!C|08~_kVUrb%PcPG;%l70?uK`B$h zj5vpQhU~YH}Jx+iK@u!QLF9K&cV1!=q2Z*&+AI1GpEmVbNx1gLHp7lMNT z{3y()hO-v7X6Jox!S$xKNA#M57T>hyFzJpqX4Ed%9dES&FKXRUj?*G&F)i!6aQ0t- zPaSf!?>uDv%By-?;lFWlfTDGL_!9I@n3M8`> zv3VH(w}C|hLf_^X`;dI%1aA%GbK0Xok0@JdJ3`%9m5qB^59`Y;5spp%L3pui><4@!)<=4nT3_t$4J~&6 z_BBblo5e0%*NidUC|xQFMl0PFrZYYh`lSn{qxDKi>2_1SP`Z)-seZGr_!-^su;JfV z^YC0}9$1B;1!e`UkJu6%>uzPD&ck~k0~I7qlVpn8IHac%$@wLxK#+3GQWb@NU^d7c z>p_M`5oBl)QgNt_jKX>t1&~NYVd52NS>pGT<)LOWC)eA(4Mt4!Ml9W8A8}J3_GN)Y zWhynGBPX_q0Zk0(KQpTdo%_%w6zx0tQkLas&-SZ&h$uW&)<(f{P>Y<^8h{Ir zs|jI5T$;s$Fe8Vt4#2nJBMP;DWV7t%D>i^?rr2anZO@dm;r4eRXa*Z%+-egWy~kig z3fEN|I;`H7E`ny`9nRYN-#uqCE_+ZO$6Qhye0^pwGA%VrZi88cKOKyiSuJ}t8!|?> zA&3~%*G$2#)R|e;P%GaUrX^EIovK;TbZ!AN90~b!g-y_Pd7&0C!;wu2m_3M|)FNdR zBxB*SPLVRU&h#A7qGiDzdHM@BhL)j5bQ#To(99e}!T5@wNi+0L*33Tw&A-wWiIiB- z0_F8J7nYo$m|Ar>q*izMY8_s89r!~mO^`irJH&a2pN)U_J56!f*j9=eFPTlDO$Xj5 zp^UJ+UfUu#65f8W%dS(_C)IaR4bA`Fi%<&-)P%WF;r&r)1kLyJHe$ZNq|NtO3~l*w~19`5**t42=0vuDdt+);aVBaKP($p zk|&*l&75_{ZQ>Eq6S&S;urrU+W@z*t?ZH>PMe#9b zeLa2~k&07Okgu5%@?p|JY>pl}2cjLOVKYjgq5Ed-Qt$kkYFVHj#FdVWs8a|4P1m=I zc45`CsTuUtJFxS_I?$~d9dZxQapU6r_2h@3-^dTa+7o_==I21Xov&R3!bX=!D&_d#x*pg-W z(esY9-VC8iY!}G#8DB9Tb*Fw|6~D%I61H#omGt5<>|V7mApe)y2iW<^+5ijV6IPzkoXCO(Pp`bJK;(qF< zoY>zY4|r@6K;VchpT67hw9Z!`ES|BFCz})!xOH zWX~r_p{B>i&=r-bGjN8%hKP8kZy&$<69ne{O^=@w_kzm!`E!A@d&uz`b&z4N*o5tp z+F2*q2Gj<^+pTRt0dw>~VA9qTHlrxWN8?<;{t`5oMQ+t~2p+PTdi8_u#f7G+;bf~Q)}Qa=}H^&h42q}jl^3y?FZt8sHj_5}y|XrbAnx&gf5~vh`wgtv6V#o~GiVslphugrD6b;V zB&Y=OAIN10q@TtF0#x8R;+P^`M;>+%$q|q9Fnjsc2|*O5N!NunDV}N4X`FP?Mtb@# z9Bow_V8|fiWz-82r(U9Lwupk~dQScE5=m--x)WG*%7078kZ|1$Wz)D|$7|I;>+lgcTuQ;}^kC0C0bhv}G1%KR4-Tne zJ>AX1VFZr}d%6bOkE9gu$z@ISiF=eF& z_FX|^BCtzza5KQm?&WyjSt^DTtlQ{aLff%-jO~%Rm5kQ<>Wf$4SttA&BzICaR@4qT zwmz4}^M>lc73}weoOK(aV+S|AI!uJ&HyV?Nfgs9kMH$5lWUM);#W*y)jmtoVurwH4 z=ausiEKa5|Aky2ZoYD#)mOFx!8Mz$F984{WJcp$%gs@*T`rvfp%hfQ%D@vAr~ow$U3;S?}zmyYy=1Zj=T!X+4Qk zlO-C$A?ES!#{nIHXc7Pr%SI4a)&Ndt?bUao>khd}@7>Rlw{00P)3Ml!eGe(E9s=Re z9Ck-HqXwk`r5z%|7DtpfKEh-!*0c5TqC|$J+yj`y(86yqE$bNTj$hVc_9W`-E`Ce4 zK?qa>i#YTOEf{);nCVQQ>0Q@x?;??IK}PU?+Rr%qB*!mM206=r4ots1AV6D-Jv z*<)h+d0p@k1AOo?L*t_bd~8VW#7EU*k$fyisxu#NqrJLFJP~~S{w9r&zL9*4b;m*+ zrphk(Ca;v=9xHBF%LSZ9a#2xVl!3;%~IAioeNns_dj}OvhIdF0_$(jw}^TXCeA`Yw8S`?Z^FeYW?#aA{_WI<4ns8_!4NE` zVDSb1fhHlfW-HSjOO3!!Xxd5!TkxvbLqd8|lQPaveh|sUE>>ef4Li2Hn)=r&W?)C$ zN393_cDtcL6Q7me($lh*`W|N|p+?jd4qVfB;cnG7@?08ODKMisE@~ZfXLe9eiEByM zl$EGeR}U#CQIZLWBdUlZk(LVb08b8e?)cl5g9ey^r!>Q7MwaC;@2J6%R^W>@?7=PN zzw6P0^)w0^eIMg5eSBZKvk(^Dj1t=H#ilGar#uSEnwemtzznje0nBusyj-XfIVZuU zxSojWgDU+TjVYY0nPs9s$v9XPXP0)a0y-V!<=_7df$tclu?`ka%c?{c;&Z{AP00C> zoG;}ruuX%F+Djwm9gGJXj)S2x*!@f&#Cs175p6VQrtiX8C>oiW;wN_bEvm6?D9+Po zO2*_&QMJqSah4BHro6ThIWRrefZy(|H!AV_uuEz==|&v0IC1G-pi1jW#&q=cbNGA` zlbOx%5qgbOigvf%DCXPm?mhxgXpMekk_LH&Bn%D1(xi4FUnuPE^mW z@{xX!!A`-I>9wv&mWhe{>L*bzt#177BUqEFhzcS2?1$$po~DHocDIEAP zYkSogI>&p5{ICHBZj3J>$Vs8OrtpRYJ19fX4&*kuhX~Fl|C!wQmtg8ybPn!@lkaa99oRC3Alqst;~pph!C0;*O#LGV%1k@H|^5bsPRHN+S3G)+$^LwX|P`q zZ0oK6E_(QtpE3tS*?=LpP>~uX1t%U`{9jBT+>nLy4zyXbo(`uRc8D9ucG$(w{Vk7w z3GuL~CH<+QPD2MYGXmelit7*I?m>CfFA>gTa|?P_PGm@DDZ-bATk7mnK%+3Ahx30t zlUX^aAf9IOn-S7ua{;i|R{~r48C{4|IxIo)4_k7J|J$5H4$D!`4(g158zfdMI+u|} zy!bPG+mL28V=xf+;vwh(@kuA&l^bd)%$&N3_`qQpY-cw#PooVc%OM;jgJNT0hXZh> zC}>Un#43Ae2NEl8C?1x<#+O*670*h&j05?=f$e84YoA#QX=DRJaX>b?P;04v5=13k z5uQ~HHKdlTj19w#(SSk_%Y*|?i3UQ1J;9uhm?T~Kt_QV|5o^Dp8Qfx6=xp;9_rZul zZ*Z`NRR$i>u|&)v4`oV3j1aH>4g?`Pru(St0ApPOrkNzbBav6qtBF5WT$`CuvEFwk zMuft0#+R91`j2B<&PO%z@;1l^%xko8o<^%;Lz-SyHmYZ2u7UnUSLU z2%8#ArLl+7*EZBhBLLMw|FI_;sz#%O?5#Fy#|r!q*7L5mkUj z=fFAh?w}#m)4zMxK7QqHI-Ib!lrW+1yRrSOZ&^AFt@=6vffUS>)zDma-lx{{BaYIb z0G-vrvk$$*TE5~Ykbz2JlMWfFHP{cOUBFur-L1e=N43b)&-XOZ`MFwzQr!2_F+^B1 zyBu%C@nLp~2M76K2!3go9Yz)|(ZQB47AGKpi$Lh{={R=K7hlsRa)4nrhxyG+urhMjBR`TsPR`pu;4^*YaBoH0RrHqd$B z_b4oi=t;;9zT!5J9M|e;UAD;Zn&+!!i(;004qykVM?;L6;d=Im$~(_F4Bt2O1npx= z3An?ET8KvDyf4~I6AV-z;>$D$mWJ|CSlVo*Tm#mE`i#NQmHj(Ia|EVWeWK+nG6pcY z5uFg>8c>3(vf*Axz%hG8y#cFwrb)xTHctCgrN?R4|<9$XDAt^yq z6rQzbJ!>W08xxW^k&mNj8G-@p?QEPvVl3j3QHLf@<>h~34W-SsHh$-ISZQa9pJi6y zY07u-)|^cF=}Xv`H}b33AqDq^s>MTX;dqqY*=U!?8Li7MU}ZTDN{G9ee8n#?zu~SA zUvVe+!(9SjaStK{xa-YV6vVT;L9oq$*;et^K%<&~!UdIC;y2rt!6RfY92`1n!)6E) zORuAoY3YR=Qk-|tGHC~M-1OLv;A~dR)L!%zx(ptY-hZ7tj(Y7dmK*4`P=7I>6!B{a zR->r}I57BWMV2eTnQMTQfF^p1A!j2Rj_6R!#a&$*%C5Y+u~5M|8hy}?;fE#6dVEA3-bj_ma~tJT{sNKNlz}9 zG%@>i`AV(-%2J_4urzy)>o)8Rdw#2z*zBIE8?U{(@k$0YJVRF6$oVf^iM<~+U{Y9W zTgy=p`4S$!h7C)BEUhA7p`%&Z1{kZpu;W&7Y63YjvB*!rk;6^d;9q_ei_squMn&`{ z4QaAWc!Z%8w^s3B@cg)?otJ-w?!!?JgLI$O@&zxyND_87I^_9At9a2S?gKBT_zKL+ zBo>Ews;yt8-PshVMMI5QvP0%9U$}1Chp z77c(FpD4i8!IzpjLKVZ0&og}TVp zU?E{&?l;1oP#Ci2A85y84pBv2h(qA11}GlJ69&)s59uQcp^9uoEegO%RFp599yXC< zzp$ZxJs_wBgxIR;gq`X*uM9&$h!PL2oAvM`l!c8KimU#1D~1WyW{HSbEa_)=e$H~V zFr>1AGGUdT2Aed7XXC2Q#&RHCdqW31<)JP`_+MW+Vr4>IkSW&U5fTy;JptiUp)^^d zYGsc@(;fhup`a$gy)U~Hih_;EocNcm@O7+~-Ae`mHh`Td(BPSt+9lW7ol^&3rxJK; zV06d#gMG0|z3sBQ3KKgPFSWCFwOwCH`h@v|udqVOg42W(A=`{=aCn_fv)`a0@DP;O zwosc+Dywk~8iR_*I%uKFiy|fgeSXEHqjDEmM<0W`u6Hzh^;hRc>n*lB&sgOt*ok#i zUGjFoxBn%~gbjOQ*rD+MJb#x%#FiM_B61j1?cDs8k|A)sFRi}Q~mCVFm^y&G#aKfuS zMWwTzwU4|S|E87M4z_>J&eunrtH)zND60Mjf5eZ}-MCoAee|fTq|H=?EFRJ5HDWws zC_@?%}X(cp6Tbfd#%ycAZY|uwr^1eR*wJRZe zjPcXCUOX^jlQzeZEx=*kRsSNl2pjtJLJ~Y4xsyY1MJU`y5fhA~tO&6T%W8u^#$tPn;yAz)dh zp`Wt#R`^vLFl@7QDQO-Wgp+2~g`13w(K`+uu^-Y3B@9eORD9uK) z4JZWef4rZm37?6+*EUzkXtw_yD$n+w@QaPK!6;IDIWnjDq8ga@zh5whagy+ULJ>(*tCar-;8?*?eKk(VgCO{N>iX^ zBo&Bn8@ddFL#vh5pdJjde@+%)2O+7iDx|0~`GfQ*o;VsuGlK5g#m#izX;1H2G>WoG zf4bP(Q=3qvA*N4o=SKYrvZWB74|7xcm+Cgm)M!#@t#)`48|j_aYE>BTw9V|$aK~2x z2Q$h()kOygu$_!Ed9=$+XY#7Fc*X)X16$eFtuu@DiNPlJQrjUuY#CRp3sBC>2D_oN z@4)*?Q5S{IsX1CSQChQd41j8t$`$|=cgM<`+AD1*l}GS#RV%Jp&iB%Q2iL8UMKd&; znF2iRj%*7)OKnDrC>%F@9)#ofWt70*IOP1_Hw=}~OWaA2J zz)GS4#|FlhOT78d!f8Zk_^>BBlb?>|6p0emu?nmaCj(1RK2n~`5hD@hAHEY?A7?>i)!R>rXSq^IfC%xlC{l8qF0 zBPfjBh8GoKd^pbpYVks!zv}gp9;LucS>X$<9)ufxO2C6kQk-wsqEGoP0v^{aHMZe3 zIPjU}=djI+H;Lex?f6&d#ZVnqOZ$?)!F_+vD!u#?4M&tZ%x{@YcmV$fl$Ul9g32wUYJe9kM?2aWrV>%(9>w-=lte7Tjn~4g{p_pQ!i4M!}=NUfUlm*gA+0 z?lZjtjNm?73p$A8to|1cb2Y>ygsSL=D!|9+rS~vCn6+2k-~;n2I^QOK%`^v7>~34? z70guAQ3ob;Hjtg?Y$A*EVwm;Bb`+Kuy*80OW26r&He?%uN4Q+@aZ@wd6Ro_0z|%J%IHZo=4uV$qVbX zbIRtwX!63SV{k7W#K$Uw{!%IObgh9}o3=4$6_=C3XYjvm$P`~;qXc`)II7SaM-q5n4U9HBL-$V@ zD)qnc2k`>XBSMnac#XI|0La~6T?90{bGMD(B9q|o>LfT~*(f0t?qcqlHi8`Dr<0_Y zUnX@%ApJU{YNZ)|MSL zsKqXSa2*k+_w5{~lBoU&5d)=|Nz$8dQa$xYA4zJ&GLBw-vr;2`8xi%kV^8Q3x{Fag ziyM8EgNUzIr=AEW6g#ztY^x?3_2z;vq7ozz{(FFTR=apx1k*TryrhKAcs$K-sj34# zoxBxF2$9+4itCc$250bF_F#lFku<*RgvJAnmb?kaagc4wMuf#G zr_ly}_#I~AS+;z~0ku_NNf2L1PGr)EIEq%0ybxKiD&K$_-ZGX#rukXaVHF!RN0AzX z-9xP8TXHGg#cA@Nt^mfGaFJ>YhA;<}>BcQSve3$7x zV@@O9gxYjpphb|E;EjN4#LGfoBG&P;P~Nwi3gVq~aS@M)-9|8Py-9nEsdZMJK7U?K zXM#@0{V$ws-A8Rm!+aXVgV?n3q1>Y38Qg1t=Ow&2;Dz>!bpFEl$I*W9hqsMB#Z+PZ z6egcPQ<)4jRBI0ndN%$Pk{9q}_=+?Qe=yghK6tC zD=1z9{7(qq)-C>!5dPa5{_hCCf4BG@zi0e|eyN75_Zu2Iy8FLC!yye#&XPR_T6aJDJ2FMu$;+Y0vy0eOR5cHp4k9)xzeQQQ z#n1aFKqpAXWQXXCvo;uwoLZ0BjPa+{7Z%r@!RGjGLLrn9!ZzcxO-mu-jpYqy&8yuUr zy^V>kkO)W#6M)L>YH%8884lcdaIcNyGbjb>pA+;(qArjBj_&(t_q*u+3hX!F>*)TB zc3(mF-{W4h-wwQp6%Xs-O`zfhYoM%MkO%AmM)0MNbS~U3Ub0T1*FI)oP>9+)5vB3} zgEKy?9T8T8UhAj~VSJ5#8&oLv_B;k5R}3r5sf3@`5;kChMRy4TQOF zK&B456`(9>j%jBK(}rWpOwhKAlkmc5O&fj-+E@n(^4hJaE(>ohsvx6{C(>!OVZF<^ z5ZA6ls;wvX8>Iz_56k(RxUdDp84AM1`%dxX_6|aPNH*S1%Vyjv&yYNrdAt)f}R95JGK{mpJTh7qA4g6 z;dp7B=v*`ydk_k7A&7}e>syQwBg75bSv}??BE*9{to%@bwE%8Iobm-$!&nD`Z@ z0I?M*h@IIGBjky#ykkmV;zgWfW_|QH_=%EZA(JC}ra3(SJS0!u1g==2DhR;r1rw;d81PpzSuEDPu+JQ0H31b4b-pALt6o}wU)AyF_nC4|goxVW01jl=OX*o$x+G=$FKp%O~vn+8=nlb=QgRZ53WlwJ!rM^7yosRcbW9#yW zvgN-c#;T@NcIojnyR@y39{zU zmfqemcrW>_Ce(+p8y!;!J;MUO?le#-;*audFMyt=I7l!G_$%kBBQ@&Vxy1yBb-bK(*Z+wp(My%AuYmc1O#r4BCxnK0a)(-THPkp zjiPlZ%G#m5F&2vgv41kFL)nKu7o2MuA{=ROlco`4*^m!>*dwfF_VN&28iE*4@_7as ztV0xuREMNoVW-yzfynJ3Qjhoi9>OCYv8#|k#9@6k^7O{0 z1xd#-(m3Nl`l5)PYk>9YB(%vO1me5y2nkJj9VB&SLo+YDMcamK@IH(MqJ&Y{ldqVC zTg2pI&;ANqF>vE61_Kb+Z_DV#o=wfY__7{2~ z#7)n+LC!l!&J^1Dx&Ej%iM{$4f*qX&?&^=)lVn1p_t23L{ill#VQsda-u|j6>qA2h zhlP5B1q^M{3+g-?BSiE>rijhZwWQM$BgLE<3Kms^==l06Fy119k<_8P4WNGt^s8ud z?-9K1j4lp38ZmhP?Ho5nT1&i6fw~&;;FNO5YvB&UNe4PAW_*j(fV~ce_0(qx#1-Qm zM1BV%g!MIXj#JDa&#<$qwzli5ms5L0^&}n5lX>pV9(hrul%MHjOKY|ac9^n zf)#d#l@e@PXIM7Dtes)Y2sW%UOd#0JonaXSi|q^>L$K433LUBiziCLb7rDD*sR>mZ z(O`Jnf>@P;9Gr=H8yDt>qYs!SvcbuW9BB@ebopSo$Fxd=6*SZ{<0dk|%* z)C_6a#q(L>&u#t5Zo`$wf#Ym;wx$KE3cIwJSuP_;U@mE)xl;F$bmvJt!WckoSg&~y zYKGZq3L9{+FtrO7N2|^}>Gz<?>k6vBz@iEFr%V8bO;+=-V&0wMeV+ZvH^%43|3cR2^etI>? z2aN|B)7|p%T2dGNVe5hq=1oWoF(l7nJZ6cPAd;)(foHo85nsAIib8ylJjjl)(Lj@z zHa^4S@>gB)@h_AQ$ph~!#6wy|nk=+F`7}$UJ{caCN3dVqwjdA6D@+#O(!0)atS?d) z{c&sz@3^$cWVi6zAaD-zM%GK6aFOEGU{uLjN1O0%2z6y3_XytnPib{1mQyT$jFw)5 zQY@XKr7yuKvUH-B&JU*JwDf+2B3OPqj4kNdf@wude*gwp{vj=$Olhp2$k^@_U>>#A z^$fI5p8;Wy;IowMU`JS_we;>XxN!cCrFb=+j~p4hI4uQ}W+gbai;Tt>hqG|ILNs&+3n=H2Y)syyExdqfxNob=*KE%(KAmItM0LSsZ0s>RSaObhCaP z)r+o804ZD<2F1SgMgO^9M}ixb?TZt2=){YF-BL!v%|#%#}k8l|7JH&@$fG~fU^t;b8_dcIvY7Jd;(-hk&_sd`1T?sx^Cjx=j_J+&_TdIb%)~svOtFe0 zRCZ@Wrd$AFHo~8xR^wG@B%GiMmdixQM+`#eaX1=Q7*6+);QAVmtK;W3>^&wW3ev(j zW&g{8z=NXkED-U|dioU|957_=DtH_Q&XpCgtXkBJ*0yl8D>0piaRw32tZ1BbnCikf z0|;jAYJUbLg#SICYoV6$v?;ovNiSy=-}F(nIA&rw zytEqjTN=M@A_aeB>JB=#Ip7sDrCF!3HG?t+6t`oQftDq_4CG+Ib{g1Io`4{u&W{}@ zLCLOW9L|3Nhwt>?6nxSy&u*_{Z;AQhB$dfAF-!C#I?*bwPeb(%=TX+>)bm|N4L$$h*Li~MfS=R`#z#XYPFU~c!v@xgx6sL&Yv_e%Jv70rk6s@0jl|o zmY7Ki3lh+JYALga%J(ERoJ>Nya6X1!@y_fNzq$o;I9vDF@11Dky?c?QC!6rthP}sQ ziQciX?wcBo%SjG+u*bqWYUzIU&p?x=#EJcvhoL*wHB>mV)>5rja>Gz7CU2?~vRpk8 zn0VP&dUacfI-at?(Gxs=!#HTx*tv}*BG~Ea`4m!^fS`LeJ*sA0jxwHR^&C76Jzj+) z18C`_G{yNMHF;8s`uGpjQwg6?PssxoQ2)4KQLenMwc>LE^dYU4 z6aGo}S4jWmgeJN_L-$g`Wa1V}{1CU7iF{&++bcd+l$;vuimofP3L6nyuQ$za$?#Mr--;2XiJLX`!?VN)*AfiR z5;qImL(dY6y;vl-Mlr*^wQ0XMhw^lO*FSTrb1FSJKR8&tuaDI0#8 zmUgGGHHH0X?++Q!zsz^(ecHc+ep4>(Td@b4HvwQ3f3}MOzxwz-Yt4xs_L}cv?S{*U zU(@r*z5JI`r)@@@-mEM}|M@ScOxx_gG^2?gc)%M{FU_d-|BT0OPvkV?)f3w91bk+& z;s+kT!862a&P6L83Ac$jUo@b}e<5}8X8-A&CY$9VUoip|Lk+Fszpbf$t2_$m7YxR0MBQ0n()TUOgz_N;R z7IdPu}x^z#M#w@pa?-Jig2DeJtwx1k&_< zyEzbO#+Tb02&Ccr6uy=CR^xjLU-Les@lE>-U*Ihy-2d>QZ8qYNLU0K6UG+)23ep*X z2nxk3IkW!7rPIgUDRp(?aqE=HW6f??ad}CZ`HsQmcbJPy+~$H3PjP-9j^ce|%>{Xd zMXr3ayTn}P%F8zoF7LxV{BV|eUY>it%L$~+soaRUuDl}mTrRJ)w5ZUT=PoQM9`ST} zNip|)Qn@?N?I}+hn^f{l(qb*Su2qBJ=E${6$>S;PNES z4S=f*#mZdVyu$MG!s0pR(lQiqbMBJT+){2{-ojjW$unS@D|dsbd7Qh<m8dQ$E zXXbOy%#WxCp!~vl=!HUblcr~k$C84Aa!7rSs~CM`qAvW_vkC~l$&bcnl2oR*qQ*5SD4p5Y+W|x#}<21K0KRVC#AM(nbC1tMi=n|U5 zneIf@GtX5v2l(UC77k7unR6ePMJjAAf>yiohxg%z4P!cuYiUWD8yCI5ap#^(Hm#f+ zHW!JxuA)+Inm$nS%I0|HL9Mir@Gvw#E8DE21BS7~KAMU26qkESi8$HGu#N+b$t>UX ztS1i|`W&;2tfVk?YW9fq;pv3(@IwiGg2rPihA&kahUn%5hzwSYAp%t&Yjzful)ISW zAx550a+Q^plqHRQK53q-ygYA?3nmxFbZC+j!bloBIDheC!i%golS+gu(yER&;o=Z6HP%>~aVIp>xXI$fBTo_Icqs0XE0P*_}8 zJ~tOyf$1x0Y_X@PXz`PaF>C0b|8+@Bd?lqxi=P~t3LY)(w1|z`K zQ$%w`%2V{*(Xu7#yg z7+1ayNbQUt9Mrnbl5or^gP-%5XD@QQ$_ZJcTbftqF3c+m>Ndk;&HzImVMY<1a@ZCZ z8DC zT!PKe=KSIkbCgA3$#QdHF`LtXZ8nEz6CwlCL>E{gjT##ZW6i_nna43>0bB1WxX)Zz z4yWf?Phpu$uW}l7CJj*Y6D6h0eR;Bv8O4U0bt_{YSDNRZd!Ly;*|-B|5%a3K0KSwt zB}g-s%9c`DG9D<3zgkiAqty9jW(H5mcNOG$ieQVzq>V{qg`8AKw`1mUG)Gc7Wk$JC|28CDo*B+H7oe%(YYlH z&}l`bq26GfyDZqp`jyiE;k%6d@&EAE{9E#oaV&&RzHm6*!}Y|s7rw2p%}km(6F9%< zaqq`SWA~aHlMkO8PVwA1fdCz@Scq!|uIru-1P*~nJFb^ND6SY=zaW-^E1gJ2U6(-g~oGMS>sv@F=@y=9_cQoH;Xd_uiRvoe&Hd1KkC>2pR`{Kj;IX-v(W# z`g*hZbE;ozHV4tpG~Q>J2krYxv-ug&`L8#d&lCN2v-ujdzxc~$^I_1Bfj$BHJm@+o zC3zO`ojsv^%OT&jgS~xZ6X6J(QDenslMcewL@!=bQEC@Hqxi&9c}|o_C?|qDP(L2*Q*p3<9iK~(yBAa?f}wN%e1EVnF)(TSp1LeQ zVG!sTgBXQ?JO>>|Nry+730va{{7F>m+XQ44#`ZdrMWVSu<2wi}LI7+^1p7_| z!Ax~^)P+%pH)_~kOWQ(qxvJy#MG>&+8uzJkoWc zd$s3WZ{WASeoT=mbX`s3L-X@wt=as^jVNfpO%D#K*-(#vNlk>d`InYMLH|-^a$8_& zvFFavpnvI9DBxe358S$=AB;1Dy)=w71M?7Hg|252zyBqga9Y=7OV^ZE&{mkTsY<}4 zma4=+;0M>*%98^s40X>!_v{y&%_;I7#^bzya;Rsf-7lK!w3eR4yu2HEHP{n6*L&W- z+H;}%k*-GraX`1Be*R*!`FGL*`yQeBc_A_4)&{iwX3GvJqr!_mlk#J$Z7LE)8wzYz=^B+!F?3Ir)j6> z-BZx@^e>vtJ5Z)XbMX?eu4nM7t47Qi{R&td82yPBc)b7e;ki?{)NVOp+WZn6YipCn zJBI7VpELO1UZGP>1w)UL4;~>OT<97P?p`GyoQDs_&h<_NPX$i<`-5YE;0&;|oRJtH ze?J6WPcAl_JCQeY?eI{~9C}FQt+eHVEe~vYV9Ntr9@z50mIt;xu;qdOLl11;VvO3< z4|7Vba~Y3nI^9S2+(=SRxylt+`E6VtG-7yAG?^hTKeuKw`koot=YM9(h$7cXp0(-m zf>Phl4W;{SN^)=gmwz?OR9>h1cqH2EpcK2pP}=vPMEe+&Wb5VcY#%*?P@-p6N{EDa zBoEU(iVy}yGb_&Rr7?A|ohip^}b|8mBK?}eu6AN;N%ge~j- zuCZeg|F6q<1kq`e%eQbn{@9MAp4t9qB*f$5Ru~H#B+hh>=_#farYlSzW4gw4o#{oU zmzZua-DIj>p>zr{4Ks}~jWeBNdWva<=?c@wn65EhXL^z8C8irpH<_v%xP7K!rZJ{* zrgKbBF|9CNVfq-;HKyxKFEYKvbc5+8Q*|S^&os<5#x%~-+$~Wx0DY$(gaQ(H{-*fe_ zFN789kNN_^R}b7VaB$$2JBNK&_QrRd-Rav7MWae8X14pU{@gg2yLPF8f!7WMXhCQ3 z>Bd5gl3t(hh2Rho-HHaT*S)6&=l5Oj-{wyepZk%$I}qr47u)7I5y(Z-<;FrG6B&yEs4dp&4}#Vk zrOY9HHk?nPs1FO4$M#3|kA@>T`FDJ8I3k}cPJqhAd_9t_CW{&Dr{{4RUkyjHscZxX zFLbhuW3xk99OwXCXyglYoE$0S7jh}IK;k&WOg$*fIM|G2E7W*r4awS>QfjeUE|nX# zNTYNb!2^+c68)$_!e~krlC?$AN5|wi@3ykZTHQkI#G=BXVLn&Nq;X17S9`QYdtggs zWM5yy5HTIYusAzPL`B$r^7(NpWTNzo*cpdarm-;qk8i_{u9j)YUr#NMXY1*+eqE zGB>=|3Nay(>Mr%i%VK2ccf^O9RlOEkKexJ#PW<#c>!+QzTaiK1d)1)3@gaPUalID} z>~_Z28Q1&Ez@`}2Yae;pJAQ!i@NLK->HTBS8a`S-Q`v=D7_8MI8|04SW<1u!C-pd8nj{zV( zn-3X&y$_4*RgACPY;e8zitJ5{$5#xl_gs;^UGR4rT<@bI8wcKLypA!ydzw7p!JiZT zd>?08QHuU(9@$@U;SVt`_kF_uUf@1;joQ6o5EOO#pvGU?x-Vl`eS`JL{iNVO)cjYq z?(+oS1m3B?Uj1rnJugvnUl9$4dg-CsPWp%Rm3q5+_6truVcz)33*onQY`b zJr8XFXAb}mb(o(M9(qoC;CYRQ)CT)mpTVGwWe@&$1Ha*NbJRzGcUl);(D+Uj+HeYe z%R~Q$2mUA4BloW@ZFn&3G>+Rn@S8pGLmv2?2cGi48y@(>z(cs+hFyMm+=Ks99{5*1 z@TY-y^8a%l{Qva8cY0hOrabU@JP1?2Vy=GO=fPj_z*ju*4*(yw`OQ3l;H=q$|63mT z(;oQmJ@BRnex>g3<@0$b@J{n^zXv`8ywiBm|7qSy&s#n43m*8I2mWb|W4+&OR**g? zMfS6V`v)(-nB9#V=8Ym5DVoUbsD;=wD_eqg9R(~wv1&0{N*AC8aZj0A-E`Y3l*<)cHkv9I(zcH1uwvFj zahJ;ZxZ!;8L0zrh$Y$Zobf%igQOI@GsuwMUG?oy@X<6yAl`E7Nk_9VWFIQ_;vayVC zY8@(vh|XAN4jPlZr9+`DL<&~VDE%mCr5nZK8Hm_Q7C7q;RzxF{-C~zzO`S*_owO#8 zO;{FW@!)DrPBSQ+H3Ev`!|*XD7ipao@4T(V4@LV1?FF682{h zS&lIFRJoRoADNvQKWy!d?v38AD&^BQ;DD;a_2r^Pf#AhV5ngj}XX}|l!GTy7!XZnk z${Eb02U|7!qNCK^3IfOL*$y?iVz~rCOh)t#LD-qLC>dp0c|^n?JZRNwsbr~b-vO0b z&O0sAE*GmQj9r^kth7uaeZ^z}!P*WeOOfObPp59fWfaYi&^(8BYv$xpt3B6F9z9Gm?Ib;}D{JOB)MGQiYSc1m%$ZEa zA&V{{=G*~kbuw|C$}EbFH0HuxmA3cx9)M%3nbeYFKoH-bq-kx{%2_;{FZ25c&ep8d za*{laK-RNP+uGAC+bE@QmC_xuV3N$-`;XkaaBOtRa@wK?3LSH`ldhF5v-)Wu>t*L; zquq+66Vd~Ms~Sc)gBlt`Y@v}bwniMv%{vwj2%b)+9SG#o4d>*stc6;Q{YkSOJ2d1# zY@_7crs%gMQm$!73*KB~+tZuIz_u{iVKppIhaGfbq)S$*QM5Q{GERLPLEm0vqLvU# zERMKcr8-P~9k=ah4&A0T?7%DxT>98pr-8#fcKQ~6DS0e*dZ)#qH+#B@w=5hGymZ+D z#rrpi4R^3v8JgN~AU2$ML1G^(-(DH}M)`FZr&!D3G)2v&uDI`gNbCp6}Md{~9 zMUjo>%ABcXuq#KA=qgITW8f}pe@8Qm{L8hNhBhe~9p)#@NJyS9nJ|uqD~jc#@M~S9 zcNu_X78*HlCQJ0f1Q+wAY+3Tjg@tNnNlMsd%t#&{lq|tmi|ukbMN#T4h1F8dl+C@? zfAdZIdOm#cc&*b8JV#Oz`Z7~&eG{)e-G~Bi@mlg@EH9L|&srjKpDn?z8&Lc!&b*}maxN&8_BScvwp}N;{Yjt$_=x<|Y`@TX zJP%PK8_9PfFXxAOU?d{)at*E`E3WKSRB}{U@I?0zxV7-=1Xt2=;4a=%h}DS>g(w|Bm5s`%C17{t;~3 zZ1NI!(8q=gpV8Kyzk>f06*PCHec3M#v3!^vOY4u4$cbH_07m^6dD(}KeV@1y7fqG6 zB=SOgP;Zx48_aykP-#b0h@6xMAxxOGAKoxEArJYmscwA_bNjItVlpBhap7buC85`_ zyg}RFhq!&WUXc^dS9!=6Sw16-Ox^NyF5||C^+zl}`y%p(J>*~hLnHX+7m+{C^6q{< z%<^HEaZ(g|zlZ$Ceql~5B%65bPi}wA}82c5BUPiUoDJG-SSj! zU3+Z253~GwzX5Loc_ltFf93b*3ScsK4X61Yd&bbGWZ`k8yZvF{o#g*a7Z#*1Q0iNOE> literal 0 HcmV?d00001 diff --git a/llmc/bpe_tokenizer.h b/llmc/bpe_tokenizer.h new file mode 100644 index 0000000..2de2878 --- /dev/null +++ b/llmc/bpe_tokenizer.h @@ -0,0 +1,407 @@ +/* +BPE Tokenizer loaded from GGUF metadata. +Supports SentencePiece-style (LLaMA 2) and tiktoken-style (LLaMA 3) vocabularies. +*/ +#ifndef BPE_TOKENIZER_H +#define BPE_TOKENIZER_H + +#include +#include +#include +#include +#include "gguf.h" + +// ---------------------------------------------------------------------------- +// BPE Tokenizer + +#define BPE_MAX_VOCAB 200000 +#define BPE_MAX_MERGES 200000 +#define BPE_MAX_TOKEN 512 + +typedef struct { + char **tokens; // vocab: token strings (null-terminated) + float *scores; // token scores (for SentencePiece) + int *token_type; // token type flags + int vocab_size; + int bos_id; + int eos_id; + + // merge rules (for explicit BPE like tiktoken) + char **merge_left; + char **merge_right; + int n_merges; + int has_merges; // 0 = SentencePiece score-based, 1 = explicit merges + + // byte-level fallback table + int byte_token[256]; // byte value -> token id, -1 if none +} BPETokenizer; + +// ---------------------------------------------------------------------------- +// UTF-8 helpers + +static int utf8_char_len(unsigned char c) { + if (c < 0x80) return 1; + if (c < 0xE0) return 2; + if (c < 0xF0) return 3; + return 4; +} + +// Check if s starts with the UTF-8 encoding of U+2581 (▁, the SentencePiece space) +// U+2581 encodes as E2 96 81 +static int is_sp_space(const char *s) { + return (unsigned char)s[0] == 0xE2 && + (unsigned char)s[1] == 0x96 && + (unsigned char)s[2] == 0x81; +} + +// ---------------------------------------------------------------------------- +// Load from GGUF + +static void bpe_tokenizer_init(BPETokenizer *tok, GGUFContext *ctx) { + memset(tok, 0, sizeof(*tok)); + + // bos/eos + tok->bos_id = (int)gguf_get_u64(ctx, "tokenizer.ggml.bos_token_id", 1); + tok->eos_id = (int)gguf_get_u64(ctx, "tokenizer.ggml.eos_token_id", 2); + + // tokens array + GGUFValue *tokens_val = gguf_find_kv(ctx, "tokenizer.ggml.tokens"); + if (!tokens_val || tokens_val->type != GGUF_TYPE_ARRAY) { + fprintf(stderr, "BPE: no tokenizer.ggml.tokens in GGUF\n"); + exit(EXIT_FAILURE); + } + tok->vocab_size = (int)tokens_val->arr.count; + tok->tokens = (char **)calloc(tok->vocab_size, sizeof(char *)); + for (int i = 0; i < tok->vocab_size; i++) { + tok->tokens[i] = strdup(tokens_val->arr.items[i].str); + } + + // scores (optional) + GGUFValue *scores_val = gguf_find_kv(ctx, "tokenizer.ggml.scores"); + if (scores_val && scores_val->type == GGUF_TYPE_ARRAY) { + tok->scores = (float *)calloc(tok->vocab_size, sizeof(float)); + for (int i = 0; i < tok->vocab_size && (uint64_t)i < scores_val->arr.count; i++) { + GGUFValue *sv = &scores_val->arr.items[i]; + tok->scores[i] = (sv->type == GGUF_TYPE_FLOAT32) ? sv->f32 : 0.0f; + } + } + + // token types (optional) + GGUFValue *type_val = gguf_find_kv(ctx, "tokenizer.ggml.token_type"); + if (type_val && type_val->type == GGUF_TYPE_ARRAY) { + tok->token_type = (int *)calloc(tok->vocab_size, sizeof(int)); + for (int i = 0; i < tok->vocab_size && (uint64_t)i < type_val->arr.count; i++) { + GGUFValue *tv = &type_val->arr.items[i]; + tok->token_type[i] = (tv->type == GGUF_TYPE_INT32) ? tv->i32 : 0; + } + } + + // explicit merges (optional, tiktoken-style) + GGUFValue *merges_val = gguf_find_kv(ctx, "tokenizer.ggml.merges"); + if (merges_val && merges_val->type == GGUF_TYPE_ARRAY && merges_val->arr.count > 0) { + tok->has_merges = 1; + tok->n_merges = (int)merges_val->arr.count; + tok->merge_left = (char **)calloc(tok->n_merges, sizeof(char *)); + tok->merge_right = (char **)calloc(tok->n_merges, sizeof(char *)); + for (int i = 0; i < tok->n_merges; i++) { + const char *entry = merges_val->arr.items[i].str; + // format: "left right" + const char *sp = strchr(entry, ' '); + if (sp) { + size_t llen = (size_t)(sp - entry); + tok->merge_left[i] = (char *)malloc(llen + 1); + memcpy(tok->merge_left[i], entry, llen); + tok->merge_left[i][llen] = '\0'; + tok->merge_right[i] = strdup(sp + 1); + } else { + tok->merge_left[i] = strdup(entry); + tok->merge_right[i] = strdup(""); + } + } + } + + // build byte fallback table + memset(tok->byte_token, -1, sizeof(tok->byte_token)); + // Look for tokens of the form <0xNN> + for (int i = 0; i < tok->vocab_size; i++) { + const char *t = tok->tokens[i]; + unsigned int byte_val; + if (sscanf(t, "<0x%02X>", &byte_val) == 1 && byte_val < 256) { + tok->byte_token[byte_val] = i; + } + } +} + +// ---------------------------------------------------------------------------- +// Decode: token_id -> string + +static const char *bpe_decode(BPETokenizer *tok, int token_id) { + if (token_id < 0 || token_id >= tok->vocab_size) return ""; + return tok->tokens[token_id]; +} + +// Decode a token, converting SentencePiece ▁ to space +static void bpe_decode_piece(BPETokenizer *tok, int token_id, char *out, int out_size) { + const char *piece = bpe_decode(tok, token_id); + int j = 0; + const char *p = piece; + while (*p && j < out_size - 1) { + if (is_sp_space(p) && p[3] == '\0' || is_sp_space(p)) { + out[j++] = ' '; + p += 3; // skip 3-byte UTF-8 sequence + } else { + out[j++] = *p++; + } + } + out[j] = '\0'; +} + +// ---------------------------------------------------------------------------- +// Symbol list for BPE encoding + +typedef struct BPESym { + int token_id; // -1 if raw byte + char text[BPE_MAX_TOKEN]; + int prev, next; // doubly linked list indices +} BPESym; + +// Find token id by exact string match +static int bpe_find_token(BPETokenizer *tok, const char *s) { + // linear scan; for large vocabs a hash table would be better + for (int i = 0; i < tok->vocab_size; i++) { + if (strcmp(tok->tokens[i], s) == 0) return i; + } + return -1; +} + +// Find best merge pair by score (SentencePiece style) +static int bpe_find_best_pair(BPETokenizer *tok, BPESym *syms, int n, int *out_i) { + float best_score = -1e38f; + int best_idx = -1; + int merged_id = -1; + + for (int i = 0; i < n - 1; ) { + if (syms[i].next < 0) { i++; continue; } + int j = syms[i].next; + // try merging syms[i] and syms[j] + char merged[BPE_MAX_TOKEN * 2]; + int mlen = snprintf(merged, sizeof(merged), "%s%s", syms[i].text, syms[j].text); + if (mlen < (int)sizeof(merged)) { + int tid = bpe_find_token(tok, merged); + if (tid >= 0) { + float score = tok->scores ? tok->scores[tid] : 0.0f; + if (score > best_score) { + best_score = score; + best_idx = i; + merged_id = tid; + } + } + } + i = j; + } + *out_i = best_idx; + return merged_id; +} + +// BPE encode using merge rules (tiktoken-style) +static int bpe_merge_rank(BPETokenizer *tok, const char *left, const char *right) { + for (int i = 0; i < tok->n_merges; i++) { + if (strcmp(tok->merge_left[i], left) == 0 && + strcmp(tok->merge_right[i], right) == 0) { + return i; + } + } + return INT32_MAX; +} + +// Main encode function +// Returns number of tokens. out_tokens must have capacity for len+2 tokens. +static int bpe_encode(BPETokenizer *tok, const char *text, int add_bos, + int *out_tokens, int max_tokens) { + int n_out = 0; + if (add_bos && n_out < max_tokens) { + out_tokens[n_out++] = tok->bos_id; + } + + size_t text_len = strlen(text); + if (text_len == 0) return n_out; + + // SentencePiece: prepend ▁ (U+2581 = E2 96 81) to represent leading space + // We handle this by checking if first token has ▁ prefix in vocab + + // Build initial symbol list from UTF-8 characters + // (or byte-level for unknown chars) + // We use a simple array with a linked list overlay + int max_syms = (int)(text_len + 4); + BPESym *syms = (BPESym *)calloc(max_syms, sizeof(BPESym)); + if (!syms) { fprintf(stderr, "BPE: OOM\n"); exit(EXIT_FAILURE); } + + int n_syms = 0; + + // SentencePiece: treat text as having a space prepended + // Check if vocab uses ▁ convention + int sp_style = (bpe_find_token(tok, "\xE2\x96\x81") >= 0 || + bpe_find_token(tok, "\xE2\x96\x81 ") >= 0); + + if (sp_style) { + // Prepend ▁ to first character + const char *p = text; + int first = 1; + while (*p && n_syms < max_syms) { + int clen = utf8_char_len((unsigned char)*p); + char buf[BPE_MAX_TOKEN]; + int blen = 0; + if (first) { + // prepend ▁ (3 bytes) + buf[0] = '\xE2'; buf[1] = '\x96'; buf[2] = '\x81'; + blen = 3; + first = 0; + } + for (int k = 0; k < clen && blen < (int)sizeof(buf) - 1; k++) { + buf[blen++] = p[k]; + } + buf[blen] = '\0'; + int tid = bpe_find_token(tok, buf); + if (tid < 0) { + // try without ▁ prefix + char buf2[BPE_MAX_TOKEN]; + strncpy(buf2, buf + (blen > clen ? 3 : 0), sizeof(buf2)); + tid = bpe_find_token(tok, buf2); + if (tid < 0 && blen == 1) { + // byte fallback + tid = tok->byte_token[(unsigned char)buf[0]]; + } + } + syms[n_syms].token_id = tid; + strncpy(syms[n_syms].text, buf, BPE_MAX_TOKEN - 1); + syms[n_syms].prev = n_syms - 1; + syms[n_syms].next = n_syms + 1; + n_syms++; + p += clen; + } + } else { + // Byte-level BPE (GPT-2 style or tiktoken) + const char *p = text; + while (*p && n_syms < max_syms) { + int clen = utf8_char_len((unsigned char)*p); + char buf[BPE_MAX_TOKEN]; + int blen = 0; + for (int k = 0; k < clen && blen < (int)sizeof(buf) - 1; k++) { + buf[blen++] = p[k]; + } + buf[blen] = '\0'; + int tid = bpe_find_token(tok, buf); + if (tid < 0) { + // byte fallback + for (int k = 0; k < blen && n_syms < max_syms; k++) { + syms[n_syms].token_id = tok->byte_token[(unsigned char)buf[k]]; + syms[n_syms].text[0] = buf[k]; + syms[n_syms].text[1] = '\0'; + syms[n_syms].prev = n_syms - 1; + syms[n_syms].next = n_syms + 1; + n_syms++; + } + p += clen; + continue; + } + syms[n_syms].token_id = tid; + strncpy(syms[n_syms].text, buf, BPE_MAX_TOKEN - 1); + syms[n_syms].prev = n_syms - 1; + syms[n_syms].next = n_syms + 1; + n_syms++; + p += clen; + } + } + + if (n_syms > 0) { + syms[0].prev = -1; + syms[n_syms - 1].next = -1; + } + + // BPE merge loop + if (!tok->has_merges) { + // SentencePiece score-based merges + while (1) { + int best_i; + int merged_id = bpe_find_best_pair(tok, syms, n_syms, &best_i); + if (merged_id < 0 || best_i < 0) break; + int j = syms[best_i].next; + // merge best_i and j into best_i + char merged[BPE_MAX_TOKEN * 2]; + snprintf(merged, sizeof(merged), "%s%s", syms[best_i].text, syms[j].text); + strncpy(syms[best_i].text, merged, BPE_MAX_TOKEN - 1); + syms[best_i].token_id = merged_id; + // unlink j + syms[best_i].next = syms[j].next; + if (syms[j].next >= 0) syms[syms[j].next].prev = best_i; + } + } else { + // Explicit merge rules: find lowest-rank pair, merge, repeat + while (1) { + int best_rank = INT32_MAX; + int best_i = -1; + int best_j = -1; + for (int i = 0; i >= 0 && syms[i].next >= 0; ) { + int j = syms[i].next; + if (j < 0) break; + int rank = bpe_merge_rank(tok, syms[i].text, syms[j].text); + if (rank < best_rank) { + best_rank = rank; + best_i = i; + best_j = j; + } + i = j; + } + if (best_i < 0 || best_rank == INT32_MAX) break; + // merge best_i and best_j + char merged[BPE_MAX_TOKEN * 2]; + snprintf(merged, sizeof(merged), "%s%s", syms[best_i].text, syms[best_j].text); + strncpy(syms[best_i].text, merged, BPE_MAX_TOKEN - 1); + int tid = bpe_find_token(tok, merged); + syms[best_i].token_id = (tid >= 0) ? tid : -1; + syms[best_i].next = syms[best_j].next; + if (syms[best_j].next >= 0) syms[syms[best_j].next].prev = best_i; + } + } + + // Collect tokens from linked list + for (int i = 0; i >= 0 && n_out < max_tokens; ) { + int tid = syms[i].token_id; + if (tid >= 0) { + out_tokens[n_out++] = tid; + } else { + // byte fallback + const char *t = syms[i].text; + while (*t && n_out < max_tokens) { + int bt = tok->byte_token[(unsigned char)*t]; + if (bt >= 0) out_tokens[n_out++] = bt; + t++; + } + } + if (syms[i].next < 0) break; + i = syms[i].next; + } + + free(syms); + return n_out; +} + +// ---------------------------------------------------------------------------- +// Free + +static void bpe_tokenizer_free(BPETokenizer *tok) { + for (int i = 0; i < tok->vocab_size; i++) free(tok->tokens[i]); + free(tok->tokens); + free(tok->scores); + free(tok->token_type); + if (tok->has_merges) { + for (int i = 0; i < tok->n_merges; i++) { + free(tok->merge_left[i]); + free(tok->merge_right[i]); + } + free(tok->merge_left); + free(tok->merge_right); + } +} + +#endif // BPE_TOKENIZER_H diff --git a/llmc/gguf.h b/llmc/gguf.h new file mode 100644 index 0000000..9278ef8 --- /dev/null +++ b/llmc/gguf.h @@ -0,0 +1,460 @@ +/* +GGUF file format parser. +Supports GGUF v3, tensor types F32/F16/Q4_0/Q8_0. +*/ +#ifndef GGUF_H +#define GGUF_H + +#include +#include +#include +#include +#include + +// ---------------------------------------------------------------------------- +// GGUF types + +#define GGUF_MAGIC 0x46554747 // "GGUF" +#define GGUF_ALIGNMENT 32 + +typedef enum { + GGUF_TYPE_UINT8 = 0, + GGUF_TYPE_INT8 = 1, + GGUF_TYPE_UINT16 = 2, + GGUF_TYPE_INT16 = 3, + GGUF_TYPE_UINT32 = 4, + GGUF_TYPE_INT32 = 5, + GGUF_TYPE_FLOAT32 = 6, + GGUF_TYPE_BOOL = 7, + GGUF_TYPE_STRING = 8, + GGUF_TYPE_ARRAY = 9, + GGUF_TYPE_UINT64 = 10, + GGUF_TYPE_INT64 = 11, + GGUF_TYPE_FLOAT64 = 12, +} GGUFValueType; + +typedef enum { + GGML_TYPE_F32 = 0, + GGML_TYPE_F16 = 1, + GGML_TYPE_Q4_0 = 2, + GGML_TYPE_Q4_1 = 3, + GGML_TYPE_Q8_0 = 8, +} GGMLType; + +#define GGML_TYPE_COUNT 16 + +static const int ggml_blk_size[GGML_TYPE_COUNT] = { + [GGML_TYPE_F32] = 1, + [GGML_TYPE_F16] = 1, + [GGML_TYPE_Q4_0] = 32, + [GGML_TYPE_Q4_1] = 32, + [GGML_TYPE_Q8_0] = 32, +}; + +static const int ggml_type_size[GGML_TYPE_COUNT] = { + [GGML_TYPE_F32] = 4, + [GGML_TYPE_F16] = 2, + [GGML_TYPE_Q4_0] = 18, // 2 (f16 scale) + 16 (32 x 4bit) + [GGML_TYPE_Q4_1] = 20, // 2 + 2 + 16 + [GGML_TYPE_Q8_0] = 34, // 2 (f16 scale) + 32 (int8 quants) +}; + +// ---------------------------------------------------------------------------- +// GGUF value (tagged union) + +typedef struct GGUFValue GGUFValue; + +typedef struct { + GGUFValueType type; + uint64_t count; + GGUFValue *items; +} GGUFArray; + +struct GGUFValue { + GGUFValueType type; + union { + uint8_t u8; + int8_t i8; + uint16_t u16; + int16_t i16; + uint32_t u32; + int32_t i32; + float f32; + uint8_t bool_; + uint64_t u64; + int64_t i64; + double f64; + char *str; // null-terminated, heap allocated + GGUFArray arr; + }; +}; + +// ---------------------------------------------------------------------------- +// GGUF metadata key-value pair + +typedef struct { + char *key; // null-terminated + GGUFValue value; +} GGUFMetaKV; + +// ---------------------------------------------------------------------------- +// GGUF tensor info + +#define GGUF_MAX_DIMS 4 + +typedef struct { + char *name; + uint32_t n_dims; + uint64_t dims[GGUF_MAX_DIMS]; + GGMLType type; + uint64_t offset; // offset from start of tensor data section + void *data; // pointer into mmap'd or malloc'd buffer + size_t data_size; +} GGUFTensor; + +// ---------------------------------------------------------------------------- +// GGUF context + +typedef struct { + uint32_t version; + uint64_t n_tensors; + uint64_t n_kv; + GGUFMetaKV *kv; + GGUFTensor *tensors; + + // raw file data (malloc'd) + void *data; + size_t data_size; + size_t data_offset; // offset in file where tensor data begins +} GGUFContext; + +// ---------------------------------------------------------------------------- +// Internal read helpers + +static void gguf_read_check(FILE *f, void *dst, size_t n, const char *label) { + if (fread(dst, 1, n, f) != n) { + fprintf(stderr, "GGUF: failed to read %s\n", label); + exit(EXIT_FAILURE); + } +} + +static char *gguf_read_string(FILE *f) { + uint64_t len; + gguf_read_check(f, &len, sizeof(len), "string length"); + char *s = (char *)malloc(len + 1); + if (!s) { fprintf(stderr, "GGUF: OOM\n"); exit(EXIT_FAILURE); } + gguf_read_check(f, s, len, "string data"); + s[len] = '\0'; + return s; +} + +static void gguf_read_value(FILE *f, GGUFValue *v, GGUFValueType type); + +static void gguf_read_array(FILE *f, GGUFArray *arr) { + uint32_t elem_type; + gguf_read_check(f, &elem_type, sizeof(elem_type), "array elem type"); + arr->type = (GGUFValueType)elem_type; + gguf_read_check(f, &arr->count, sizeof(arr->count), "array count"); + arr->items = (GGUFValue *)calloc(arr->count, sizeof(GGUFValue)); + if (!arr->items) { fprintf(stderr, "GGUF: OOM\n"); exit(EXIT_FAILURE); } + for (uint64_t i = 0; i < arr->count; i++) { + gguf_read_value(f, &arr->items[i], arr->type); + } +} + +static void gguf_read_value(FILE *f, GGUFValue *v, GGUFValueType type) { + v->type = type; + switch (type) { + case GGUF_TYPE_UINT8: gguf_read_check(f, &v->u8, 1, "u8"); break; + case GGUF_TYPE_INT8: gguf_read_check(f, &v->i8, 1, "i8"); break; + case GGUF_TYPE_UINT16: gguf_read_check(f, &v->u16, 2, "u16"); break; + case GGUF_TYPE_INT16: gguf_read_check(f, &v->i16, 2, "i16"); break; + case GGUF_TYPE_UINT32: gguf_read_check(f, &v->u32, 4, "u32"); break; + case GGUF_TYPE_INT32: gguf_read_check(f, &v->i32, 4, "i32"); break; + case GGUF_TYPE_FLOAT32: gguf_read_check(f, &v->f32, 4, "f32"); break; + case GGUF_TYPE_BOOL: gguf_read_check(f, &v->bool_,1, "bool"); break; + case GGUF_TYPE_UINT64: gguf_read_check(f, &v->u64, 8, "u64"); break; + case GGUF_TYPE_INT64: gguf_read_check(f, &v->i64, 8, "i64"); break; + case GGUF_TYPE_FLOAT64: gguf_read_check(f, &v->f64, 8, "f64"); break; + case GGUF_TYPE_STRING: v->str = gguf_read_string(f); break; + case GGUF_TYPE_ARRAY: gguf_read_array(f, &v->arr); break; + default: + fprintf(stderr, "GGUF: unknown value type %d\n", type); + exit(EXIT_FAILURE); + } +} + +// ---------------------------------------------------------------------------- +// Main load function + +static GGUFContext *gguf_load(const char *path) { + FILE *f = fopen(path, "rb"); + if (!f) { + fprintf(stderr, "GGUF: cannot open '%s'\n", path); + exit(EXIT_FAILURE); + } + + // magic + uint32_t magic; + gguf_read_check(f, &magic, 4, "magic"); + if (magic != GGUF_MAGIC) { + fprintf(stderr, "GGUF: bad magic 0x%08X (expected 0x%08X)\n", magic, GGUF_MAGIC); + exit(EXIT_FAILURE); + } + + GGUFContext *ctx = (GGUFContext *)calloc(1, sizeof(GGUFContext)); + if (!ctx) { fprintf(stderr, "GGUF: OOM\n"); exit(EXIT_FAILURE); } + + gguf_read_check(f, &ctx->version, 4, "version"); + gguf_read_check(f, &ctx->n_tensors, 8, "n_tensors"); + gguf_read_check(f, &ctx->n_kv, 8, "n_kv"); + + // metadata key-value pairs + ctx->kv = (GGUFMetaKV *)calloc(ctx->n_kv, sizeof(GGUFMetaKV)); + if (!ctx->kv) { fprintf(stderr, "GGUF: OOM\n"); exit(EXIT_FAILURE); } + + for (uint64_t i = 0; i < ctx->n_kv; i++) { + ctx->kv[i].key = gguf_read_string(f); + uint32_t vtype; + gguf_read_check(f, &vtype, 4, "kv value type"); + gguf_read_value(f, &ctx->kv[i].value, (GGUFValueType)vtype); + } + + // tensor info + ctx->tensors = (GGUFTensor *)calloc(ctx->n_tensors, sizeof(GGUFTensor)); + if (!ctx->tensors) { fprintf(stderr, "GGUF: OOM\n"); exit(EXIT_FAILURE); } + + for (uint64_t i = 0; i < ctx->n_tensors; i++) { + GGUFTensor *t = &ctx->tensors[i]; + t->name = gguf_read_string(f); + gguf_read_check(f, &t->n_dims, 4, "n_dims"); + for (uint32_t d = 0; d < t->n_dims; d++) { + gguf_read_check(f, &t->dims[d], 8, "dim"); + } + uint32_t ttype; + gguf_read_check(f, &ttype, 4, "tensor type"); + t->type = (GGMLType)ttype; + gguf_read_check(f, &t->offset, 8, "offset"); + } + + // tensor data starts at next alignment boundary + long header_end = ftell(f); + long align_end = ((header_end + GGUF_ALIGNMENT - 1) / GGUF_ALIGNMENT) * GGUF_ALIGNMENT; + ctx->data_offset = (size_t)align_end; + + // compute total tensor data size + uint64_t max_end = 0; + for (uint64_t i = 0; i < ctx->n_tensors; i++) { + GGUFTensor *t = &ctx->tensors[i]; + // compute number of elements + uint64_t ne = 1; + for (uint32_t d = 0; d < t->n_dims; d++) ne *= t->dims[d]; + // compute bytes + int bsz = ggml_blk_size[t->type]; + int tsz = ggml_type_size[t->type]; + uint64_t n_blocks = (ne + bsz - 1) / bsz; + t->data_size = (size_t)(n_blocks * tsz); + uint64_t end = t->offset + t->data_size; + if (end > max_end) max_end = end; + } + ctx->data_size = (size_t)max_end; + + // load tensor data + ctx->data = malloc(ctx->data_size); + if (!ctx->data && ctx->data_size > 0) { + fprintf(stderr, "GGUF: OOM for tensor data (%zu bytes)\n", ctx->data_size); + exit(EXIT_FAILURE); + } + fseek(f, (long)ctx->data_offset, SEEK_SET); + size_t got = fread(ctx->data, 1, ctx->data_size, f); + if (got != ctx->data_size) { + fprintf(stderr, "GGUF: partial tensor data read (%zu / %zu)\n", got, ctx->data_size); + exit(EXIT_FAILURE); + } + fclose(f); + + // set data pointers + for (uint64_t i = 0; i < ctx->n_tensors; i++) { + GGUFTensor *t = &ctx->tensors[i]; + t->data = (char *)ctx->data + t->offset; + } + + return ctx; +} + +// ---------------------------------------------------------------------------- +// Metadata lookup helpers + +static GGUFValue *gguf_find_kv(GGUFContext *ctx, const char *key) { + for (uint64_t i = 0; i < ctx->n_kv; i++) { + if (strcmp(ctx->kv[i].key, key) == 0) { + return &ctx->kv[i].value; + } + } + return NULL; +} + +static const char *gguf_get_str(GGUFContext *ctx, const char *key, const char *def) { + GGUFValue *v = gguf_find_kv(ctx, key); + if (!v || v->type != GGUF_TYPE_STRING) return def; + return v->str; +} + +static uint64_t gguf_get_u64(GGUFContext *ctx, const char *key, uint64_t def) { + GGUFValue *v = gguf_find_kv(ctx, key); + if (!v) return def; + switch (v->type) { + case GGUF_TYPE_UINT8: return v->u8; + case GGUF_TYPE_INT8: return (uint64_t)v->i8; + case GGUF_TYPE_UINT16: return v->u16; + case GGUF_TYPE_INT16: return (uint64_t)v->i16; + case GGUF_TYPE_UINT32: return v->u32; + case GGUF_TYPE_INT32: return (uint64_t)v->i32; + case GGUF_TYPE_UINT64: return v->u64; + case GGUF_TYPE_INT64: return (uint64_t)v->i64; + default: return def; + } +} + +static float gguf_get_f32(GGUFContext *ctx, const char *key, float def) { + GGUFValue *v = gguf_find_kv(ctx, key); + if (!v) return def; + if (v->type == GGUF_TYPE_FLOAT32) return v->f32; + if (v->type == GGUF_TYPE_FLOAT64) return (float)v->f64; + return def; +} + +static GGUFTensor *gguf_find_tensor(GGUFContext *ctx, const char *name) { + for (uint64_t i = 0; i < ctx->n_tensors; i++) { + if (strcmp(ctx->tensors[i].name, name) == 0) { + return &ctx->tensors[i]; + } + } + return NULL; +} + +// ---------------------------------------------------------------------------- +// F16 -> F32 conversion + +static inline float f16_to_f32(uint16_t h) { + uint32_t sign = (uint32_t)(h & 0x8000) << 16; + uint32_t exponent = (h >> 10) & 0x1F; + uint32_t mantissa = h & 0x3FF; + uint32_t result; + if (exponent == 0) { + if (mantissa == 0) { + result = sign; + } else { + // denormal + exponent = 1; + while (!(mantissa & 0x400)) { mantissa <<= 1; exponent--; } + mantissa &= 0x3FF; + result = sign | ((exponent + 127 - 15) << 23) | (mantissa << 13); + } + } else if (exponent == 31) { + // inf or nan + result = sign | 0x7F800000 | (mantissa << 13); + } else { + result = sign | ((exponent + 127 - 15) << 23) | (mantissa << 13); + } + float f; + memcpy(&f, &result, 4); + return f; +} + +// ---------------------------------------------------------------------------- +// Dequantize a tensor row to float + +// Returns number of elements written to out. +// out must have space for `ne` floats. +static void gguf_dequantize_row(GGUFTensor *t, int64_t row, float *out) { + uint64_t row_size = t->dims[0]; // elements per row (first dim is fastest) + uint8_t *src = (uint8_t *)t->data; + + if (t->type == GGML_TYPE_F32) { + float *p = (float *)src + row * row_size; + memcpy(out, p, row_size * sizeof(float)); + return; + } + + if (t->type == GGML_TYPE_F16) { + uint16_t *p = (uint16_t *)src + row * row_size; + for (uint64_t i = 0; i < row_size; i++) { + out[i] = f16_to_f32(p[i]); + } + return; + } + + int blk_size = ggml_blk_size[t->type]; + int blk_bytes = ggml_type_size[t->type]; + uint64_t n_blocks_per_row = (row_size + blk_size - 1) / blk_size; + uint8_t *row_data = src + row * n_blocks_per_row * blk_bytes; + + if (t->type == GGML_TYPE_Q8_0) { + // block = [f16 scale (2 bytes)] [32 x int8 (32 bytes)] + for (uint64_t b = 0; b < n_blocks_per_row; b++) { + uint8_t *blk = row_data + b * 34; + uint16_t scale_bits; + memcpy(&scale_bits, blk, 2); + float scale = f16_to_f32(scale_bits); + int8_t *quants = (int8_t *)(blk + 2); + uint64_t base = b * 32; + for (int k = 0; k < 32 && (base + k) < row_size; k++) { + out[base + k] = scale * (float)quants[k]; + } + } + return; + } + + if (t->type == GGML_TYPE_Q4_0) { + // block = [f16 scale (2 bytes)] [16 bytes = 32 x 4-bit quants] + for (uint64_t b = 0; b < n_blocks_per_row; b++) { + uint8_t *blk = row_data + b * 18; + uint16_t scale_bits; + memcpy(&scale_bits, blk, 2); + float scale = f16_to_f32(scale_bits); + uint8_t *quants = blk + 2; + uint64_t base = b * 32; + for (int k = 0; k < 16 && (base + k * 2) < row_size; k++) { + uint8_t byte = quants[k]; + int lo = (int)(byte & 0xF) - 8; + int hi = (int)(byte >> 4) - 8; + out[base + k * 2] = scale * (float)lo; + out[base + k * 2 + 1] = scale * (float)hi; + } + } + return; + } + + fprintf(stderr, "GGUF: unsupported tensor type %d for dequantization\n", t->type); + exit(EXIT_FAILURE); +} + +// ---------------------------------------------------------------------------- +// Free + +static void gguf_value_free(GGUFValue *v) { + if (v->type == GGUF_TYPE_STRING) { free(v->str); } + else if (v->type == GGUF_TYPE_ARRAY) { + for (uint64_t i = 0; i < v->arr.count; i++) { + gguf_value_free(&v->arr.items[i]); + } + free(v->arr.items); + } +} + +static void gguf_free(GGUFContext *ctx) { + for (uint64_t i = 0; i < ctx->n_kv; i++) { + free(ctx->kv[i].key); + gguf_value_free(&ctx->kv[i].value); + } + free(ctx->kv); + for (uint64_t i = 0; i < ctx->n_tensors; i++) { + free(ctx->tensors[i].name); + } + free(ctx->tensors); + free(ctx->data); + free(ctx); +} + +#endif // GGUF_H diff --git a/serve.c b/serve.c new file mode 100644 index 0000000..39f1f4f --- /dev/null +++ b/serve.c @@ -0,0 +1,916 @@ +/* +llm-serve: LLaMA inference server with HTTP API. +Loads GGUF model files, serves OpenAI-compatible /v1/completions endpoint. + +Usage: + ./llm-serve -m model.gguf [-p port] [-c context_size] [-t threads] +*/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef _OPENMP +#include +#endif + +#include "llmc/gguf.h" +#include "llmc/bpe_tokenizer.h" +#include "llmc/sampler.h" + +// ============================================================================ +// LLaMA Model Config + +typedef struct { + int vocab_size; + int embed_dim; // embedding_length / n_embd + int n_layers; + int n_heads; // Q heads + int n_kv_heads; // KV heads (GQA) + int ffn_dim; // feed_forward_length + int context_len; + float rope_theta; // rope.freq_base + float rms_eps; // attention.layer_norm_rms_epsilon + int head_dim; // embed_dim / n_heads +} LlamaConfig; + +// ============================================================================ +// LLaMA Model Weights (pointers into GGUF data, dequantized lazily) + +typedef struct { + // token embedding table [vocab_size, embed_dim] + GGUFTensor *token_embd; + + // per-layer + GGUFTensor **attn_norm; // [n_layers] RMSNorm weight [embed_dim] + GGUFTensor **attn_q; // [n_layers] [n_heads * head_dim, embed_dim] + GGUFTensor **attn_k; // [n_layers] [n_kv_heads * head_dim, embed_dim] + GGUFTensor **attn_v; // [n_layers] [n_kv_heads * head_dim, embed_dim] + GGUFTensor **attn_out; // [n_layers] [embed_dim, n_heads * head_dim] + GGUFTensor **ffn_norm; // [n_layers] RMSNorm weight [embed_dim] + GGUFTensor **ffn_gate; // [n_layers] [ffn_dim, embed_dim] + GGUFTensor **ffn_up; // [n_layers] [ffn_dim, embed_dim] + GGUFTensor **ffn_down; // [n_layers] [embed_dim, ffn_dim] + + // output + GGUFTensor *output_norm; // [embed_dim] + GGUFTensor *output; // [vocab_size, embed_dim] (may be NULL if tied) +} LlamaWeights; + +// ============================================================================ +// KV Cache + +typedef struct { + float *k; // [n_layers, context_len, n_kv_heads, head_dim] + float *v; // [n_layers, context_len, n_kv_heads, head_dim] + int n_layers, context_len, n_kv_heads, head_dim; +} KVCache; + +static KVCache kvcache_alloc(int n_layers, int ctx_len, int n_kv_heads, int head_dim) { + KVCache c; + c.n_layers = n_layers; + c.context_len = ctx_len; + c.n_kv_heads = n_kv_heads; + c.head_dim = head_dim; + size_t sz = (size_t)n_layers * ctx_len * n_kv_heads * head_dim; + c.k = (float *)calloc(sz, sizeof(float)); + c.v = (float *)calloc(sz, sizeof(float)); + if (!c.k || !c.v) { fprintf(stderr, "OOM for KV cache\n"); exit(EXIT_FAILURE); } + return c; +} + +static float *kvcache_k(KVCache *c, int layer, int pos) { + return c->k + ((size_t)layer * c->context_len + pos) * c->n_kv_heads * c->head_dim; +} + +static float *kvcache_v(KVCache *c, int layer, int pos) { + return c->v + ((size_t)layer * c->context_len + pos) * c->n_kv_heads * c->head_dim; +} + +// ============================================================================ +// Run state (activations) + +typedef struct { + float *x; // [embed_dim] current residual + float *xb; // [embed_dim] scratch + float *xb2; // [embed_dim] scratch + float *hb; // [ffn_dim] + float *hb2; // [ffn_dim] + float *q; // [n_heads * head_dim] + float *k; // [n_kv_heads * head_dim] + float *v; // [n_kv_heads * head_dim] + float *att; // [n_heads * context_len] + float *logits; // [vocab_size] +} RunState; + +static RunState runstate_alloc(LlamaConfig *cfg) { + RunState s; + int d = cfg->embed_dim; + int ff = cfg->ffn_dim; + int qd = cfg->n_heads * cfg->head_dim; + int kvd = cfg->n_kv_heads * cfg->head_dim; + s.x = (float *)calloc(d, sizeof(float)); + s.xb = (float *)calloc(d, sizeof(float)); + s.xb2 = (float *)calloc(d, sizeof(float)); + s.hb = (float *)calloc(ff, sizeof(float)); + s.hb2 = (float *)calloc(ff, sizeof(float)); + s.q = (float *)calloc(qd, sizeof(float)); + s.k = (float *)calloc(kvd, sizeof(float)); + s.v = (float *)calloc(kvd, sizeof(float)); + s.att = (float *)calloc(cfg->n_heads * cfg->context_len, sizeof(float)); + s.logits = (float *)calloc(cfg->vocab_size, sizeof(float)); + if (!s.x || !s.xb || !s.xb2 || !s.hb || !s.hb2 || + !s.q || !s.k || !s.v || !s.att || !s.logits) { + fprintf(stderr, "OOM for run state\n"); exit(EXIT_FAILURE); + } + return s; +} + +// ============================================================================ +// Math primitives + +static void rmsnorm(float *out, const float *x, const float *weight, int n, float eps) { + float ss = 0.0f; + for (int i = 0; i < n; i++) ss += x[i] * x[i]; + ss = 1.0f / sqrtf(ss / (float)n + eps); + for (int i = 0; i < n; i++) out[i] = weight[i] * (x[i] * ss); +} + +static void softmax(float *x, int n) { + float max_val = x[0]; + for (int i = 1; i < n; i++) if (x[i] > max_val) max_val = x[i]; + float sum = 0.0f; + for (int i = 0; i < n; i++) { x[i] = expf(x[i] - max_val); sum += x[i]; } + for (int i = 0; i < n; i++) x[i] /= sum; +} + +static inline float silu(float x) { return x / (1.0f + expf(-x)); } + +// Matrix-vector multiply: out[m] = W[m,n] @ in[n] +// W is stored as a GGUFTensor (rows=m, cols=n), dequantized row by row. +static void matvec(float *out, GGUFTensor *W, const float *in, int m, int n) { + float *row = (float *)malloc(n * sizeof(float)); + if (!row) { fprintf(stderr, "OOM matvec\n"); exit(EXIT_FAILURE); } + #pragma omp parallel for schedule(dynamic,16) + for (int i = 0; i < m; i++) { + // dequantize row i + float *row_local = (float *)malloc(n * sizeof(float)); + gguf_dequantize_row(W, i, row_local); + float dot = 0.0f; + for (int j = 0; j < n; j++) dot += row_local[j] * in[j]; + out[i] = dot; + free(row_local); + } + free(row); +} + +// ============================================================================ +// RoPE + +static void rope_apply(float *q, float *k, int pos, LlamaConfig *cfg) { + int hd = cfg->head_dim; + float theta = cfg->rope_theta; + // Q + for (int h = 0; h < cfg->n_heads; h++) { + float *qh = q + h * hd; + for (int i = 0; i < hd / 2; i++) { + float freq = 1.0f / powf(theta, (float)(2 * i) / (float)hd); + float val = (float)pos * freq; + float cos_val = cosf(val), sin_val = sinf(val); + float q0 = qh[2*i], q1 = qh[2*i+1]; + qh[2*i] = q0 * cos_val - q1 * sin_val; + qh[2*i+1] = q0 * sin_val + q1 * cos_val; + } + } + // K + for (int h = 0; h < cfg->n_kv_heads; h++) { + float *kh = k + h * hd; + for (int i = 0; i < hd / 2; i++) { + float freq = 1.0f / powf(theta, (float)(2 * i) / (float)hd); + float val = (float)pos * freq; + float cos_val = cosf(val), sin_val = sinf(val); + float k0 = kh[2*i], k1 = kh[2*i+1]; + kh[2*i] = k0 * cos_val - k1 * sin_val; + kh[2*i+1] = k0 * sin_val + k1 * cos_val; + } + } +} + +// ============================================================================ +// Forward pass (single token at position `pos`) + +static void llama_forward(LlamaConfig *cfg, LlamaWeights *w, RunState *s, + KVCache *kv, int token_id, int pos) { + int d = cfg->embed_dim; + int hd = cfg->head_dim; + int nh = cfg->n_heads; + int nkv = cfg->n_kv_heads; + int kv_mul = nh / nkv; // GQA expansion factor + int ff = cfg->ffn_dim; + float eps = cfg->rms_eps; + + // --- Token embedding --- + // token_embd: [vocab_size, embed_dim] stored as (vocab_size rows, embed_dim cols) + gguf_dequantize_row(w->token_embd, token_id, s->x); + + // --- Transformer layers --- + for (int l = 0; l < cfg->n_layers; l++) { + // RMSNorm before attention + float *norm_w = (float *)malloc(d * sizeof(float)); + gguf_dequantize_row(w->attn_norm[l], 0, norm_w); + rmsnorm(s->xb, s->x, norm_w, d, eps); + free(norm_w); + + // Q, K, V projections + matvec(s->q, w->attn_q[l], s->xb, nh * hd, d); + matvec(s->k, w->attn_k[l], s->xb, nkv * hd, d); + matvec(s->v, w->attn_v[l], s->xb, nkv * hd, d); + + // RoPE + rope_apply(s->q, s->k, pos, cfg); + + // Store K, V in cache + memcpy(kvcache_k(kv, l, pos), s->k, nkv * hd * sizeof(float)); + memcpy(kvcache_v(kv, l, pos), s->v, nkv * hd * sizeof(float)); + + // Attention: for each Q head + memset(s->xb, 0, d * sizeof(float)); + float scale = 1.0f / sqrtf((float)hd); + + for (int h = 0; h < nh; h++) { + float *qh = s->q + h * hd; + int kvh = h / kv_mul; // which KV head to use (GQA) + float *att = s->att + h * cfg->context_len; + + // Dot product Q with all K in cache [0..pos] + for (int t = 0; t <= pos; t++) { + float *kh = kvcache_k(kv, l, t) + kvh * hd; + float dot = 0.0f; + for (int i = 0; i < hd; i++) dot += qh[i] * kh[i]; + att[t] = dot * scale; + } + softmax(att, pos + 1); + + // Weighted sum of V + float *xbh = s->xb + h * hd; + memset(xbh, 0, hd * sizeof(float)); + for (int t = 0; t <= pos; t++) { + float *vh = kvcache_v(kv, l, t) + kvh * hd; + float a = att[t]; + for (int i = 0; i < hd; i++) xbh[i] += a * vh[i]; + } + } + + // Output projection + residual + matvec(s->xb2, w->attn_out[l], s->xb, d, nh * hd); + for (int i = 0; i < d; i++) s->x[i] += s->xb2[i]; + + // RMSNorm before FFN + float *fnorm_w = (float *)malloc(d * sizeof(float)); + gguf_dequantize_row(w->ffn_norm[l], 0, fnorm_w); + rmsnorm(s->xb, s->x, fnorm_w, d, eps); + free(fnorm_w); + + // SwiGLU FFN + matvec(s->hb, w->ffn_gate[l], s->xb, ff, d); + matvec(s->hb2, w->ffn_up[l], s->xb, ff, d); + for (int i = 0; i < ff; i++) s->hb[i] = silu(s->hb[i]) * s->hb2[i]; + matvec(s->xb, w->ffn_down[l], s->hb, d, ff); + + // Residual + for (int i = 0; i < d; i++) s->x[i] += s->xb[i]; + } + + // --- Output norm + logits --- + float *onorm_w = (float *)malloc(d * sizeof(float)); + gguf_dequantize_row(w->output_norm, 0, onorm_w); + rmsnorm(s->xb, s->x, onorm_w, d, eps); + free(onorm_w); + + GGUFTensor *out_W = w->output ? w->output : w->token_embd; + matvec(s->logits, out_W, s->xb, cfg->vocab_size, d); +} + +// ============================================================================ +// Sampling helpers + +static int sample_argmax(const float *logits, int n) { + int best = 0; + for (int i = 1; i < n; i++) if (logits[i] > logits[best]) best = i; + return best; +} + +static int sample_topp(const float *logits, int n, float temperature, float top_p, + unsigned long long *rng_state) { + if (temperature <= 0.0f) return sample_argmax(logits, n); + + // Temperature scaling + softmax + float *probs = (float *)malloc(n * sizeof(float)); + float max_l = logits[0]; + for (int i = 1; i < n; i++) if (logits[i] > max_l) max_l = logits[i]; + float sum = 0.0f; + for (int i = 0; i < n; i++) { probs[i] = expf((logits[i] - max_l) / temperature); sum += probs[i]; } + for (int i = 0; i < n; i++) probs[i] /= sum; + + // Top-p nucleus sampling + // Sort by prob descending (simple insertion sort for small n, but vocab can be large) + // For efficiency: find cumulative sum threshold + float coin = random_f32(rng_state); + if (top_p >= 1.0f) { + // just sample from full distribution + float cdf = 0.0f; + for (int i = 0; i < n; i++) { + cdf += probs[i]; + if (coin < cdf) { free(probs); return i; } + } + free(probs); + return n - 1; + } + + // Build sorted index array (selection sort subset up to top_p) + int *idx = (int *)malloc(n * sizeof(int)); + for (int i = 0; i < n; i++) idx[i] = i; + float cumsum = 0.0f; + int nucleus_size = 0; + for (int i = 0; i < n - 1; i++) { + // find max in [i, n) + int max_j = i; + for (int j = i + 1; j < n; j++) { + if (probs[idx[j]] > probs[idx[max_j]]) max_j = j; + } + int tmp = idx[i]; idx[i] = idx[max_j]; idx[max_j] = tmp; + cumsum += probs[idx[i]]; + nucleus_size = i + 1; + if (cumsum >= top_p) break; + } + if (nucleus_size == 0) nucleus_size = 1; + + // Re-normalize nucleus + float nsum = 0.0f; + for (int i = 0; i < nucleus_size; i++) nsum += probs[idx[i]]; + float cdf = 0.0f; + int result = idx[nucleus_size - 1]; + for (int i = 0; i < nucleus_size; i++) { + cdf += probs[idx[i]] / nsum; + if (coin < cdf) { result = idx[i]; break; } + } + free(probs); + free(idx); + return result; +} + +// ============================================================================ +// Model loading + +typedef struct { + LlamaConfig cfg; + LlamaWeights weights; + KVCache kv; + RunState state; + BPETokenizer tokenizer; + GGUFContext *gguf; + unsigned long long rng_state; +} LlamaModel; + +static GGUFTensor *require_tensor(GGUFContext *ctx, const char *name) { + GGUFTensor *t = gguf_find_tensor(ctx, name); + if (!t) { + fprintf(stderr, "WARN: tensor '%s' not found\n", name); + } + return t; +} + +static LlamaModel *llama_load(const char *path, int context_override) { + fprintf(stderr, "Loading GGUF model: %s\n", path); + GGUFContext *ctx = gguf_load(path); + + const char *arch = gguf_get_str(ctx, "general.architecture", "llama"); + + char key[256]; + LlamaConfig cfg; + memset(&cfg, 0, sizeof(cfg)); + + #define ARCH_KEY(k) (snprintf(key, sizeof(key), "%s.%s", arch, k), key) + + cfg.vocab_size = (int)gguf_get_u64(ctx, ARCH_KEY("vocab_size"), 32000); + cfg.embed_dim = (int)gguf_get_u64(ctx, ARCH_KEY("embedding_length"), 4096); + cfg.n_layers = (int)gguf_get_u64(ctx, ARCH_KEY("block_count"), 32); + cfg.n_heads = (int)gguf_get_u64(ctx, ARCH_KEY("attention.head_count"), 32); + cfg.n_kv_heads = (int)gguf_get_u64(ctx, ARCH_KEY("attention.head_count_kv"), cfg.n_heads); + cfg.ffn_dim = (int)gguf_get_u64(ctx, ARCH_KEY("feed_forward_length"), 11008); + cfg.context_len = (int)gguf_get_u64(ctx, ARCH_KEY("context_length"), 2048); + cfg.rope_theta = gguf_get_f32(ctx, ARCH_KEY("rope.freq_base"), 10000.0f); + cfg.rms_eps = gguf_get_f32(ctx, ARCH_KEY("attention.layer_norm_rms_epsilon"), 1e-5f); + cfg.head_dim = cfg.embed_dim / cfg.n_heads; + + if (context_override > 0 && context_override < cfg.context_len) + cfg.context_len = context_override; + + // Also try vocab size from tokenizer + { + GGUFValue *tv = gguf_find_kv(ctx, "tokenizer.ggml.tokens"); + if (tv && tv->type == GGUF_TYPE_ARRAY && (int)tv->arr.count > cfg.vocab_size) + cfg.vocab_size = (int)tv->arr.count; + } + + fprintf(stderr, " arch=%s layers=%d heads=%d kv_heads=%d embed=%d ffn=%d ctx=%d\n", + arch, cfg.n_layers, cfg.n_heads, cfg.n_kv_heads, + cfg.embed_dim, cfg.ffn_dim, cfg.context_len); + + // Load weights + LlamaWeights w; + memset(&w, 0, sizeof(w)); + w.token_embd = require_tensor(ctx, "token_embd.weight"); + w.output_norm = require_tensor(ctx, "output_norm.weight"); + w.output = gguf_find_tensor(ctx, "output.weight"); // may be NULL (tied) + + w.attn_norm = (GGUFTensor **)calloc(cfg.n_layers, sizeof(GGUFTensor *)); + w.attn_q = (GGUFTensor **)calloc(cfg.n_layers, sizeof(GGUFTensor *)); + w.attn_k = (GGUFTensor **)calloc(cfg.n_layers, sizeof(GGUFTensor *)); + w.attn_v = (GGUFTensor **)calloc(cfg.n_layers, sizeof(GGUFTensor *)); + w.attn_out = (GGUFTensor **)calloc(cfg.n_layers, sizeof(GGUFTensor *)); + w.ffn_norm = (GGUFTensor **)calloc(cfg.n_layers, sizeof(GGUFTensor *)); + w.ffn_gate = (GGUFTensor **)calloc(cfg.n_layers, sizeof(GGUFTensor *)); + w.ffn_up = (GGUFTensor **)calloc(cfg.n_layers, sizeof(GGUFTensor *)); + w.ffn_down = (GGUFTensor **)calloc(cfg.n_layers, sizeof(GGUFTensor *)); + + for (int l = 0; l < cfg.n_layers; l++) { + snprintf(key, sizeof(key), "blk.%d.attn_norm.weight", l); + w.attn_norm[l] = require_tensor(ctx, key); + snprintf(key, sizeof(key), "blk.%d.attn_q.weight", l); + w.attn_q[l] = require_tensor(ctx, key); + snprintf(key, sizeof(key), "blk.%d.attn_k.weight", l); + w.attn_k[l] = require_tensor(ctx, key); + snprintf(key, sizeof(key), "blk.%d.attn_v.weight", l); + w.attn_v[l] = require_tensor(ctx, key); + snprintf(key, sizeof(key), "blk.%d.attn_output.weight", l); + w.attn_out[l] = require_tensor(ctx, key); + snprintf(key, sizeof(key), "blk.%d.ffn_norm.weight", l); + w.ffn_norm[l] = require_tensor(ctx, key); + snprintf(key, sizeof(key), "blk.%d.ffn_gate.weight", l); + w.ffn_gate[l] = require_tensor(ctx, key); + snprintf(key, sizeof(key), "blk.%d.ffn_up.weight", l); + w.ffn_up[l] = require_tensor(ctx, key); + snprintf(key, sizeof(key), "blk.%d.ffn_down.weight", l); + w.ffn_down[l] = require_tensor(ctx, key); + } + + LlamaModel *model = (LlamaModel *)calloc(1, sizeof(LlamaModel)); + model->cfg = cfg; + model->weights = w; + model->gguf = ctx; + model->kv = kvcache_alloc(cfg.n_layers, cfg.context_len, cfg.n_kv_heads, cfg.head_dim); + model->state = runstate_alloc(&cfg); + model->rng_state = (unsigned long long)time(NULL); + + bpe_tokenizer_init(&model->tokenizer, ctx); + + fprintf(stderr, " vocab_size=%d bos=%d eos=%d\n", + model->tokenizer.vocab_size, + model->tokenizer.bos_id, + model->tokenizer.eos_id); + fprintf(stderr, "Model loaded.\n"); + return model; +} + +// ============================================================================ +// Generate tokens + +typedef struct { + int *tokens; + int n_tokens; + float temperature; + float top_p; + int max_new_tokens; + int stream; + int client_fd; +} GenParams; + +// Returns generated text (malloc'd). Caller must free. +static char *generate(LlamaModel *model, GenParams *p) { + LlamaConfig *cfg = &model->cfg; + BPETokenizer *tok = &model->tokenizer; + + // Reset KV cache + int kv_stride = cfg->context_len * cfg->n_kv_heads * cfg->head_dim; + memset(model->kv.k, 0, (size_t)cfg->n_layers * kv_stride * sizeof(float)); + memset(model->kv.v, 0, (size_t)cfg->n_layers * kv_stride * sizeof(float)); + + size_t out_cap = 4096; + char *out_text = (char *)malloc(out_cap); + if (!out_text) { fprintf(stderr, "OOM\n"); exit(EXIT_FAILURE); } + out_text[0] = '\0'; + size_t out_len = 0; + + int pos = 0; + int n_prompt = p->n_tokens; + + // Prefill prompt tokens + for (int i = 0; i < n_prompt && pos < cfg->context_len; i++, pos++) { + llama_forward(cfg, &model->weights, &model->state, &model->kv, p->tokens[i], pos); + } + + int n_gen = 0; + char piece_buf[BPE_MAX_TOKEN]; + char sse_buf[BPE_MAX_TOKEN + 128]; + + while (n_gen < p->max_new_tokens && pos < cfg->context_len) { + // Sample next token + int next; + if (p->temperature <= 0.0f) { + next = sample_argmax(model->state.logits, cfg->vocab_size); + } else { + next = sample_topp(model->state.logits, cfg->vocab_size, + p->temperature, p->top_p, &model->rng_state); + } + + // EOS check + if (next == tok->eos_id) break; + + // Decode token to text + bpe_decode_piece(tok, next, piece_buf, sizeof(piece_buf)); + + // Append to output + size_t plen = strlen(piece_buf); + if (out_len + plen + 1 > out_cap) { + out_cap *= 2; + out_text = (char *)realloc(out_text, out_cap); + if (!out_text) { fprintf(stderr, "OOM\n"); exit(EXIT_FAILURE); } + } + memcpy(out_text + out_len, piece_buf, plen); + out_len += plen; + out_text[out_len] = '\0'; + + // Stream SSE if requested + if (p->stream && p->client_fd >= 0) { + int slen = snprintf(sse_buf, sizeof(sse_buf), + "data: {\"choices\":[{\"text\":\"%s\",\"finish_reason\":null}]}\n\n", + piece_buf); + send(p->client_fd, sse_buf, slen, MSG_NOSIGNAL); + } + + // Forward for next token + llama_forward(cfg, &model->weights, &model->state, &model->kv, next, pos); + pos++; + n_gen++; + } + + if (p->stream && p->client_fd >= 0) { + const char *done = "data: [DONE]\n\n"; + send(p->client_fd, done, strlen(done), MSG_NOSIGNAL); + } + + return out_text; +} + +// ============================================================================ +// JSON helpers + +// Simple JSON string escape into dst (returns chars written, not including \0) +static int json_escape(const char *src, char *dst, int dst_cap) { + int j = 0; + for (int i = 0; src[i] && j < dst_cap - 4; i++) { + unsigned char c = (unsigned char)src[i]; + if (c == '"') { dst[j++] = '\\'; dst[j++] = '"'; } + else if (c == '\\') { dst[j++] = '\\'; dst[j++] = '\\'; } + else if (c == '\n') { dst[j++] = '\\'; dst[j++] = 'n'; } + else if (c == '\r') { dst[j++] = '\\'; dst[j++] = 'r'; } + else if (c == '\t') { dst[j++] = '\\'; dst[j++] = 't'; } + else if (c < 0x20) { j += snprintf(dst+j, dst_cap-j, "\\u%04X", c); } + else { dst[j++] = (char)c; } + } + dst[j] = '\0'; + return j; +} + +// Extract string value for a key from JSON (very naive, no nesting) +// Returns pointer into buf (null-terminated), or NULL. +static char *json_get_str(const char *json, const char *key, char *buf, int buf_size) { + char search[128]; + snprintf(search, sizeof(search), "\"%s\"", key); + const char *p = strstr(json, search); + if (!p) return NULL; + p += strlen(search); + while (*p == ' ' || *p == ':' || *p == ' ') p++; + if (*p != '"') return NULL; + p++; + int i = 0; + while (*p && *p != '"' && i < buf_size - 1) { + if (*p == '\\' && *(p+1)) { + p++; + if (*p == 'n') buf[i++] = '\n'; + else if (*p == 't') buf[i++] = '\t'; + else if (*p == 'r') buf[i++] = '\r'; + else buf[i++] = *p; + } else { + buf[i++] = *p; + } + p++; + } + buf[i] = '\0'; + return buf; +} + +static float json_get_float(const char *json, const char *key, float def) { + char search[128]; + snprintf(search, sizeof(search), "\"%s\"", key); + const char *p = strstr(json, search); + if (!p) return def; + p += strlen(search); + while (*p == ' ' || *p == ':') p++; + if (*p == '"') return def; + return (float)atof(p); +} + +static int json_get_int(const char *json, const char *key, int def) { + char search[128]; + snprintf(search, sizeof(search), "\"%s\"", key); + const char *p = strstr(json, search); + if (!p) return def; + p += strlen(search); + while (*p == ' ' || *p == ':') p++; + if (*p == '"') return def; + return atoi(p); +} + +static int json_get_bool(const char *json, const char *key, int def) { + char search[128]; + snprintf(search, sizeof(search), "\"%s\"", key); + const char *p = strstr(json, search); + if (!p) return def; + p += strlen(search); + while (*p == ' ' || *p == ':') p++; + if (strncmp(p, "true", 4) == 0) return 1; + if (strncmp(p, "false", 5) == 0) return 0; + return def; +} + +// ============================================================================ +// HTTP server + +#define HTTP_MAX_REQUEST (1 << 20) // 1 MB + +static void send_response(int fd, int status, const char *content_type, + const char *body, int body_len, int is_stream) { + char header[512]; + int hlen; + if (is_stream) { + hlen = snprintf(header, sizeof(header), + "HTTP/1.1 %d OK\r\n" + "Content-Type: text/event-stream\r\n" + "Cache-Control: no-cache\r\n" + "Connection: keep-alive\r\n" + "\r\n", status); + } else { + hlen = snprintf(header, sizeof(header), + "HTTP/1.1 %d OK\r\n" + "Content-Type: %s\r\n" + "Content-Length: %d\r\n" + "Connection: close\r\n" + "\r\n", status, content_type, body_len); + } + send(fd, header, hlen, MSG_NOSIGNAL); + if (body && body_len > 0 && !is_stream) + send(fd, body, body_len, MSG_NOSIGNAL); +} + +static void send_error(int fd, int status, const char *msg) { + char body[256]; + int blen = snprintf(body, sizeof(body), + "{\"error\":{\"message\":\"%s\",\"code\":%d}}", msg, status); + char header[256]; + int hlen = snprintf(header, sizeof(header), + "HTTP/1.1 %d Error\r\n" + "Content-Type: application/json\r\n" + "Content-Length: %d\r\n" + "Connection: close\r\n" + "\r\n", status, blen); + send(fd, header, hlen, MSG_NOSIGNAL); + send(fd, body, blen, MSG_NOSIGNAL); +} + +static void handle_request(int fd, LlamaModel *model, const char *method, + const char *path, const char *body) { + // GET /health + if (strcmp(method, "GET") == 0 && strcmp(path, "/health") == 0) { + const char *resp = "{\"status\":\"ok\"}"; + send_response(fd, 200, "application/json", resp, (int)strlen(resp), 0); + return; + } + + // POST /v1/completions + if (strcmp(method, "POST") == 0 && + (strcmp(path, "/v1/completions") == 0 || strcmp(path, "/completions") == 0)) { + + if (!body || body[0] == '\0') { + send_error(fd, 400, "empty body"); return; + } + + char prompt_buf[65536]; + if (!json_get_str(body, "prompt", prompt_buf, sizeof(prompt_buf))) { + send_error(fd, 400, "missing prompt"); return; + } + + float temperature = json_get_float(body, "temperature", 0.7f); + float top_p = json_get_float(body, "top_p", 0.9f); + int max_tokens = json_get_int(body, "max_tokens", 128); + int stream = json_get_bool(body, "stream", 0); + + if (max_tokens <= 0) max_tokens = 128; + if (max_tokens > model->cfg.context_len) max_tokens = model->cfg.context_len; + + // Tokenize prompt + int *prompt_tokens = (int *)malloc((strlen(prompt_buf) + 4) * sizeof(int)); + int n_prompt = bpe_encode(&model->tokenizer, prompt_buf, 1, + prompt_tokens, (int)strlen(prompt_buf) + 4); + + // Limit prompt to context + int ctx = model->cfg.context_len; + if (n_prompt >= ctx) n_prompt = ctx - 1; + + fprintf(stderr, "Generating: prompt_tokens=%d max_new=%d temp=%.2f\n", + n_prompt, max_tokens, temperature); + + GenParams gp; + gp.tokens = prompt_tokens; + gp.n_tokens = n_prompt; + gp.temperature = temperature; + gp.top_p = top_p; + gp.max_new_tokens = max_tokens; + gp.stream = stream; + gp.client_fd = stream ? fd : -1; + + if (stream) { + send_response(fd, 200, NULL, NULL, 0, 1); + } + + char *gen_text = generate(model, &gp); + free(prompt_tokens); + + if (!stream) { + // Escape generated text for JSON + size_t esc_size = strlen(gen_text) * 6 + 4; + char *esc_text = (char *)malloc(esc_size); + json_escape(gen_text, esc_text, (int)esc_size); + + size_t resp_size = esc_size + 256; + char *resp = (char *)malloc(resp_size); + int rlen = snprintf(resp, resp_size, + "{\"choices\":[{\"text\":\"%s\",\"finish_reason\":\"stop\"}]," + "\"usage\":{\"prompt_tokens\":%d,\"completion_tokens\":%d}}", + esc_text, n_prompt, (int)strlen(gen_text)); + + send_response(fd, 200, "application/json", resp, rlen, 0); + free(resp); + free(esc_text); + } + + free(gen_text); + return; + } + + send_error(fd, 404, "not found"); +} + +static void serve_loop(LlamaModel *model, int port) { + int server_fd = socket(AF_INET, SOCK_STREAM, 0); + if (server_fd < 0) { perror("socket"); exit(EXIT_FAILURE); } + + int opt = 1; + setsockopt(server_fd, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt)); + + struct sockaddr_in addr; + memset(&addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = INADDR_ANY; + addr.sin_port = htons((uint16_t)port); + + if (bind(server_fd, (struct sockaddr *)&addr, sizeof(addr)) < 0) { + perror("bind"); exit(EXIT_FAILURE); + } + if (listen(server_fd, 16) < 0) { + perror("listen"); exit(EXIT_FAILURE); + } + + fprintf(stderr, "llm-serve listening on http://0.0.0.0:%d\n", port); + fprintf(stderr, " POST /v1/completions\n"); + fprintf(stderr, " GET /health\n"); + + char *req_buf = (char *)malloc(HTTP_MAX_REQUEST); + if (!req_buf) { fprintf(stderr, "OOM\n"); exit(EXIT_FAILURE); } + + while (1) { + struct sockaddr_in client_addr; + socklen_t clen = sizeof(client_addr); + int client_fd = accept(server_fd, (struct sockaddr *)&client_addr, &clen); + if (client_fd < 0) { perror("accept"); continue; } + + // Read request + int total = 0; + while (total < HTTP_MAX_REQUEST - 1) { + int n = (int)recv(client_fd, req_buf + total, HTTP_MAX_REQUEST - 1 - total, 0); + if (n <= 0) break; + total += n; + req_buf[total] = '\0'; + // Check if we have a complete HTTP request + if (strstr(req_buf, "\r\n\r\n")) { + // Check Content-Length + const char *cl_hdr = strstr(req_buf, "Content-Length:"); + if (!cl_hdr) cl_hdr = strstr(req_buf, "content-length:"); + if (cl_hdr) { + int cl = atoi(cl_hdr + 15); + const char *body_start = strstr(req_buf, "\r\n\r\n"); + if (body_start) { + int body_received = (int)(total - (body_start + 4 - req_buf)); + if (body_received >= cl) break; + } + } else { + break; + } + } + } + req_buf[total] = '\0'; + + // Parse request line + char method[16] = {0}, path[256] = {0}; + sscanf(req_buf, "%15s %255s", method, path); + + // Find body + const char *body_sep = strstr(req_buf, "\r\n\r\n"); + const char *body = body_sep ? body_sep + 4 : ""; + + handle_request(client_fd, model, method, path, body); + close(client_fd); + } + + free(req_buf); + close(server_fd); +} + +// ============================================================================ +// CLI + +static void print_usage(const char *prog) { + fprintf(stderr, + "Usage: %s [options]\n" + " -m, --model Path to GGUF model file (required)\n" + " -p, --port Server port (default: 8080)\n" + " -c, --context Context size override\n" + " -t, --threads Number of OpenMP threads (default: 4)\n" + " -h, --help Show help\n", + prog); +} + +int main(int argc, char *argv[]) { + const char *model_path = NULL; + int port = 8080; + int context_size = 0; + int n_threads = 4; + + for (int i = 1; i < argc; i++) { + if ((strcmp(argv[i], "-m") == 0 || strcmp(argv[i], "--model") == 0) && i+1 < argc) { + model_path = argv[++i]; + } else if ((strcmp(argv[i], "-p") == 0 || strcmp(argv[i], "--port") == 0) && i+1 < argc) { + port = atoi(argv[++i]); + } else if ((strcmp(argv[i], "-c") == 0 || strcmp(argv[i], "--context") == 0) && i+1 < argc) { + context_size = atoi(argv[++i]); + } else if ((strcmp(argv[i], "-t") == 0 || strcmp(argv[i], "--threads") == 0) && i+1 < argc) { + n_threads = atoi(argv[++i]); + } else if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) { + print_usage(argv[0]); + return 0; + } else { + fprintf(stderr, "Unknown argument: %s\n", argv[i]); + print_usage(argv[0]); + return 1; + } + } + + if (!model_path) { + fprintf(stderr, "Error: -m is required\n"); + print_usage(argv[0]); + return 1; + } + +#ifdef _OPENMP + omp_set_num_threads(n_threads); + fprintf(stderr, "OpenMP threads: %d\n", n_threads); +#else + (void)n_threads; +#endif + + signal(SIGPIPE, SIG_IGN); + + LlamaModel *model = llama_load(model_path, context_size); + serve_loop(model, port); + + return 0; +} diff --git a/test_serve_integration.sh b/test_serve_integration.sh new file mode 100755 index 0000000..9de1b40 --- /dev/null +++ b/test_serve_integration.sh @@ -0,0 +1,242 @@ +#!/usr/bin/env bash +# Integration test for llm-serve: happy-path smoke test. +# Starts the server, checks /health, sends a completion request, +# validates response structure, and tears down. +# +# Usage: ./test_serve_integration.sh [path/to/model.gguf] +# Requires: curl, jq + +set -euo pipefail + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[0;33m' +NC='\033[0m' + +PASS=0 +FAIL=0 + +pass() { ((PASS++)); echo -e " ${GREEN}✓${NC} $1"; } +fail() { ((FAIL++)); echo -e " ${RED}✗${NC} $1"; } + +# --------------------------------------------------------------------------- +# Config +MODEL="${1:-$HOME/Projects/TestInference/models/tinyllama-1.1b-chat-v1.0.Q8_0.gguf}" +PORT=18199 # high port to avoid conflicts +SERVER_PID="" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" + +cleanup() { + if [[ -n "$SERVER_PID" ]] && kill -0 "$SERVER_PID" 2>/dev/null; then + kill "$SERVER_PID" 2>/dev/null || true + wait "$SERVER_PID" 2>/dev/null || true + fi +} +trap cleanup EXIT + +# --------------------------------------------------------------------------- +# Preflight checks + +echo "=== llm-serve integration test ===" +echo "" + +if [[ ! -f "$MODEL" ]]; then + echo -e "${RED}ERROR: Model not found: $MODEL${NC}" + echo "Pass a GGUF model path as the first argument." + exit 1 +fi + +for cmd in curl jq; do + if ! command -v "$cmd" &>/dev/null; then + echo -e "${RED}ERROR: $cmd not found${NC}" + exit 1 + fi +done + +# --------------------------------------------------------------------------- +# Build + +echo "Building llm-serve..." +cd "$SCRIPT_DIR" +make llm-serve 2>&1 | tail -3 +if [[ ! -x "./llm-serve" ]]; then + echo -e "${RED}ERROR: Build failed — ./llm-serve not found${NC}" + exit 1 +fi +echo "" + +# --------------------------------------------------------------------------- +# Start server + +echo "Starting server on port $PORT..." +./llm-serve -m "$MODEL" -p "$PORT" -c 256 -t 2 & +SERVER_PID=$! + +# Wait for the server to become ready (up to 30s for model load) +READY=0 +for i in $(seq 1 60); do + if curl -sf "http://127.0.0.1:$PORT/health" >/dev/null 2>&1; then + READY=1 + break + fi + # Check if process died + if ! kill -0 "$SERVER_PID" 2>/dev/null; then + echo -e "${RED}ERROR: Server exited prematurely${NC}" + exit 1 + fi + sleep 0.5 +done + +if [[ "$READY" -ne 1 ]]; then + echo -e "${RED}ERROR: Server didn't become ready in 30s${NC}" + exit 1 +fi +echo "Server ready (PID $SERVER_PID)." +echo "" + +# --------------------------------------------------------------------------- +# Test 1: GET /health + +echo "Test 1: GET /health" +HEALTH=$(curl -sf "http://127.0.0.1:$PORT/health") +if echo "$HEALTH" | jq -e '.status == "ok"' >/dev/null 2>&1; then + pass "/health returns {\"status\":\"ok\"}" +else + fail "/health unexpected response: $HEALTH" +fi + +# --------------------------------------------------------------------------- +# Test 2: POST /v1/completions — basic generation + +echo "Test 2: POST /v1/completions (basic)" +RESP=$(curl -sf -X POST "http://127.0.0.1:$PORT/v1/completions" \ + -H "Content-Type: application/json" \ + -d '{ + "prompt": "The capital of France is", + "max_tokens": 16, + "temperature": 0.0 + }') + +# 2a: Response is valid JSON +if echo "$RESP" | jq . >/dev/null 2>&1; then + pass "Response is valid JSON" +else + fail "Response is not valid JSON: $RESP" +fi + +# 2b: Has choices array +if echo "$RESP" | jq -e '.choices | length > 0' >/dev/null 2>&1; then + pass "Response has non-empty choices array" +else + fail "Missing or empty choices array" +fi + +# 2c: choices[0].text is a non-empty string +GEN_TEXT=$(echo "$RESP" | jq -r '.choices[0].text // ""') +if [[ -n "$GEN_TEXT" ]]; then + pass "Generated text is non-empty: \"$(echo "$GEN_TEXT" | head -c 80)...\"" +else + fail "Generated text is empty" +fi + +# 2d: Has finish_reason +if echo "$RESP" | jq -e '.choices[0].finish_reason' >/dev/null 2>&1; then + pass "Response includes finish_reason" +else + fail "Missing finish_reason" +fi + +# 2e: Has usage info +if echo "$RESP" | jq -e '.usage.prompt_tokens > 0' >/dev/null 2>&1; then + pass "Response includes usage.prompt_tokens > 0" +else + fail "Missing or zero usage.prompt_tokens" +fi + +if echo "$RESP" | jq -e '.usage.completion_tokens >= 0' >/dev/null 2>&1; then + pass "Response includes usage.completion_tokens" +else + fail "Missing usage.completion_tokens" +fi + +# --------------------------------------------------------------------------- +# Test 3: POST /v1/completions with temperature/top_p + +echo "Test 3: POST /v1/completions (with sampling params)" +RESP2=$(curl -sf -X POST "http://127.0.0.1:$PORT/v1/completions" \ + -H "Content-Type: application/json" \ + -d '{ + "prompt": "Once upon a time", + "max_tokens": 8, + "temperature": 0.8, + "top_p": 0.95 + }') + +GEN_TEXT2=$(echo "$RESP2" | jq -r '.choices[0].text // ""') +if [[ -n "$GEN_TEXT2" ]]; then + pass "Sampling generation returned text: \"$(echo "$GEN_TEXT2" | head -c 80)...\"" +else + fail "Sampling generation returned empty text" +fi + +# --------------------------------------------------------------------------- +# Test 4: Error handling — missing prompt + +echo "Test 4: Error handling" +ERR_RESP=$(curl -s -o /dev/null -w "%{http_code}" -X POST "http://127.0.0.1:$PORT/v1/completions" \ + -H "Content-Type: application/json" \ + -d '{}') + +if [[ "$ERR_RESP" == "400" ]]; then + pass "Empty body returns 400" +else + fail "Expected 400 for missing prompt, got $ERR_RESP" +fi + +# --------------------------------------------------------------------------- +# Test 5: 404 on unknown route + +echo "Test 5: Unknown route" +NOT_FOUND=$(curl -s -o /dev/null -w "%{http_code}" "http://127.0.0.1:$PORT/v1/nonexistent") + +if [[ "$NOT_FOUND" == "404" ]]; then + pass "Unknown route returns 404" +else + fail "Expected 404, got $NOT_FOUND" +fi + +# --------------------------------------------------------------------------- +# Test 6: Streaming + +echo "Test 6: POST /v1/completions (streaming)" +STREAM_RESP=$(curl -sf -N -X POST "http://127.0.0.1:$PORT/v1/completions" \ + -H "Content-Type: application/json" \ + --max-time 60 \ + -d '{ + "prompt": "Hello", + "max_tokens": 4, + "temperature": 0.0, + "stream": true + }' 2>&1 || true) + +if echo "$STREAM_RESP" | grep -q "data:"; then + pass "Streaming response contains SSE data lines" +else + fail "Streaming response missing SSE data lines" +fi + +if echo "$STREAM_RESP" | grep -q '\[DONE\]'; then + pass "Streaming response ends with [DONE]" +else + fail "Streaming response missing [DONE] sentinel" +fi + +# --------------------------------------------------------------------------- +# Summary + +echo "" +echo "=== Results: ${GREEN}$PASS passed${NC}, ${RED}$FAIL failed${NC} ===" + +if [[ "$FAIL" -gt 0 ]]; then + exit 1 +fi