Created by: punitkoura
Patch Description This change ensures that our API returns None as the first token score in echo mode. This is in line with how OpenAI API operates.
Testing steps Before the change
curl -k http://$ADDRESS/completions -H "Authorization: Bearer Punit" -H "Content-Type: application/json" -d '{
"prompt": [133, 313, 1224, 15, 5, 856, 17527, 594, 4],
"temperature": 1.0,
"max_tokens": 0, "min_tokens": 0,
"top_p": 1.0, "n": 1, "best_of": 1,
"echo": true, "logprobs": 0, "seed": 1
}' | jq .
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
100 775 100 578 100 197 1191 406 --:--:-- --:--:-- --:--:-- 1594
{
"choices": [
{
"logprobs": {
"finish_reason": "length",
"text_offset": [
0,
3,
7,
14,
17,
21,
23,
26,
28
],
"token_logprobs": [
null,
-3.4349148273468018,
-5.929329872131348,
-7.783979892730713,
-3.8459229469299316,
-1.0909258127212524,
-4.3222551345825195,
-0.008717338554561138,
-0.018318286165595055,
-2.4386134147644043
],
"tokens": [
"</s>",
"The",
" man",
" turned",
" on",
" the",
" f",
"auc",
"et",
"."
],
"top_logprobs": null
},
"text": "The man turned on the faucet."
}
],
"created": 1660233923,
"id": "1aa3c12f-a3d1-4f5b-8025-3eb22a7dd3d3",
"model": "30B/reshard_no_os/",
"object": "text_completion"
}
After the change
curl -k http://$ADDRESS/completions -H "Authorization: Bearer Punit" -H "Content-Type: application/json" -d '{
"prompt": [133, 313, 1224, 15, 5, 856, 17527, 594, 4],
"temperature": 1.0,
"max_tokens": 0, "min_tokens": 0,
"top_p": 1.0, "n": 1, "best_of": 1,
"echo": true, "logprobs": 0, "seed": 1
}' | jq .
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
100 748 100 551 100 197 1960 701 --:--:-- --:--:-- --:--:-- 2652
{
"choices": [
{
"logprobs": {
"finish_reason": "length",
"text_offset": [
0,
3,
7,
14,
17,
21,
23,
26,
28
],
"token_logprobs": [
null,
-5.929329872131348,
-7.783979892730713,
-3.8459229469299316,
-1.0909258127212524,
-4.3222551345825195,
-0.008717338554561138,
-0.018318286165595055,
-2.4386134147644043
],
"tokens": [
"The",
" man",
" turned",
" on",
" the",
" f",
"auc",
"et",
"."
],
"top_logprobs": null
},
"text": "The man turned on the faucet."
}
],
"created": 1660233768,
"id": "01c269be-0096-4154-b132-51fd5419f95d",
"model": "30B/reshard_no_os/",
"object": "text_completion"
}
Note that after the change, we do not return a score for (and trim it from the returned tokens), and set the score for the first word "The" to None (instead of -3.4349148273468018)
A similar change is made for top_logprobs. After the change
curl -k http://$ADDRESS/completions -H "Authorization: Bearer Punit" -H "Content-Type: application/json" -d '{
"prompt": [133, 313, 1224, 15, 5, 856, 17527, 594, 4],
"temperature": 1.0,
"max_tokens": 0, "min_tokens": 0,
"top_p": 1.0, "n": 1, "best_of": 1,
"echo": true, "logprobs": 2, "seed": 1
}' | jq .
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
100 1196 100 999 100 197 3517 693 --:--:-- --:--:-- --:--:-- 4211
{
"choices": [
{
"logprobs": {
"finish_reason": "length",
"text_offset": [
0,
3,
7,
14,
17,
21,
23,
26,
28
],
"token_logprobs": [
null,
-5.929329872131348,
-7.783979892730713,
-3.8459229469299316,
-1.0909258127212524,
-4.3222551345825195,
-0.008717338554561138,
-0.018318286165595055,
-2.4386134147644043
],
"tokens": [
"The",
" man",
" turned",
" on",
" the",
" f",
"auc",
"et",
"."
],
"top_logprobs": [
null,
{
" first": -4.359017372131348,
" only": -3.9527673721313477
},
{
" is": -2.362104892730713,
" who": -1.5808547735214233
},
{
" his": -2.4084229469299316,
" into": -2.2209229469299316
},
{
" his": -1.4503008127212524,
" the": -1.0909258127212524
},
{
" TV": -2.9472548961639404,
" water": -2.5878798961639404
},
{
"auc": -0.008717338554561138,
"ount": -6.1024675369262695
},
{
"et": -0.018318286165595055,
"ets": -4.033943176269531
},
{
" and": -1.1573634147644043,
",": -1.7042384147644043
}
]
},
"text": "The man turned on the faucet."
}
],
"created": 1660235018,
"id": "0144410a-a8f5-48ea-8955-df9ad1da7b7e",
"model": "30B/reshard_no_os/",
"object": "text_completion"
}