Token-level streaming support

[cpacker]

Feature: add token-level streaming support for the API

Rough high-level implementation:

• For OpenAI / function-calling APIs that support streaming
  • Transparently use the function-calling streaming and pass the outputs back
• For local LLMs / non-function-calling APIs where we are using custom prompt formatters
  • Stream back the raw completion tokens to the model wrapper / output parser
  • The output parser acts as middleware and buffers the stream to mimic an OpenAI function-calling API token stream request

---

Example SSE response from a function calling OpenAI request:

curl https://api.openai.com/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer $OPENAI_API_KEY" \
-d '{
  "model": "gpt-4",
  "messages": [
    {
      "role": "user",
      "content": "What is the weather like in Boston?"
    }
  ],
  "tools": [
    {
      "type": "function",
      "function": {
        "name": "get_current_weather",
        "description": "Get the current weather in a given location",
        "parameters": {
          "type": "object",
          "properties": {
            "location": {
              "type": "string",
              "description": "The city and state, e.g. San Francisco, CA"
            },
            "unit": {
              "type": "string",
              "enum": ["celsius", "fahrenheit"]
            }
          },
          "required": ["location"]
        }
      }
    }
  ],
  "tool_choice": "auto",
  "stream": true
}'

data: {"id":"chatcmpl-9A0pokSpYASGLPfIzV8jx6tuvBUtz","object":"chat.completion.chunk","created":1712173516,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"role":"assistant","content":null,"tool_calls":[{"index":0,"id":"call_telkvFkERZ3MW3eLzHsVrRhl","type":"function","function":{"name":"get_current_weather","arguments":""}}]},"logprobs":null,"finish_reason":null}]}

data: {"id":"chatcmpl-9A0pokSpYASGLPfIzV8jx6tuvBUtz","object":"chat.completion.chunk","created":1712173516,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\n"}}]},"logprobs":null,"finish_reason":null}]}

data: {"id":"chatcmpl-9A0pokSpYASGLPfIzV8jx6tuvBUtz","object":"chat.completion.chunk","created":1712173516,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":" "}}]},"logprobs":null,"finish_reason":null}]}

data: {"id":"chatcmpl-9A0pokSpYASGLPfIzV8jx6tuvBUtz","object":"chat.completion.chunk","created":1712173516,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":" \""}}]},"logprobs":null,"finish_reason":null}]}

data: {"id":"chatcmpl-9A0pokSpYASGLPfIzV8jx6tuvBUtz","object":"chat.completion.chunk","created":1712173516,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"location"}}]},"logprobs":null,"finish_reason":null}]}

data: {"id":"chatcmpl-9A0pokSpYASGLPfIzV8jx6tuvBUtz","object":"chat.completion.chunk","created":1712173516,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\":"}}]},"logprobs":null,"finish_reason":null}]}

data: {"id":"chatcmpl-9A0pokSpYASGLPfIzV8jx6tuvBUtz","object":"chat.completion.chunk","created":1712173516,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":" \""}}]},"logprobs":null,"finish_reason":null}]}

data: {"id":"chatcmpl-9A0pokSpYASGLPfIzV8jx6tuvBUtz","object":"chat.completion.chunk","created":1712173516,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"Boston"}}]},"logprobs":null,"finish_reason":null}]}

data: {"id":"chatcmpl-9A0pokSpYASGLPfIzV8jx6tuvBUtz","object":"chat.completion.chunk","created":1712173516,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\"\n"}}]},"logprobs":null,"finish_reason":null}]}

data: {"id":"chatcmpl-9A0pokSpYASGLPfIzV8jx6tuvBUtz","object":"chat.completion.chunk","created":1712173516,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"}"}}]},"logprobs":null,"finish_reason":null}]}

data: {"id":"chatcmpl-9A0pokSpYASGLPfIzV8jx6tuvBUtz","object":"chat.completion.chunk","created":1712173516,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"tool_calls"}]}

data: [DONE]