Streaming & Token Management

pip install openai tiktoken

import os
import time
from openai import OpenAI

# Initialize the client
client = OpenAI(api_key=os.getenv("OPENAI_API_KEY"))

# Let's define a model to use. 
# GPT-4o-mini is fast and cheap, perfect for testing.
MODEL = "gpt-4o-mini"

Step 2: implementing Streaming

Add this code to your file to see the raw chunks:

print("--- STREAMING DEMO ---")

# 1. Create the request with stream=True
stream = client.chat.completions.create(
    model=MODEL,
    messages=[{"role": "user", "content": "Count to 5 and tell me a joke."}],
    stream=True  # <--- This is the magic switch
)

# 2. Iterate through the stream
print("Receiving chunks...")
for chunk in stream:
    # 3. Extract the content delta
    # In streaming, we look for 'delta', not 'message'
    content = chunk.choices[0].delta.content
    
    if content:
        print(f"Chunk received: '{content}'")

You will see the output looks strange. It prints each word (or part of a word) on a new line.

Chunk received: 'Count'
Chunk received: 'ing'
Chunk received: ' to'
Chunk received: ' '
Chunk received: '5'
...

This proves it's working! We are receiving data instantly, piece by piece.

Step 3: Making it Look Like Typing

Replace the loop in your code with this version:

print("\n--- SMOOTH STREAMING ---")

stream = client.chat.completions.create(
    model=MODEL,
    messages=[{"role": "user", "content": "Explain quantum physics in one sentence."}],
    stream=True
)

print("AI: ", end="") # Start the line

full_response = ""

for chunk in stream:
    content = chunk.choices[0].delta.content
    
    if content:
        # Print to console without a newline
        print(content, end="", flush=True)
        
        # We also need to save the text to memory to use it later
        full_response += content

print() # Print a final newline at the very end
print("-" * 20)
print(f"Final stored string: {full_response}")

Step 4: Counting Tokens with Tiktoken

Now that we have the visual part handled, let's talk about the hidden cost.

LLMs read tokens. A token is roughly 0.75 words.

* "apple" = 1 token

* "ing" = 1 token

* "The" = 1 token

Add this helper function to the top of your script:

import tiktoken

def count_tokens(text, model="gpt-4o-mini"):
    try:
        # Get the encoding for the specific model
        encoding = tiktoken.encoding_for_model(model)
    except KeyError:
        # Fallback if model not found
        encoding = tiktoken.get_encoding("cl100k_base")
    
    # Encode the text into a list of integers (tokens)
    token_integers = encoding.encode(text)
    
    # Return the length of that list
    return len(token_integers)

# Test it out
test_sentence = "Streaming is super cool!"
print(f"\nSentence: '{test_sentence}'")
print(f"Tokens: {count_tokens(test_sentence)}")

Step 5: The Final Chatbot with Cost Calculation

Now we combine everything. We will build a loop that:

Here is the complete, runnable application:

import os
import tiktoken
from openai import OpenAI

# --- CONFIGURATION ---
client = OpenAI(api_key=os.getenv("OPENAI_API_KEY"))
MODEL = "gpt-4o-mini"
INPUT_PRICE_PER_1M = 0.15  # $0.15 per 1 million tokens
OUTPUT_PRICE_PER_1M = 0.60 # $0.60 per 1 million tokens

# --- HELPER FUNCTIONS ---
def count_tokens(text):
    """Returns the number of tokens in a text string."""
    try:
        encoding = tiktoken.encoding_for_model(MODEL)
    except KeyError:
        encoding = tiktoken.get_encoding("cl100k_base")
    return len(encoding.encode(text))

def calculate_cost(input_tokens, output_tokens):
    """Returns the cost in dollars."""
    input_cost = (input_tokens / 1_000_000) * INPUT_PRICE_PER_1M
    output_cost = (output_tokens / 1_000_000) * OUTPUT_PRICE_PER_1M
    return input_cost + output_cost

# --- MAIN APP ---
def main():
    print(f"Starting Chatbot ({MODEL})...")
    print("Type 'quit' to exit.\n")

    # We keep chat history to maintain context
    chat_history = [
        {"role": "system", "content": "You are a helpful, concise assistant."}
    ]

    total_session_cost = 0.0

    while True:
        user_input = input("\nYou: ")
        if user_input.lower() in ['quit', 'exit']:
            break

        # Add user message to history
        chat_history.append({"role": "user", "content": user_input})

        # 1. Calculate Input Tokens (we have to count the WHOLE history)
        # In a real app, we'd count the JSON overhead too, but this is a good approximation
        full_conversation_text = "".join([m["content"] for m in chat_history])
        input_tokens = count_tokens(full_conversation_text)

        print("AI: ", end="")
        
        # 2. Stream the Response
        stream = client.chat.completions.create(
            model=MODEL,
            messages=chat_history,
            stream=True
        )

        full_response_text = ""
        
        for chunk in stream:
            content = chunk.choices[0].delta.content
            if content:
                print(content, end="", flush=True)
                full_response_text += content
        
        print() # Newline after AI finishes

        # 3. Update History
        chat_history.append({"role": "assistant", "content": full_response_text})

        # 4. Calculate Output Tokens & Cost
        output_tokens = count_tokens(full_response_text)
        cost = calculate_cost(input_tokens, output_tokens)
        total_session_cost += cost

        # 5. Display Metrics
        print(f"\n[Metrics] In: {input_tokens} toks | Out: {output_tokens} toks | Cost: ${cost:.6f}")
        print(f"[Session Total]: ${total_session_cost:.6f}")

if __name__ == "__main__":
    main()

Understanding the Output

When you run this, notice two things:

Now You Try

You have a professional engine. Now let's tune it. Try these three extensions:

Challenge Project: The Infinite Memory (Summarizer)

* Take the oldest 3 messages (excluding the system prompt).

* Delete the old messages from the list.

* Insert a new message with role "system" that says: "Previous conversation summary: [INSERT SUMMARY HERE]".

* You cannot stream the summary generation; just use a standard call for that part.