use tiktoken_rs::p50k_base;

let bpe = p50k_base().unwrap();
let tokens = bpe.encode_with_special_tokens(
  "This is a sentence   with spaces"
);
println!("Token count: {}", tokens.len());

use tiktoken_rs::{get_chat_completion_max_tokens, ChatCompletionRequestMessage};

let messages = vec![
    ChatCompletionRequestMessage {
        content: "You are a helpful assistant that only speaks French.".to_string(),
        role: "system".to_string(),
        name: None,
    },
    ChatCompletionRequestMessage {
        content: "Hello, how are you?".to_string(),
        role: "user".to_string(),
        name: None,
    },
    ChatCompletionRequestMessage {
        content: "Parlez-vous francais?".to_string(),
        role: "system".to_string(),
        name: None,
    },
];
let max_tokens = get_chat_completion_max_tokens("gpt-4", &messages).unwrap();
println!("max_tokens: {}", max_tokens);

use tiktoken_rs::async_openai::get_chat_completion_max_tokens;
use async_openai::types::{ChatCompletionRequestMessage, Role};

let messages = vec![
    ChatCompletionRequestMessage {
        content: "You are a helpful assistant that only speaks French.".to_string(),
        role: Role::System,
        name: None,
    },
    ChatCompletionRequestMessage {
        content: "Hello, how are you?".to_string(),
        role: Role::User,
        name: None,
    },
    ChatCompletionRequestMessage {
        content: "Parlez-vous francais?".to_string(),
        role: Role::System,
        name: None,
    },
];
let max_tokens = get_chat_completion_max_tokens("gpt-4", &messages).unwrap();
println!("max_tokens: {}", max_tokens);