new-api/service/token_estimator_test.go

package service

import (
	"testing"

	"github.com/stretchr/testify/require"
)

func TestEstimateTokenClassifiesWordNumberAndUnicodeBoundaries(t *testing.T) {
	tests := []struct {
		name string
		text string
		want int
	}{
		{name: "empty", text: "", want: 0},
		{name: "latin word", text: "hello", want: 2},
		{name: "latin number transition", text: "v2", want: 3},
		{name: "cjk", text: "你", want: 1},
		{name: "emoji", text: "🙂", want: 3},
		{name: "url delimiter", text: "/", want: 1},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			require.Equal(t, tt.want, EstimateToken(OpenAI, tt.text))
		})
	}
}

func TestEstimateTokenByModelSelectsProviderAndEmptyInputIsFree(t *testing.T) {
	require.Equal(t, 0, EstimateTokenByModel("gpt-5", ""))
	require.Equal(t, EstimateToken(Gemini, "hello"), EstimateTokenByModel("GEMINI-2.5-PRO", "hello"))
	require.Equal(t, EstimateToken(Claude, "hello"), EstimateTokenByModel("claude-sonnet", "hello"))
	require.Equal(t, EstimateToken(OpenAI, "hello"), EstimateTokenByModel("other-model", "hello"))
}