Du kannst nicht mehr als 25 Themen auswählen Themen müssen entweder mit einem Buchstaben oder einer Ziffer beginnen. Sie können Bindestriche („-“) enthalten und bis zu 35 Zeichen lang sein.
 
 
 

584 Zeilen
20 KiB

  1. package dto
  2. import (
  3. "encoding/json"
  4. "strings"
  5. "github.com/QuantumNous/new-api/common"
  6. "github.com/QuantumNous/new-api/logger"
  7. "github.com/QuantumNous/new-api/types"
  8. "github.com/gin-gonic/gin"
  9. )
  10. type GeminiChatRequest struct {
  11. Requests []GeminiChatRequest `json:"requests,omitempty"` // For batch requests
  12. Contents []GeminiChatContent `json:"contents"`
  13. SafetySettings []GeminiChatSafetySettings `json:"safetySettings,omitempty"`
  14. GenerationConfig GeminiChatGenerationConfig `json:"generationConfig,omitempty"`
  15. Tools json.RawMessage `json:"tools,omitempty"`
  16. ToolConfig *ToolConfig `json:"toolConfig,omitempty"`
  17. SystemInstructions *GeminiChatContent `json:"systemInstruction,omitempty"`
  18. CachedContent string `json:"cachedContent,omitempty"`
  19. }
  20. // UnmarshalJSON allows GeminiChatRequest to accept both snake_case and camelCase fields.
  21. func (r *GeminiChatRequest) UnmarshalJSON(data []byte) error {
  22. type Alias GeminiChatRequest
  23. var aux struct {
  24. Alias
  25. SystemInstructionSnake *GeminiChatContent `json:"system_instruction,omitempty"`
  26. }
  27. if err := common.Unmarshal(data, &aux); err != nil {
  28. return err
  29. }
  30. *r = GeminiChatRequest(aux.Alias)
  31. if aux.SystemInstructionSnake != nil {
  32. r.SystemInstructions = aux.SystemInstructionSnake
  33. }
  34. return nil
  35. }
  36. type ToolConfig struct {
  37. FunctionCallingConfig *FunctionCallingConfig `json:"functionCallingConfig,omitempty"`
  38. RetrievalConfig *RetrievalConfig `json:"retrievalConfig,omitempty"`
  39. }
  40. type FunctionCallingConfig struct {
  41. Mode FunctionCallingConfigMode `json:"mode,omitempty"`
  42. AllowedFunctionNames []string `json:"allowedFunctionNames,omitempty"`
  43. }
  44. type FunctionCallingConfigMode string
  45. type RetrievalConfig struct {
  46. LatLng *LatLng `json:"latLng,omitempty"`
  47. LanguageCode string `json:"languageCode,omitempty"`
  48. }
  49. type LatLng struct {
  50. Latitude *float64 `json:"latitude,omitempty"`
  51. Longitude *float64 `json:"longitude,omitempty"`
  52. }
  53. // createGeminiFileSource 根据数据内容创建正确类型的 FileSource
  54. func createGeminiFileSource(data string, mimeType string) *types.FileSource {
  55. if strings.HasPrefix(data, "http://") || strings.HasPrefix(data, "https://") {
  56. return types.NewURLFileSource(data)
  57. }
  58. return types.NewBase64FileSource(data, mimeType)
  59. }
  60. func (r *GeminiChatRequest) GetTokenCountMeta() *types.TokenCountMeta {
  61. var files []*types.FileMeta = make([]*types.FileMeta, 0)
  62. var maxTokens int
  63. if r.GenerationConfig.MaxOutputTokens > 0 {
  64. maxTokens = int(r.GenerationConfig.MaxOutputTokens)
  65. }
  66. var inputTexts []string
  67. for _, content := range r.Contents {
  68. for _, part := range content.Parts {
  69. if part.Text != "" {
  70. inputTexts = append(inputTexts, part.Text)
  71. }
  72. if part.InlineData != nil && part.InlineData.Data != "" {
  73. mimeType := part.InlineData.MimeType
  74. source := createGeminiFileSource(part.InlineData.Data, mimeType)
  75. var fileType types.FileType
  76. if strings.HasPrefix(mimeType, "image/") {
  77. fileType = types.FileTypeImage
  78. } else if strings.HasPrefix(mimeType, "audio/") {
  79. fileType = types.FileTypeAudio
  80. } else if strings.HasPrefix(mimeType, "video/") {
  81. fileType = types.FileTypeVideo
  82. } else {
  83. fileType = types.FileTypeFile
  84. }
  85. files = append(files, &types.FileMeta{
  86. FileType: fileType,
  87. Source: source,
  88. MimeType: mimeType,
  89. })
  90. }
  91. }
  92. }
  93. inputText := strings.Join(inputTexts, "\n")
  94. return &types.TokenCountMeta{
  95. CombineText: inputText,
  96. Files: files,
  97. MaxTokens: maxTokens,
  98. }
  99. }
  100. func (r *GeminiChatRequest) IsStream(c *gin.Context) bool {
  101. if c.Query("alt") == "sse" {
  102. return true
  103. }
  104. // Native Gemini API uses URL action to indicate streaming:
  105. // /v1beta/models/{model}:streamGenerateContent
  106. if strings.Contains(c.Request.URL.Path, "streamGenerateContent") {
  107. return true
  108. }
  109. return false
  110. }
  111. func (r *GeminiChatRequest) SetModelName(modelName string) {
  112. // GeminiChatRequest does not have a model field, so this method does nothing.
  113. }
  114. func (r *GeminiChatRequest) GetTools() []GeminiChatTool {
  115. var tools []GeminiChatTool
  116. if strings.HasPrefix(string(r.Tools), "[") {
  117. // is array
  118. if err := common.Unmarshal(r.Tools, &tools); err != nil {
  119. logger.LogError(nil, "error_unmarshalling_tools: "+err.Error())
  120. return nil
  121. }
  122. } else if strings.HasPrefix(string(r.Tools), "{") {
  123. // is object
  124. singleTool := GeminiChatTool{}
  125. if err := common.Unmarshal(r.Tools, &singleTool); err != nil {
  126. logger.LogError(nil, "error_unmarshalling_single_tool: "+err.Error())
  127. return nil
  128. }
  129. tools = []GeminiChatTool{singleTool}
  130. }
  131. return tools
  132. }
  133. func (r *GeminiChatRequest) SetTools(tools []GeminiChatTool) {
  134. if len(tools) == 0 {
  135. r.Tools = json.RawMessage("[]")
  136. return
  137. }
  138. // Marshal the tools to JSON
  139. data, err := common.Marshal(tools)
  140. if err != nil {
  141. logger.LogError(nil, "error_marshalling_tools: "+err.Error())
  142. return
  143. }
  144. r.Tools = data
  145. }
  146. type GeminiThinkingConfig struct {
  147. IncludeThoughts bool `json:"includeThoughts,omitempty"`
  148. ThinkingBudget *int `json:"thinkingBudget,omitempty"`
  149. // TODO Conflict with thinkingbudget.
  150. ThinkingLevel string `json:"thinkingLevel,omitempty"`
  151. }
  152. // UnmarshalJSON allows GeminiThinkingConfig to accept both snake_case and camelCase fields.
  153. func (c *GeminiThinkingConfig) UnmarshalJSON(data []byte) error {
  154. type Alias GeminiThinkingConfig
  155. var aux struct {
  156. Alias
  157. IncludeThoughtsSnake *bool `json:"include_thoughts,omitempty"`
  158. ThinkingBudgetSnake *int `json:"thinking_budget,omitempty"`
  159. ThinkingLevelSnake string `json:"thinking_level,omitempty"`
  160. }
  161. if err := common.Unmarshal(data, &aux); err != nil {
  162. return err
  163. }
  164. *c = GeminiThinkingConfig(aux.Alias)
  165. if aux.IncludeThoughtsSnake != nil {
  166. c.IncludeThoughts = *aux.IncludeThoughtsSnake
  167. }
  168. if aux.ThinkingBudgetSnake != nil {
  169. c.ThinkingBudget = aux.ThinkingBudgetSnake
  170. }
  171. if aux.ThinkingLevelSnake != "" {
  172. c.ThinkingLevel = aux.ThinkingLevelSnake
  173. }
  174. return nil
  175. }
  176. func (c *GeminiThinkingConfig) SetThinkingBudget(budget int) {
  177. c.ThinkingBudget = &budget
  178. }
  179. type GeminiInlineData struct {
  180. MimeType string `json:"mimeType"`
  181. Data string `json:"data"`
  182. }
  183. // UnmarshalJSON custom unmarshaler for GeminiInlineData to support snake_case and camelCase for MimeType
  184. func (g *GeminiInlineData) UnmarshalJSON(data []byte) error {
  185. type Alias GeminiInlineData // Use type alias to avoid recursion
  186. var aux struct {
  187. Alias
  188. MimeTypeSnake string `json:"mime_type"`
  189. }
  190. if err := common.Unmarshal(data, &aux); err != nil {
  191. return err
  192. }
  193. *g = GeminiInlineData(aux.Alias) // Copy other fields if any in future
  194. // Prioritize snake_case if present
  195. if aux.MimeTypeSnake != "" {
  196. g.MimeType = aux.MimeTypeSnake
  197. } else if aux.MimeType != "" { // Fallback to camelCase from Alias
  198. g.MimeType = aux.MimeType
  199. }
  200. // g.Data would be populated by aux.Alias.Data
  201. return nil
  202. }
  203. type FunctionCall struct {
  204. FunctionName string `json:"name"`
  205. Arguments any `json:"args"`
  206. }
  207. type GeminiFunctionResponse struct {
  208. Name string `json:"name"`
  209. Response map[string]interface{} `json:"response"`
  210. WillContinue json.RawMessage `json:"willContinue,omitempty"`
  211. Scheduling json.RawMessage `json:"scheduling,omitempty"`
  212. Parts json.RawMessage `json:"parts,omitempty"`
  213. ID json.RawMessage `json:"id,omitempty"`
  214. }
  215. type GeminiPartExecutableCode struct {
  216. Language string `json:"language,omitempty"`
  217. Code string `json:"code,omitempty"`
  218. }
  219. type GeminiPartCodeExecutionResult struct {
  220. Outcome string `json:"outcome,omitempty"`
  221. Output string `json:"output,omitempty"`
  222. }
  223. type GeminiFileData struct {
  224. MimeType string `json:"mimeType,omitempty"`
  225. FileUri string `json:"fileUri,omitempty"`
  226. }
  227. type GeminiPart struct {
  228. Text string `json:"text,omitempty"`
  229. Thought bool `json:"thought,omitempty"`
  230. InlineData *GeminiInlineData `json:"inlineData,omitempty"`
  231. FunctionCall *FunctionCall `json:"functionCall,omitempty"`
  232. ThoughtSignature json.RawMessage `json:"thoughtSignature,omitempty"`
  233. FunctionResponse *GeminiFunctionResponse `json:"functionResponse,omitempty"`
  234. // Optional. Media resolution for the input media.
  235. MediaResolution json.RawMessage `json:"mediaResolution,omitempty"`
  236. VideoMetadata json.RawMessage `json:"videoMetadata,omitempty"`
  237. FileData *GeminiFileData `json:"fileData,omitempty"`
  238. ExecutableCode *GeminiPartExecutableCode `json:"executableCode,omitempty"`
  239. CodeExecutionResult *GeminiPartCodeExecutionResult `json:"codeExecutionResult,omitempty"`
  240. }
  241. // UnmarshalJSON custom unmarshaler for GeminiPart to support snake_case and camelCase for InlineData
  242. func (p *GeminiPart) UnmarshalJSON(data []byte) error {
  243. // Alias to avoid recursion during unmarshalling
  244. type Alias GeminiPart
  245. var aux struct {
  246. Alias
  247. InlineDataSnake *GeminiInlineData `json:"inline_data,omitempty"` // snake_case variant
  248. }
  249. if err := common.Unmarshal(data, &aux); err != nil {
  250. return err
  251. }
  252. // Assign fields from alias
  253. *p = GeminiPart(aux.Alias)
  254. // Prioritize snake_case for InlineData if present
  255. if aux.InlineDataSnake != nil {
  256. p.InlineData = aux.InlineDataSnake
  257. } else if aux.InlineData != nil { // Fallback to camelCase from Alias
  258. p.InlineData = aux.InlineData
  259. }
  260. // Other fields like Text, FunctionCall etc. are already populated via aux.Alias
  261. return nil
  262. }
  263. type GeminiChatContent struct {
  264. Role string `json:"role,omitempty"`
  265. Parts []GeminiPart `json:"parts"`
  266. }
  267. type GeminiChatSafetySettings struct {
  268. Category string `json:"category"`
  269. Threshold string `json:"threshold"`
  270. }
  271. type GeminiChatTool struct {
  272. GoogleSearch any `json:"googleSearch,omitempty"`
  273. GoogleSearchRetrieval any `json:"googleSearchRetrieval,omitempty"`
  274. CodeExecution any `json:"codeExecution,omitempty"`
  275. FunctionDeclarations any `json:"functionDeclarations,omitempty"`
  276. URLContext any `json:"urlContext,omitempty"`
  277. }
  278. type GeminiChatGenerationConfig struct {
  279. Temperature *float64 `json:"temperature,omitempty"`
  280. TopP float64 `json:"topP,omitempty"`
  281. TopK float64 `json:"topK,omitempty"`
  282. MaxOutputTokens uint `json:"maxOutputTokens,omitempty"`
  283. CandidateCount int `json:"candidateCount,omitempty"`
  284. StopSequences []string `json:"stopSequences,omitempty"`
  285. ResponseMimeType string `json:"responseMimeType,omitempty"`
  286. ResponseSchema any `json:"responseSchema,omitempty"`
  287. ResponseJsonSchema json.RawMessage `json:"responseJsonSchema,omitempty"`
  288. PresencePenalty *float32 `json:"presencePenalty,omitempty"`
  289. FrequencyPenalty *float32 `json:"frequencyPenalty,omitempty"`
  290. ResponseLogprobs bool `json:"responseLogprobs,omitempty"`
  291. Logprobs *int32 `json:"logprobs,omitempty"`
  292. EnableEnhancedCivicAnswers *bool `json:"enableEnhancedCivicAnswers,omitempty"`
  293. MediaResolution MediaResolution `json:"mediaResolution,omitempty"`
  294. Seed int64 `json:"seed,omitempty"`
  295. ResponseModalities []string `json:"responseModalities,omitempty"`
  296. ThinkingConfig *GeminiThinkingConfig `json:"thinkingConfig,omitempty"`
  297. SpeechConfig json.RawMessage `json:"speechConfig,omitempty"` // RawMessage to allow flexible speech config
  298. ImageConfig json.RawMessage `json:"imageConfig,omitempty"` // RawMessage to allow flexible image config
  299. }
  300. // UnmarshalJSON allows GeminiChatGenerationConfig to accept both snake_case and camelCase fields.
  301. func (c *GeminiChatGenerationConfig) UnmarshalJSON(data []byte) error {
  302. type Alias GeminiChatGenerationConfig
  303. var aux struct {
  304. Alias
  305. TopPSnake float64 `json:"top_p,omitempty"`
  306. TopKSnake float64 `json:"top_k,omitempty"`
  307. MaxOutputTokensSnake uint `json:"max_output_tokens,omitempty"`
  308. CandidateCountSnake int `json:"candidate_count,omitempty"`
  309. StopSequencesSnake []string `json:"stop_sequences,omitempty"`
  310. ResponseMimeTypeSnake string `json:"response_mime_type,omitempty"`
  311. ResponseSchemaSnake any `json:"response_schema,omitempty"`
  312. ResponseJsonSchemaSnake json.RawMessage `json:"response_json_schema,omitempty"`
  313. PresencePenaltySnake *float32 `json:"presence_penalty,omitempty"`
  314. FrequencyPenaltySnake *float32 `json:"frequency_penalty,omitempty"`
  315. ResponseLogprobsSnake bool `json:"response_logprobs,omitempty"`
  316. EnableEnhancedCivicAnswersSnake *bool `json:"enable_enhanced_civic_answers,omitempty"`
  317. MediaResolutionSnake MediaResolution `json:"media_resolution,omitempty"`
  318. ResponseModalitiesSnake []string `json:"response_modalities,omitempty"`
  319. ThinkingConfigSnake *GeminiThinkingConfig `json:"thinking_config,omitempty"`
  320. SpeechConfigSnake json.RawMessage `json:"speech_config,omitempty"`
  321. ImageConfigSnake json.RawMessage `json:"image_config,omitempty"`
  322. }
  323. if err := common.Unmarshal(data, &aux); err != nil {
  324. return err
  325. }
  326. *c = GeminiChatGenerationConfig(aux.Alias)
  327. // Prioritize snake_case if present
  328. if aux.TopPSnake != 0 {
  329. c.TopP = aux.TopPSnake
  330. }
  331. if aux.TopKSnake != 0 {
  332. c.TopK = aux.TopKSnake
  333. }
  334. if aux.MaxOutputTokensSnake != 0 {
  335. c.MaxOutputTokens = aux.MaxOutputTokensSnake
  336. }
  337. if aux.CandidateCountSnake != 0 {
  338. c.CandidateCount = aux.CandidateCountSnake
  339. }
  340. if len(aux.StopSequencesSnake) > 0 {
  341. c.StopSequences = aux.StopSequencesSnake
  342. }
  343. if aux.ResponseMimeTypeSnake != "" {
  344. c.ResponseMimeType = aux.ResponseMimeTypeSnake
  345. }
  346. if aux.ResponseSchemaSnake != nil {
  347. c.ResponseSchema = aux.ResponseSchemaSnake
  348. }
  349. if len(aux.ResponseJsonSchemaSnake) > 0 {
  350. c.ResponseJsonSchema = aux.ResponseJsonSchemaSnake
  351. }
  352. if aux.PresencePenaltySnake != nil {
  353. c.PresencePenalty = aux.PresencePenaltySnake
  354. }
  355. if aux.FrequencyPenaltySnake != nil {
  356. c.FrequencyPenalty = aux.FrequencyPenaltySnake
  357. }
  358. if aux.ResponseLogprobsSnake {
  359. c.ResponseLogprobs = aux.ResponseLogprobsSnake
  360. }
  361. if aux.EnableEnhancedCivicAnswersSnake != nil {
  362. c.EnableEnhancedCivicAnswers = aux.EnableEnhancedCivicAnswersSnake
  363. }
  364. if aux.MediaResolutionSnake != "" {
  365. c.MediaResolution = aux.MediaResolutionSnake
  366. }
  367. if len(aux.ResponseModalitiesSnake) > 0 {
  368. c.ResponseModalities = aux.ResponseModalitiesSnake
  369. }
  370. if aux.ThinkingConfigSnake != nil {
  371. c.ThinkingConfig = aux.ThinkingConfigSnake
  372. }
  373. if len(aux.SpeechConfigSnake) > 0 {
  374. c.SpeechConfig = aux.SpeechConfigSnake
  375. }
  376. if len(aux.ImageConfigSnake) > 0 {
  377. c.ImageConfig = aux.ImageConfigSnake
  378. }
  379. return nil
  380. }
  381. type MediaResolution string
  382. type GeminiChatCandidate struct {
  383. Content GeminiChatContent `json:"content"`
  384. FinishReason *string `json:"finishReason"`
  385. Index int64 `json:"index"`
  386. SafetyRatings []GeminiChatSafetyRating `json:"safetyRatings"`
  387. }
  388. type GeminiChatSafetyRating struct {
  389. Category string `json:"category"`
  390. Probability string `json:"probability"`
  391. }
  392. type GeminiChatPromptFeedback struct {
  393. SafetyRatings []GeminiChatSafetyRating `json:"safetyRatings"`
  394. BlockReason *string `json:"blockReason,omitempty"`
  395. }
  396. type GeminiChatResponse struct {
  397. Candidates []GeminiChatCandidate `json:"candidates"`
  398. PromptFeedback *GeminiChatPromptFeedback `json:"promptFeedback,omitempty"`
  399. UsageMetadata GeminiUsageMetadata `json:"usageMetadata"`
  400. }
  401. type GeminiUsageMetadata struct {
  402. PromptTokenCount int `json:"promptTokenCount"`
  403. ToolUsePromptTokenCount int `json:"toolUsePromptTokenCount"`
  404. CandidatesTokenCount int `json:"candidatesTokenCount"`
  405. TotalTokenCount int `json:"totalTokenCount"`
  406. ThoughtsTokenCount int `json:"thoughtsTokenCount"`
  407. CachedContentTokenCount int `json:"cachedContentTokenCount"`
  408. PromptTokensDetails []GeminiPromptTokensDetails `json:"promptTokensDetails"`
  409. ToolUsePromptTokensDetails []GeminiPromptTokensDetails `json:"toolUsePromptTokensDetails"`
  410. }
  411. type GeminiPromptTokensDetails struct {
  412. Modality string `json:"modality"`
  413. TokenCount int `json:"tokenCount"`
  414. }
  415. // Imagen related structs
  416. type GeminiImageRequest struct {
  417. Instances []GeminiImageInstance `json:"instances"`
  418. Parameters GeminiImageParameters `json:"parameters"`
  419. }
  420. type GeminiImageInstance struct {
  421. Prompt string `json:"prompt"`
  422. }
  423. type GeminiImageParameters struct {
  424. SampleCount int `json:"sampleCount,omitempty"`
  425. AspectRatio string `json:"aspectRatio,omitempty"`
  426. PersonGeneration string `json:"personGeneration,omitempty"`
  427. ImageSize string `json:"imageSize,omitempty"`
  428. }
  429. type GeminiImageResponse struct {
  430. Predictions []GeminiImagePrediction `json:"predictions"`
  431. }
  432. type GeminiImagePrediction struct {
  433. MimeType string `json:"mimeType"`
  434. BytesBase64Encoded string `json:"bytesBase64Encoded"`
  435. RaiFilteredReason string `json:"raiFilteredReason,omitempty"`
  436. SafetyAttributes any `json:"safetyAttributes,omitempty"`
  437. }
  438. // Embedding related structs
  439. type GeminiEmbeddingRequest struct {
  440. Model string `json:"model,omitempty"`
  441. Content GeminiChatContent `json:"content"`
  442. TaskType string `json:"taskType,omitempty"`
  443. Title string `json:"title,omitempty"`
  444. OutputDimensionality int `json:"outputDimensionality,omitempty"`
  445. }
  446. func (r *GeminiEmbeddingRequest) IsStream(c *gin.Context) bool {
  447. // Gemini embedding requests are not streamed
  448. return false
  449. }
  450. func (r *GeminiEmbeddingRequest) GetTokenCountMeta() *types.TokenCountMeta {
  451. var inputTexts []string
  452. for _, part := range r.Content.Parts {
  453. if part.Text != "" {
  454. inputTexts = append(inputTexts, part.Text)
  455. }
  456. }
  457. inputText := strings.Join(inputTexts, "\n")
  458. return &types.TokenCountMeta{
  459. CombineText: inputText,
  460. }
  461. }
  462. func (r *GeminiEmbeddingRequest) SetModelName(modelName string) {
  463. if modelName != "" {
  464. r.Model = modelName
  465. }
  466. }
  467. type GeminiBatchEmbeddingRequest struct {
  468. Requests []*GeminiEmbeddingRequest `json:"requests"`
  469. }
  470. func (r *GeminiBatchEmbeddingRequest) IsStream(c *gin.Context) bool {
  471. // Gemini batch embedding requests are not streamed
  472. return false
  473. }
  474. func (r *GeminiBatchEmbeddingRequest) GetTokenCountMeta() *types.TokenCountMeta {
  475. var inputTexts []string
  476. for _, request := range r.Requests {
  477. meta := request.GetTokenCountMeta()
  478. if meta != nil && meta.CombineText != "" {
  479. inputTexts = append(inputTexts, meta.CombineText)
  480. }
  481. }
  482. inputText := strings.Join(inputTexts, "\n")
  483. return &types.TokenCountMeta{
  484. CombineText: inputText,
  485. }
  486. }
  487. func (r *GeminiBatchEmbeddingRequest) SetModelName(modelName string) {
  488. if modelName != "" {
  489. for _, req := range r.Requests {
  490. req.SetModelName(modelName)
  491. }
  492. }
  493. }
  494. type GeminiEmbeddingResponse struct {
  495. Embedding ContentEmbedding `json:"embedding"`
  496. }
  497. type GeminiBatchEmbeddingResponse struct {
  498. Embeddings []*ContentEmbedding `json:"embeddings"`
  499. }
  500. type ContentEmbedding struct {
  501. Values []float64 `json:"values"`
  502. }