Não pode escolher mais do que 25 tópicos Os tópicos devem começar com uma letra ou um número, podem incluir traços ('-') e podem ter até 35 caracteres.
 
 
 

536 linhas
15 KiB

  1. package openai
  2. import (
  3. "fmt"
  4. "io"
  5. "net/http"
  6. "strings"
  7. "time"
  8. "github.com/QuantumNous/new-api/common"
  9. "github.com/QuantumNous/new-api/dto"
  10. "github.com/QuantumNous/new-api/logger"
  11. relaycommon "github.com/QuantumNous/new-api/relay/common"
  12. "github.com/QuantumNous/new-api/relay/helper"
  13. "github.com/QuantumNous/new-api/service"
  14. "github.com/QuantumNous/new-api/types"
  15. "github.com/gin-gonic/gin"
  16. )
  17. func responsesStreamIndexKey(itemID string, idx *int) string {
  18. if itemID == "" {
  19. return ""
  20. }
  21. if idx == nil {
  22. return itemID
  23. }
  24. return fmt.Sprintf("%s:%d", itemID, *idx)
  25. }
  26. func stringDeltaFromPrefix(prev string, next string) string {
  27. if next == "" {
  28. return ""
  29. }
  30. if prev != "" && strings.HasPrefix(next, prev) {
  31. return next[len(prev):]
  32. }
  33. return next
  34. }
  35. func OaiResponsesToChatHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *http.Response) (*dto.Usage, *types.NewAPIError) {
  36. if resp == nil || resp.Body == nil {
  37. return nil, types.NewOpenAIError(fmt.Errorf("invalid response"), types.ErrorCodeBadResponse, http.StatusInternalServerError)
  38. }
  39. defer service.CloseResponseBodyGracefully(resp)
  40. var responsesResp dto.OpenAIResponsesResponse
  41. body, err := io.ReadAll(resp.Body)
  42. if err != nil {
  43. return nil, types.NewOpenAIError(err, types.ErrorCodeReadResponseBodyFailed, http.StatusInternalServerError)
  44. }
  45. if err := common.Unmarshal(body, &responsesResp); err != nil {
  46. return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponseBody, http.StatusInternalServerError)
  47. }
  48. if oaiError := responsesResp.GetOpenAIError(); oaiError != nil && oaiError.Type != "" {
  49. apiErr := types.WithOpenAIError(*oaiError, resp.StatusCode)
  50. apiErr.UpstreamBody = service.TruncateBody(string(body))
  51. return nil, apiErr
  52. }
  53. chatId := helper.GetResponseID(c)
  54. chatResp, usage, err := service.ResponsesResponseToChatCompletionsResponse(&responsesResp, chatId)
  55. if err != nil {
  56. return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponseBody, http.StatusInternalServerError)
  57. }
  58. if usage == nil || usage.TotalTokens == 0 {
  59. text := service.ExtractOutputTextFromResponses(&responsesResp)
  60. usage = service.ResponseText2Usage(c, text, info.UpstreamModelName, info.GetEstimatePromptTokens())
  61. chatResp.Usage = *usage
  62. }
  63. var responseBody []byte
  64. switch info.RelayFormat {
  65. case types.RelayFormatClaude:
  66. claudeResp := service.ResponseOpenAI2Claude(chatResp, info)
  67. responseBody, err = common.Marshal(claudeResp)
  68. case types.RelayFormatGemini:
  69. geminiResp := service.ResponseOpenAI2Gemini(chatResp, info)
  70. responseBody, err = common.Marshal(geminiResp)
  71. default:
  72. responseBody, err = common.Marshal(chatResp)
  73. }
  74. if err != nil {
  75. return nil, types.NewOpenAIError(err, types.ErrorCodeJsonMarshalFailed, http.StatusInternalServerError)
  76. }
  77. service.IOCopyBytesGracefully(c, resp, responseBody)
  78. return usage, nil
  79. }
  80. func OaiResponsesToChatStreamHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *http.Response) (*dto.Usage, *types.NewAPIError) {
  81. if resp == nil || resp.Body == nil {
  82. return nil, types.NewOpenAIError(fmt.Errorf("invalid response"), types.ErrorCodeBadResponse, http.StatusInternalServerError)
  83. }
  84. defer service.CloseResponseBodyGracefully(resp)
  85. responseId := helper.GetResponseID(c)
  86. createAt := time.Now().Unix()
  87. model := info.UpstreamModelName
  88. var (
  89. usage = &dto.Usage{}
  90. outputText strings.Builder
  91. usageText strings.Builder
  92. sentStart bool
  93. sentStop bool
  94. sawToolCall bool
  95. streamErr *types.NewAPIError
  96. )
  97. toolCallIndexByID := make(map[string]int)
  98. toolCallNameByID := make(map[string]string)
  99. toolCallArgsByID := make(map[string]string)
  100. toolCallNameSent := make(map[string]bool)
  101. toolCallCanonicalIDByItemID := make(map[string]string)
  102. hasSentReasoningSummary := false
  103. needsReasoningSummarySeparator := false
  104. //reasoningSummaryTextByKey := make(map[string]string)
  105. if info.RelayFormat == types.RelayFormatClaude && info.ClaudeConvertInfo == nil {
  106. info.ClaudeConvertInfo = &relaycommon.ClaudeConvertInfo{LastMessagesType: relaycommon.LastMessageTypeNone}
  107. }
  108. sendChatChunk := func(chunk *dto.ChatCompletionsStreamResponse) bool {
  109. if chunk == nil {
  110. return true
  111. }
  112. if info.RelayFormat == types.RelayFormatOpenAI {
  113. if err := helper.ObjectData(c, chunk); err != nil {
  114. streamErr = types.NewOpenAIError(err, types.ErrorCodeBadResponse, http.StatusInternalServerError)
  115. return false
  116. }
  117. return true
  118. }
  119. chunkData, err := common.Marshal(chunk)
  120. if err != nil {
  121. streamErr = types.NewOpenAIError(err, types.ErrorCodeJsonMarshalFailed, http.StatusInternalServerError)
  122. return false
  123. }
  124. if err := HandleStreamFormat(c, info, string(chunkData), false, false); err != nil {
  125. streamErr = types.NewOpenAIError(err, types.ErrorCodeBadResponse, http.StatusInternalServerError)
  126. return false
  127. }
  128. return true
  129. }
  130. sendStartIfNeeded := func() bool {
  131. if sentStart {
  132. return true
  133. }
  134. if !sendChatChunk(helper.GenerateStartEmptyResponse(responseId, createAt, model, nil)) {
  135. return false
  136. }
  137. sentStart = true
  138. return true
  139. }
  140. //sendReasoningDelta := func(delta string) bool {
  141. // if delta == "" {
  142. // return true
  143. // }
  144. // if !sendStartIfNeeded() {
  145. // return false
  146. // }
  147. //
  148. // usageText.WriteString(delta)
  149. // chunk := &dto.ChatCompletionsStreamResponse{
  150. // Id: responseId,
  151. // Object: "chat.completion.chunk",
  152. // Created: createAt,
  153. // Model: model,
  154. // Choices: []dto.ChatCompletionsStreamResponseChoice{
  155. // {
  156. // Index: 0,
  157. // Delta: dto.ChatCompletionsStreamResponseChoiceDelta{
  158. // ReasoningContent: &delta,
  159. // },
  160. // },
  161. // },
  162. // }
  163. // if err := helper.ObjectData(c, chunk); err != nil {
  164. // streamErr = types.NewOpenAIError(err, types.ErrorCodeBadResponse, http.StatusInternalServerError)
  165. // return false
  166. // }
  167. // return true
  168. //}
  169. sendReasoningSummaryDelta := func(delta string) bool {
  170. if delta == "" {
  171. return true
  172. }
  173. if needsReasoningSummarySeparator {
  174. if strings.HasPrefix(delta, "\n\n") {
  175. needsReasoningSummarySeparator = false
  176. } else if strings.HasPrefix(delta, "\n") {
  177. delta = "\n" + delta
  178. needsReasoningSummarySeparator = false
  179. } else {
  180. delta = "\n\n" + delta
  181. needsReasoningSummarySeparator = false
  182. }
  183. }
  184. if !sendStartIfNeeded() {
  185. return false
  186. }
  187. usageText.WriteString(delta)
  188. chunk := &dto.ChatCompletionsStreamResponse{
  189. Id: responseId,
  190. Object: "chat.completion.chunk",
  191. Created: createAt,
  192. Model: model,
  193. Choices: []dto.ChatCompletionsStreamResponseChoice{
  194. {
  195. Index: 0,
  196. Delta: dto.ChatCompletionsStreamResponseChoiceDelta{
  197. ReasoningContent: &delta,
  198. },
  199. },
  200. },
  201. }
  202. if !sendChatChunk(chunk) {
  203. return false
  204. }
  205. hasSentReasoningSummary = true
  206. return true
  207. }
  208. sendToolCallDelta := func(callID string, name string, argsDelta string) bool {
  209. if callID == "" {
  210. return true
  211. }
  212. if outputText.Len() > 0 {
  213. // Prefer streaming assistant text over tool calls to match non-stream behavior.
  214. return true
  215. }
  216. if !sendStartIfNeeded() {
  217. return false
  218. }
  219. idx, ok := toolCallIndexByID[callID]
  220. if !ok {
  221. idx = len(toolCallIndexByID)
  222. toolCallIndexByID[callID] = idx
  223. }
  224. if name != "" {
  225. toolCallNameByID[callID] = name
  226. }
  227. if toolCallNameByID[callID] != "" {
  228. name = toolCallNameByID[callID]
  229. }
  230. tool := dto.ToolCallResponse{
  231. ID: callID,
  232. Type: "function",
  233. Function: dto.FunctionResponse{
  234. Arguments: argsDelta,
  235. },
  236. }
  237. tool.SetIndex(idx)
  238. if name != "" && !toolCallNameSent[callID] {
  239. tool.Function.Name = name
  240. toolCallNameSent[callID] = true
  241. }
  242. chunk := &dto.ChatCompletionsStreamResponse{
  243. Id: responseId,
  244. Object: "chat.completion.chunk",
  245. Created: createAt,
  246. Model: model,
  247. Choices: []dto.ChatCompletionsStreamResponseChoice{
  248. {
  249. Index: 0,
  250. Delta: dto.ChatCompletionsStreamResponseChoiceDelta{
  251. ToolCalls: []dto.ToolCallResponse{tool},
  252. },
  253. },
  254. },
  255. }
  256. if !sendChatChunk(chunk) {
  257. return false
  258. }
  259. sawToolCall = true
  260. // Include tool call data in the local builder for fallback token estimation.
  261. if tool.Function.Name != "" {
  262. usageText.WriteString(tool.Function.Name)
  263. }
  264. if argsDelta != "" {
  265. usageText.WriteString(argsDelta)
  266. }
  267. return true
  268. }
  269. helper.StreamScannerHandler(c, resp, info, func(data string) bool {
  270. if streamErr != nil {
  271. return false
  272. }
  273. var streamResp dto.ResponsesStreamResponse
  274. if err := common.UnmarshalJsonStr(data, &streamResp); err != nil {
  275. logger.LogError(c, "failed to unmarshal responses stream event: "+err.Error())
  276. return true
  277. }
  278. switch streamResp.Type {
  279. case "response.created":
  280. if streamResp.Response != nil {
  281. if streamResp.Response.Model != "" {
  282. model = streamResp.Response.Model
  283. }
  284. if streamResp.Response.CreatedAt != 0 {
  285. createAt = int64(streamResp.Response.CreatedAt)
  286. }
  287. }
  288. //case "response.reasoning_text.delta":
  289. //if !sendReasoningDelta(streamResp.Delta) {
  290. // return false
  291. //}
  292. //case "response.reasoning_text.done":
  293. case "response.reasoning_summary_text.delta":
  294. if !sendReasoningSummaryDelta(streamResp.Delta) {
  295. return false
  296. }
  297. case "response.reasoning_summary_text.done":
  298. if hasSentReasoningSummary {
  299. needsReasoningSummarySeparator = true
  300. }
  301. //case "response.reasoning_summary_part.added", "response.reasoning_summary_part.done":
  302. // key := responsesStreamIndexKey(strings.TrimSpace(streamResp.ItemID), streamResp.SummaryIndex)
  303. // if key == "" || streamResp.Part == nil {
  304. // break
  305. // }
  306. // // Only handle summary text parts, ignore other part types.
  307. // if streamResp.Part.Type != "" && streamResp.Part.Type != "summary_text" {
  308. // break
  309. // }
  310. // prev := reasoningSummaryTextByKey[key]
  311. // next := streamResp.Part.Text
  312. // delta := stringDeltaFromPrefix(prev, next)
  313. // reasoningSummaryTextByKey[key] = next
  314. // if !sendReasoningSummaryDelta(delta) {
  315. // return false
  316. // }
  317. case "response.output_text.delta":
  318. if !sendStartIfNeeded() {
  319. return false
  320. }
  321. if streamResp.Delta != "" {
  322. outputText.WriteString(streamResp.Delta)
  323. usageText.WriteString(streamResp.Delta)
  324. delta := streamResp.Delta
  325. chunk := &dto.ChatCompletionsStreamResponse{
  326. Id: responseId,
  327. Object: "chat.completion.chunk",
  328. Created: createAt,
  329. Model: model,
  330. Choices: []dto.ChatCompletionsStreamResponseChoice{
  331. {
  332. Index: 0,
  333. Delta: dto.ChatCompletionsStreamResponseChoiceDelta{
  334. Content: &delta,
  335. },
  336. },
  337. },
  338. }
  339. if !sendChatChunk(chunk) {
  340. return false
  341. }
  342. }
  343. case "response.output_item.added", "response.output_item.done":
  344. if streamResp.Item == nil {
  345. break
  346. }
  347. if streamResp.Item.Type != "function_call" {
  348. break
  349. }
  350. itemID := strings.TrimSpace(streamResp.Item.ID)
  351. callID := strings.TrimSpace(streamResp.Item.CallId)
  352. if callID == "" {
  353. callID = itemID
  354. }
  355. if itemID != "" && callID != "" {
  356. toolCallCanonicalIDByItemID[itemID] = callID
  357. }
  358. name := strings.TrimSpace(streamResp.Item.Name)
  359. if name != "" {
  360. toolCallNameByID[callID] = name
  361. }
  362. newArgs := streamResp.Item.Arguments
  363. prevArgs := toolCallArgsByID[callID]
  364. argsDelta := ""
  365. if newArgs != "" {
  366. if strings.HasPrefix(newArgs, prevArgs) {
  367. argsDelta = newArgs[len(prevArgs):]
  368. } else {
  369. argsDelta = newArgs
  370. }
  371. toolCallArgsByID[callID] = newArgs
  372. }
  373. if !sendToolCallDelta(callID, name, argsDelta) {
  374. return false
  375. }
  376. case "response.function_call_arguments.delta":
  377. itemID := strings.TrimSpace(streamResp.ItemID)
  378. callID := toolCallCanonicalIDByItemID[itemID]
  379. if callID == "" {
  380. callID = itemID
  381. }
  382. if callID == "" {
  383. break
  384. }
  385. toolCallArgsByID[callID] += streamResp.Delta
  386. if !sendToolCallDelta(callID, "", streamResp.Delta) {
  387. return false
  388. }
  389. case "response.function_call_arguments.done":
  390. case "response.completed":
  391. if streamResp.Response != nil {
  392. if streamResp.Response.Model != "" {
  393. model = streamResp.Response.Model
  394. }
  395. if streamResp.Response.CreatedAt != 0 {
  396. createAt = int64(streamResp.Response.CreatedAt)
  397. }
  398. if streamResp.Response.Usage != nil {
  399. if streamResp.Response.Usage.InputTokens != 0 {
  400. usage.PromptTokens = streamResp.Response.Usage.InputTokens
  401. usage.InputTokens = streamResp.Response.Usage.InputTokens
  402. }
  403. if streamResp.Response.Usage.OutputTokens != 0 {
  404. usage.CompletionTokens = streamResp.Response.Usage.OutputTokens
  405. usage.OutputTokens = streamResp.Response.Usage.OutputTokens
  406. }
  407. if streamResp.Response.Usage.TotalTokens != 0 {
  408. usage.TotalTokens = streamResp.Response.Usage.TotalTokens
  409. } else {
  410. usage.TotalTokens = usage.PromptTokens + usage.CompletionTokens
  411. }
  412. if streamResp.Response.Usage.InputTokensDetails != nil {
  413. usage.PromptTokensDetails.CachedTokens = streamResp.Response.Usage.InputTokensDetails.CachedTokens
  414. usage.PromptTokensDetails.ImageTokens = streamResp.Response.Usage.InputTokensDetails.ImageTokens
  415. usage.PromptTokensDetails.AudioTokens = streamResp.Response.Usage.InputTokensDetails.AudioTokens
  416. }
  417. if streamResp.Response.Usage.CompletionTokenDetails.ReasoningTokens != 0 {
  418. usage.CompletionTokenDetails.ReasoningTokens = streamResp.Response.Usage.CompletionTokenDetails.ReasoningTokens
  419. }
  420. }
  421. }
  422. if !sendStartIfNeeded() {
  423. return false
  424. }
  425. if !sentStop {
  426. if info.RelayFormat == types.RelayFormatClaude && info.ClaudeConvertInfo != nil {
  427. info.ClaudeConvertInfo.Usage = usage
  428. }
  429. finishReason := "stop"
  430. if sawToolCall && outputText.Len() == 0 {
  431. finishReason = "tool_calls"
  432. }
  433. stop := helper.GenerateStopResponse(responseId, createAt, model, finishReason)
  434. if !sendChatChunk(stop) {
  435. return false
  436. }
  437. sentStop = true
  438. }
  439. case "response.error", "response.failed", "error":
  440. streamErr = handleResponsesStreamError(streamResp, data)
  441. return false
  442. default:
  443. }
  444. return true
  445. })
  446. if streamErr != nil {
  447. return nil, streamErr
  448. }
  449. if usage.TotalTokens == 0 {
  450. usage = service.ResponseText2Usage(c, usageText.String(), info.UpstreamModelName, info.GetEstimatePromptTokens())
  451. }
  452. if !sentStart {
  453. if !sendChatChunk(helper.GenerateStartEmptyResponse(responseId, createAt, model, nil)) {
  454. return nil, streamErr
  455. }
  456. }
  457. if !sentStop {
  458. if info.RelayFormat == types.RelayFormatClaude && info.ClaudeConvertInfo != nil {
  459. info.ClaudeConvertInfo.Usage = usage
  460. }
  461. finishReason := "stop"
  462. if sawToolCall && outputText.Len() == 0 {
  463. finishReason = "tool_calls"
  464. }
  465. stop := helper.GenerateStopResponse(responseId, createAt, model, finishReason)
  466. if !sendChatChunk(stop) {
  467. return nil, streamErr
  468. }
  469. }
  470. if info.RelayFormat == types.RelayFormatOpenAI && info.ShouldIncludeUsage && usage != nil {
  471. if err := helper.ObjectData(c, helper.GenerateFinalUsageResponse(responseId, createAt, model, *usage)); err != nil {
  472. return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponse, http.StatusInternalServerError)
  473. }
  474. }
  475. if info.RelayFormat == types.RelayFormatOpenAI {
  476. helper.Done(c)
  477. }
  478. return usage, nil
  479. }