merge upstream

Signed-off-by: wozulong <>

dto/audio.go

@@ -1,13 +1,34 @@
 package dto
 
-type TextToSpeechRequest struct {
-	Model          string  `json:"model" binding:"required"`
-	Input          string  `json:"input" binding:"required"`
-	Voice          string  `json:"voice" binding:"required"`
-	Speed          float64 `json:"speed"`
-	ResponseFormat string  `json:"response_format"`
+type AudioRequest struct {
+	Model          string  `json:"model"`
+	Input          string  `json:"input"`
+	Voice          string  `json:"voice"`
+	Speed          float64 `json:"speed,omitempty"`
+	ResponseFormat string  `json:"response_format,omitempty"`
 }
 
 type AudioResponse struct {
 	Text string `json:"text"`
 }
+
+type WhisperVerboseJSONResponse struct {
+	Task     string    `json:"task,omitempty"`
+	Language string    `json:"language,omitempty"`
+	Duration float64   `json:"duration,omitempty"`
+	Text     string    `json:"text,omitempty"`
+	Segments []Segment `json:"segments,omitempty"`
+}
+
+type Segment struct {
+	Id               int     `json:"id"`
+	Seek             int     `json:"seek"`
+	Start            float64 `json:"start"`
+	End              float64 `json:"end"`
+	Text             string  `json:"text"`
+	Tokens           []int   `json:"tokens"`
+	Temperature      float64 `json:"temperature"`
+	AvgLogprob       float64 `json:"avg_logprob"`
+	CompressionRatio float64 `json:"compression_ratio"`
+	NoSpeechProb     float64 `json:"no_speech_prob"`
+}

dto/dalle.go

@@ -12,9 +12,11 @@ type ImageRequest struct {
 }
 
 type ImageResponse struct {
-	Created int `json:"created"`
-	Data    []struct {
-		Url     string `json:"url"`
-		B64Json string `json:"b64_json"`
-	}
+	Data    []ImageData `json:"data"`
+	Created int64       `json:"created"`
+}
+type ImageData struct {
+	Url           string `json:"url"`
+	B64Json       string `json:"b64_json"`
+	RevisedPrompt string `json:"revised_prompt"`
 }

dto/text_request.go

@@ -29,12 +29,13 @@ type GeneralOpenAIRequest struct {
 	PresencePenalty  float64         `json:"presence_penalty,omitempty"`
 	ResponseFormat   *ResponseFormat `json:"response_format,omitempty"`
 	Seed             float64         `json:"seed,omitempty"`
-	Tools            any             `json:"tools,omitempty"`
+	Tools            []ToolCall      `json:"tools,omitempty"`
 	ToolChoice       any             `json:"tool_choice,omitempty"`
 	User             string          `json:"user,omitempty"`
 	LogitBias        any             `json:"logit_bias,omitempty"`
 	LogProbs         any             `json:"logprobs,omitempty"`
 	TopLogProbs      int             `json:"top_logprobs,omitempty"`
+	Dimensions       int             `json:"dimensions,omitempty"`
 }
 
 type OpenAITools struct {
@@ -52,8 +53,8 @@ type StreamOptions struct {
 	IncludeUsage bool `json:"include_usage,omitempty"`
 }
 
-func (r GeneralOpenAIRequest) GetMaxTokens() int64 {
-	return int64(r.MaxTokens)
+func (r GeneralOpenAIRequest) GetMaxTokens() int {
+	return int(r.MaxTokens)
 }
 
 func (r GeneralOpenAIRequest) ParseInput() []string {
@@ -107,6 +108,11 @@ func (m Message) StringContent() string {
 	return string(m.Content)
 }
 
+func (m *Message) SetStringContent(content string) {
+	jsonContent, _ := json.Marshal(content)
+	m.Content = jsonContent
+}
+
 func (m Message) IsStringContent() bool {
 	var stringContent string
 	if err := json.Unmarshal(m.Content, &stringContent); err == nil {
@@ -146,7 +152,7 @@ func (m Message) ParseContent() []MediaMessage {
 					if ok {
 						subObj["detail"] = detail.(string)
 					} else {
-						subObj["detail"] = "auto"
+						subObj["detail"] = "high"
 					}
 					contentList = append(contentList, MediaMessage{
 						Type: ContentTypeImageURL,
@@ -155,7 +161,16 @@ func (m Message) ParseContent() []MediaMessage {
 							Detail: subObj["detail"].(string),
 						},
 					})
+				} else if url, ok := contentMap["image_url"].(string); ok {
+					contentList = append(contentList, MediaMessage{
+						Type: ContentTypeImageURL,
+						ImageUrl: MessageImageUrl{
+							Url:    url,
+							Detail: "high",
+						},
+					})
 				}
+
 			}
 		}
 		return contentList

dto/text_response.go

@@ -66,10 +66,6 @@ type ChatCompletionsStreamResponseChoiceDelta struct {
 	ToolCalls []ToolCall `json:"tool_calls,omitempty"`
 }
 
-func (c *ChatCompletionsStreamResponseChoiceDelta) IsEmpty() bool {
-	return c.Content == nil && len(c.ToolCalls) == 0
-}
-
 func (c *ChatCompletionsStreamResponseChoiceDelta) SetContentString(s string) {
 	c.Content = &s
 }
@@ -90,9 +86,11 @@ type ToolCall struct {
 }
 
 type FunctionCall struct {
-	Name string `json:"name,omitempty"`
+	Description string `json:"description,omitempty"`
+	Name        string `json:"name,omitempty"`
 	// call function with arguments in JSON format
-	Arguments string `json:"arguments,omitempty"`
+	Parameters any    `json:"parameters,omitempty"` // request
+	Arguments  string `json:"arguments,omitempty"`
 }
 
 type ChatCompletionsStreamResponse struct {
@@ -105,6 +103,17 @@ type ChatCompletionsStreamResponse struct {
 	Usage             *Usage                                `json:"usage"`
 }
 
+func (c *ChatCompletionsStreamResponse) GetSystemFingerprint() string {
+	if c.SystemFingerprint == nil {
+		return ""
+	}
+	return *c.SystemFingerprint
+}
+
+func (c *ChatCompletionsStreamResponse) SetSystemFingerprint(s string) {
+	c.SystemFingerprint = &s
+}
+
 type ChatCompletionsStreamResponseSimple struct {
 	Choices []ChatCompletionsStreamResponseChoice `json:"choices"`
 	Usage   *Usage                                `json:"usage"`