feat: add panic recover middleware

* feat: Refactor response parsing logic to support multiple formats

The parsing logic for responses in relay.go and relay-audio.go was refactored to support multiple response formats - 'json', 'text', 'srt', 'verbose_json', and 'vtt'. The existing `WhisperResponse` struct was renamed to `WhisperJsonResponse` and a new struct `WhisperVerboseJsonResponse` was added to support the 'verbose_json' format. Additional parsing functions were added to extract text from these new response types. This change was necessary to make the parsing logic more flexible and extendable for different types of responses.

* chore: update name

---------

Co-authored-by: JustSong <songquanpeng@foxmail.com>

common/constants.go

@@ -78,6 +78,7 @@ var QuotaForInviter = 0
 var QuotaForInvitee = 0
 var ChannelDisableThreshold = 5.0
 var AutomaticDisableChannelEnabled = false
+var AutomaticEnableChannelEnabled = false
 var QuotaRemindThreshold = 1000
 var PreConsumedQuota = 500
 var ApproximateTokenEnabled = false

common/image/image.go

@@ -0,0 +1,47 @@
+package image
+
+import (
+	"image"
+	_ "image/gif"
+	_ "image/jpeg"
+	_ "image/png"
+	"net/http"
+	"regexp"
+	"strings"
+
+	_ "golang.org/x/image/webp"
+)
+
+func GetImageSizeFromUrl(url string) (width int, height int, err error) {
+	resp, err := http.Get(url)
+	if err != nil {
+		return
+	}
+	defer resp.Body.Close()
+	img, _, err := image.DecodeConfig(resp.Body)
+	if err != nil {
+		return
+	}
+	return img.Width, img.Height, nil
+}
+
+var (
+	reg = regexp.MustCompile(`data:image/([^;]+);base64,`)
+)
+
+func GetImageSizeFromBase64(encoded string) (width int, height int, err error) {
+	encoded = strings.TrimPrefix(encoded, "data:image/png;base64,")
+	base64 := strings.NewReader(reg.ReplaceAllString(encoded, ""))
+	img, _, err := image.DecodeConfig(base64)
+	if err != nil {
+		return
+	}
+	return img.Width, img.Height, nil
+}
+
+func GetImageSize(image string) (width int, height int, err error) {
+	if strings.HasPrefix(image, "data:image/") {
+		return GetImageSizeFromBase64(image)
+	}
+	return GetImageSizeFromUrl(image)
+}

common/image/image_test.go

@@ -0,0 +1,154 @@
+package image_test
+
+import (
+	"encoding/base64"
+	"image"
+	_ "image/gif"
+	_ "image/jpeg"
+	_ "image/png"
+	"io"
+	"net/http"
+	"strconv"
+	"strings"
+	"testing"
+
+	img "one-api/common/image"
+
+	"github.com/stretchr/testify/assert"
+	_ "golang.org/x/image/webp"
+)
+
+type CountingReader struct {
+	reader    io.Reader
+	BytesRead int
+}
+
+func (r *CountingReader) Read(p []byte) (n int, err error) {
+	n, err = r.reader.Read(p)
+	r.BytesRead += n
+	return n, err
+}
+
+var (
+	cases = []struct {
+		url    string
+		format string
+		width  int
+		height int
+	}{
+		{"https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg", "jpeg", 2560, 1669},
+		{"https://upload.wikimedia.org/wikipedia/commons/9/97/Basshunter_live_performances.png", "png", 4500, 2592},
+		{"https://upload.wikimedia.org/wikipedia/commons/c/c6/TO_THE_ONE_SOMETHINGNESS.webp", "webp", 984, 985},
+		{"https://upload.wikimedia.org/wikipedia/commons/d/d0/01_Das_Sandberg-Modell.gif", "gif", 1917, 1533},
+		{"https://upload.wikimedia.org/wikipedia/commons/6/62/102Cervus.jpg", "jpeg", 270, 230},
+	}
+)
+
+func TestDecode(t *testing.T) {
+	// Bytes read: varies sometimes
+	// jpeg: 1063892
+	// png: 294462
+	// webp: 99529
+	// gif: 956153
+	// jpeg#01: 32805
+	for _, c := range cases {
+		t.Run("Decode:"+c.format, func(t *testing.T) {
+			resp, err := http.Get(c.url)
+			assert.NoError(t, err)
+			defer resp.Body.Close()
+			reader := &CountingReader{reader: resp.Body}
+			img, format, err := image.Decode(reader)
+			assert.NoError(t, err)
+			size := img.Bounds().Size()
+			assert.Equal(t, c.format, format)
+			assert.Equal(t, c.width, size.X)
+			assert.Equal(t, c.height, size.Y)
+			t.Logf("Bytes read: %d", reader.BytesRead)
+		})
+	}
+
+	// Bytes read:
+	// jpeg: 4096
+	// png: 4096
+	// webp: 4096
+	// gif: 4096
+	// jpeg#01: 4096
+	for _, c := range cases {
+		t.Run("DecodeConfig:"+c.format, func(t *testing.T) {
+			resp, err := http.Get(c.url)
+			assert.NoError(t, err)
+			defer resp.Body.Close()
+			reader := &CountingReader{reader: resp.Body}
+			config, format, err := image.DecodeConfig(reader)
+			assert.NoError(t, err)
+			assert.Equal(t, c.format, format)
+			assert.Equal(t, c.width, config.Width)
+			assert.Equal(t, c.height, config.Height)
+			t.Logf("Bytes read: %d", reader.BytesRead)
+		})
+	}
+}
+
+func TestBase64(t *testing.T) {
+	// Bytes read:
+	// jpeg: 1063892
+	// png: 294462
+	// webp: 99072
+	// gif: 953856
+	// jpeg#01: 32805
+	for _, c := range cases {
+		t.Run("Decode:"+c.format, func(t *testing.T) {
+			resp, err := http.Get(c.url)
+			assert.NoError(t, err)
+			defer resp.Body.Close()
+			data, err := io.ReadAll(resp.Body)
+			assert.NoError(t, err)
+			encoded := base64.StdEncoding.EncodeToString(data)
+			body := base64.NewDecoder(base64.StdEncoding, strings.NewReader(encoded))
+			reader := &CountingReader{reader: body}
+			img, format, err := image.Decode(reader)
+			assert.NoError(t, err)
+			size := img.Bounds().Size()
+			assert.Equal(t, c.format, format)
+			assert.Equal(t, c.width, size.X)
+			assert.Equal(t, c.height, size.Y)
+			t.Logf("Bytes read: %d", reader.BytesRead)
+		})
+	}
+
+	// Bytes read:
+	// jpeg: 1536
+	// png: 768
+	// webp: 768
+	// gif: 1536
+	// jpeg#01: 3840
+	for _, c := range cases {
+		t.Run("DecodeConfig:"+c.format, func(t *testing.T) {
+			resp, err := http.Get(c.url)
+			assert.NoError(t, err)
+			defer resp.Body.Close()
+			data, err := io.ReadAll(resp.Body)
+			assert.NoError(t, err)
+			encoded := base64.StdEncoding.EncodeToString(data)
+			body := base64.NewDecoder(base64.StdEncoding, strings.NewReader(encoded))
+			reader := &CountingReader{reader: body}
+			config, format, err := image.DecodeConfig(reader)
+			assert.NoError(t, err)
+			assert.Equal(t, c.format, format)
+			assert.Equal(t, c.width, config.Width)
+			assert.Equal(t, c.height, config.Height)
+			t.Logf("Bytes read: %d", reader.BytesRead)
+		})
+	}
+}
+
+func TestGetImageSize(t *testing.T) {
+	for i, c := range cases {
+		t.Run("Decode:"+strconv.Itoa(i), func(t *testing.T) {
+			width, height, err := img.GetImageSize(c.url)
+			assert.NoError(t, err)
+			assert.Equal(t, c.width, width)
+			assert.Equal(t, c.height, height)
+		})
+	}
+}

controller/channel-test.go

@@ -81,6 +81,9 @@ func testChannel(channel *model.Channel, request ChatRequest) (err error, openai
 		return fmt.Errorf("Error: %s\nResp body: %s", err, body), nil
 	}
 	if response.Usage.CompletionTokens == 0 {
+		if response.Error.Message == "" {
+			response.Error.Message = "补全 tokens 非预期返回 0"
+		}
 		return errors.New(fmt.Sprintf("type %s, code %v, message %s", response.Error.Type, response.Error.Code, response.Error.Message)), &response.Error
 	}
 	return nil, nil
@@ -142,20 +145,32 @@ func TestChannel(c *gin.Context) {
 var testAllChannelsLock sync.Mutex
 var testAllChannelsRunning bool = false
 
-// disable & notify
+func notifyRootUser(subject string, content string) {
-func disableChannel(channelId int, channelName string, reason string) {
 	if common.RootUserEmail == "" {
 		common.RootUserEmail = model.GetRootUserEmail()
 	}
-	model.UpdateChannelStatusById(channelId, common.ChannelStatusAutoDisabled)
-	subject := fmt.Sprintf("通道「%s」（#%d）已被禁用", channelName, channelId)
-	content := fmt.Sprintf("通道「%s」（#%d）已被禁用，原因：%s", channelName, channelId, reason)
 	err := common.SendEmail(subject, common.RootUserEmail, content)
 	if err != nil {
 		common.SysError(fmt.Sprintf("failed to send email: %s", err.Error()))
 	}
 }
 
+// disable & notify
+func disableChannel(channelId int, channelName string, reason string) {
+	model.UpdateChannelStatusById(channelId, common.ChannelStatusAutoDisabled)
+	subject := fmt.Sprintf("通道「%s」（#%d）已被禁用", channelName, channelId)
+	content := fmt.Sprintf("通道「%s」（#%d）已被禁用，原因：%s", channelName, channelId, reason)
+	notifyRootUser(subject, content)
+}
+
+// enable & notify
+func enableChannel(channelId int, channelName string) {
+	model.UpdateChannelStatusById(channelId, common.ChannelStatusEnabled)
+	subject := fmt.Sprintf("通道「%s」（#%d）已被启用", channelName, channelId)
+	content := fmt.Sprintf("通道「%s」（#%d）已被启用", channelName, channelId)
+	notifyRootUser(subject, content)
+}
+
 func testAllChannels(notify bool) error {
 	if common.RootUserEmail == "" {
 		common.RootUserEmail = model.GetRootUserEmail()
@@ -178,20 +193,21 @@ func testAllChannels(notify bool) error {
 	}
 	go func() {
 		for _, channel := range channels {
-			if channel.Status != common.ChannelStatusEnabled {
+			isChannelEnabled := channel.Status == common.ChannelStatusEnabled
-				continue
-			}
 			tik := time.Now()
 			err, openaiErr := testChannel(channel, *testRequest)
 			tok := time.Now()
 			milliseconds := tok.Sub(tik).Milliseconds()
-			if milliseconds > disableThreshold {
+			if isChannelEnabled && milliseconds > disableThreshold {
 				err = errors.New(fmt.Sprintf("响应时间 %.2fs 超过阈值 %.2fs", float64(milliseconds)/1000.0, float64(disableThreshold)/1000.0))
 				disableChannel(channel.Id, channel.Name, err.Error())
 			}
-			if shouldDisableChannel(openaiErr, -1) {
+			if isChannelEnabled && shouldDisableChannel(openaiErr, -1) {
 				disableChannel(channel.Id, channel.Name, err.Error())
 			}
+			if !isChannelEnabled && shouldEnableChannel(err, openaiErr) {
+				enableChannel(channel.Id, channel.Name)
+			}
 			channel.UpdateResponseTime(milliseconds)
 			time.Sleep(common.RequestInterval)
 		}

controller/relay-audio.go

@@ -1,6 +1,7 @@
 package controller
 
 import (
+	"bufio"
 	"bytes"
 	"context"
 	"encoding/json"
@@ -98,16 +99,17 @@ func relayAudioHelper(c *gin.Context, relayMode int) *OpenAIErrorWithStatusCode
 	fullRequestURL := getFullRequestURL(baseURL, requestURL, channelType)
 	if relayMode == RelayModeAudioTranscription && channelType == common.ChannelTypeAzure {
 		// https://learn.microsoft.com/en-us/azure/ai-services/openai/whisper-quickstart?tabs=command-line#rest-api
-		query := c.Request.URL.Query()
+		apiVersion := GetAPIVersion(c)
-		apiVersion := query.Get("api-version")
-		if apiVersion == "" {
-			apiVersion = c.GetString("api_version")
-		}
-		baseURL = c.GetString("base_url")
 		fullRequestURL = fmt.Sprintf("%s/openai/deployments/%s/audio/transcriptions?api-version=%s", baseURL, audioModel, apiVersion)
 	}
 
-	requestBody := c.Request.Body
+	requestBody := &bytes.Buffer{}
+	_, err = io.Copy(requestBody, c.Request.Body)
+	if err != nil {
+		return errorWrapper(err, "new_request_body_failed", http.StatusInternalServerError)
+	}
+	c.Request.Body = io.NopCloser(bytes.NewBuffer(requestBody.Bytes()))
+	responseFormat := c.DefaultPostForm("response_format", "json")
 
 	req, err := http.NewRequest(c.Request.Method, fullRequestURL, requestBody)
 	if err != nil {
@@ -149,12 +151,33 @@ func relayAudioHelper(c *gin.Context, relayMode int) *OpenAIErrorWithStatusCode
 		if err != nil {
 			return errorWrapper(err, "close_response_body_failed", http.StatusInternalServerError)
 		}
-		var whisperResponse WhisperResponse
+
-		err = json.Unmarshal(responseBody, &whisperResponse)
+		var openAIErr TextResponse
-		if err != nil {
+		if err = json.Unmarshal(responseBody, &openAIErr); err == nil {
-			return errorWrapper(err, "unmarshal_response_body_failed", http.StatusInternalServerError)
+			if openAIErr.Error.Message != "" {
+				return errorWrapper(fmt.Errorf("type %s, code %v, message %s", openAIErr.Error.Type, openAIErr.Error.Code, openAIErr.Error.Message), "request_error", http.StatusInternalServerError)
+			}
 		}
-		quota = countTokenText(whisperResponse.Text, audioModel)
+
+		var text string
+		switch responseFormat {
+		case "json":
+			text, err = getTextFromJSON(responseBody)
+		case "text":
+			text, err = getTextFromText(responseBody)
+		case "srt":
+			text, err = getTextFromSRT(responseBody)
+		case "verbose_json":
+			text, err = getTextFromVerboseJSON(responseBody)
+		case "vtt":
+			text, err = getTextFromVTT(responseBody)
+		default:
+			return errorWrapper(errors.New("unexpected_response_format"), "unexpected_response_format", http.StatusInternalServerError)
+		}
+		if err != nil {
+			return errorWrapper(err, "get_text_from_body_err", http.StatusInternalServerError)
+		}
+		quota = countTokenText(text, audioModel)
 		resp.Body = io.NopCloser(bytes.NewBuffer(responseBody))
 	}
 	if resp.StatusCode != http.StatusOK {
@@ -192,3 +215,48 @@ func relayAudioHelper(c *gin.Context, relayMode int) *OpenAIErrorWithStatusCode
 	}
 	return nil
 }
+
+func getTextFromVTT(body []byte) (string, error) {
+	return getTextFromSRT(body)
+}
+
+func getTextFromVerboseJSON(body []byte) (string, error) {
+	var whisperResponse WhisperVerboseJSONResponse
+	if err := json.Unmarshal(body, &whisperResponse); err != nil {
+		return "", fmt.Errorf("unmarshal_response_body_failed err :%w", err)
+	}
+	return whisperResponse.Text, nil
+}
+
+func getTextFromSRT(body []byte) (string, error) {
+	scanner := bufio.NewScanner(strings.NewReader(string(body)))
+	var builder strings.Builder
+	var textLine bool
+	for scanner.Scan() {
+		line := scanner.Text()
+		if textLine {
+			builder.WriteString(line)
+			textLine = false
+			continue
+		} else if strings.Contains(line, "-->") {
+			textLine = true
+			continue
+		}
+	}
+	if err := scanner.Err(); err != nil {
+		return "", err
+	}
+	return builder.String(), nil
+}
+
+func getTextFromText(body []byte) (string, error) {
+	return strings.TrimSuffix(string(body), "\n"), nil
+}
+
+func getTextFromJSON(body []byte) (string, error) {
+	var whisperResponse WhisperJSONResponse
+	if err := json.Unmarshal(body, &whisperResponse); err != nil {
+		return "", fmt.Errorf("unmarshal_response_body_failed err :%w", err)
+	}
+	return whisperResponse.Text, nil
+}

controller/relay-image.go

@@ -10,6 +10,7 @@ import (
 	"net/http"
 	"one-api/common"
 	"one-api/model"
+	"strings"
 
 	"github.com/gin-gonic/gin"
 )
@@ -101,8 +102,15 @@ func relayImageHelper(c *gin.Context, relayMode int) *OpenAIErrorWithStatusCode
 		baseURL = c.GetString("base_url")
 	}
 	fullRequestURL := getFullRequestURL(baseURL, requestURL, channelType)
+	if channelType == common.ChannelTypeAzure && relayMode == RelayModeImagesGenerations {
+		// https://learn.microsoft.com/en-us/azure/ai-services/openai/dall-e-quickstart?tabs=dalle3%2Ccommand-line&pivots=rest-api
+		apiVersion := GetAPIVersion(c)
+		// https://{resource_name}.openai.azure.com/openai/deployments/dall-e-3/images/generations?api-version=2023-06-01-preview
+		fullRequestURL = fmt.Sprintf("%s/openai/deployments/%s/images/generations?api-version=%s", baseURL, imageModel, apiVersion)
+	}
+
 	var requestBody io.Reader
-	if isModelMapped {
+	if isModelMapped || channelType == common.ChannelTypeAzure { // make Azure channel request body
 		jsonStr, err := json.Marshal(imageRequest)
 		if err != nil {
 			return errorWrapper(err, "marshal_text_request_failed", http.StatusInternalServerError)
@@ -127,7 +135,13 @@ func relayImageHelper(c *gin.Context, relayMode int) *OpenAIErrorWithStatusCode
 	if err != nil {
 		return errorWrapper(err, "new_request_failed", http.StatusInternalServerError)
 	}
-	req.Header.Set("Authorization", c.Request.Header.Get("Authorization"))
+	token := c.Request.Header.Get("Authorization")
+	if channelType == common.ChannelTypeAzure { // Azure authentication
+		token = strings.TrimPrefix(token, "Bearer ")
+		req.Header.Set("api-key", token)
+	} else {
+		req.Header.Set("Authorization", token)
+	}
 
 	req.Header.Set("Content-Type", c.Request.Header.Get("Content-Type"))
 	req.Header.Set("Accept", c.Request.Header.Get("Accept"))

controller/relay-text.go

@@ -129,11 +129,7 @@ func relayTextHelper(c *gin.Context, relayMode int) *OpenAIErrorWithStatusCode {
 	case APITypeOpenAI:
 		if channelType == common.ChannelTypeAzure {
 			// https://learn.microsoft.com/en-us/azure/cognitive-services/openai/chatgpt-quickstart?pivots=rest-api&tabs=command-line#rest-api
-			query := c.Request.URL.Query()
+			apiVersion := GetAPIVersion(c)
-			apiVersion := query.Get("api-version")
-			if apiVersion == "" {
-				apiVersion = c.GetString("api_version")
-			}
 			requestURL := strings.Split(requestURL, "?")[0]
 			requestURL = fmt.Sprintf("%s?api-version=%s", requestURL, apiVersion)
 			baseURL = c.GetString("base_url")
@@ -364,6 +360,9 @@ func relayTextHelper(c *gin.Context, relayMode int) *OpenAIErrorWithStatusCode {
 			if textRequest.Stream {
 				req.Header.Set("X-DashScope-SSE", "enable")
 			}
+			if c.GetString("plugin") != "" {
+				req.Header.Set("X-DashScope-Plugin", c.GetString("plugin"))
+			}
 		case APITypeTencent:
 			req.Header.Set("Authorization", apiKey)
 		case APITypePaLM:
@@ -411,6 +410,10 @@ func relayTextHelper(c *gin.Context, relayMode int) *OpenAIErrorWithStatusCode {
 	defer func(ctx context.Context) {
 		// c.Writer.Flush()
 		go func() {
+			if promptTokens != textResponse.PromptTokens {
+				common.SysError(fmt.Sprintf("prompt tokens not match, expected %d, actual %d", promptTokens, textResponse.PromptTokens))
+			}
+
 			quota := 0
 			completionRatio := common.GetCompletionRatio(textRequest.Model)
 			promptTokens = textResponse.Usage.PromptTokens

controller/relay-utils.go

@@ -3,10 +3,13 @@ package controller
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
+	"math"
 	"net/http"
 	"one-api/common"
+	"one-api/common/image"
 	"one-api/model"
 	"strconv"
 	"strings"
@@ -87,7 +90,33 @@ func countTokenMessages(messages []Message, model string) int {
 	tokenNum := 0
 	for _, message := range messages {
 		tokenNum += tokensPerMessage
-		tokenNum += getTokenNum(tokenEncoder, message.StringContent())
+		switch v := message.Content.(type) {
+		case string:
+			tokenNum += getTokenNum(tokenEncoder, v)
+		case []any:
+			for _, it := range v {
+				m := it.(map[string]any)
+				switch m["type"] {
+				case "text":
+					tokenNum += getTokenNum(tokenEncoder, m["text"].(string))
+				case "image_url":
+					imageUrl, ok := m["image_url"].(map[string]any)
+					if ok {
+						url := imageUrl["url"].(string)
+						detail := ""
+						if imageUrl["detail"] != nil {
+							detail = imageUrl["detail"].(string)
+						}
+						imageTokens, err := countImageTokens(url, detail)
+						if err != nil {
+							common.SysError("error counting image tokens: " + err.Error())
+						} else {
+							tokenNum += imageTokens
+						}
+					}
+				}
+			}
+		}
 		tokenNum += getTokenNum(tokenEncoder, message.Role)
 		if message.Name != nil {
 			tokenNum += tokensPerName
@@ -98,13 +127,81 @@ func countTokenMessages(messages []Message, model string) int {
 	return tokenNum
 }
 
+const (
+	lowDetailCost         = 85
+	highDetailCostPerTile = 170
+	additionalCost        = 85
+)
+
+// https://platform.openai.com/docs/guides/vision/calculating-costs
+// https://github.com/openai/openai-cookbook/blob/05e3f9be4c7a2ae7ecf029a7c32065b024730ebe/examples/How_to_count_tokens_with_tiktoken.ipynb
+func countImageTokens(url string, detail string) (_ int, err error) {
+	var fetchSize = true
+	var width, height int
+	// Reference: https://platform.openai.com/docs/guides/vision/low-or-high-fidelity-image-understanding
+	// detail == "auto" is undocumented on how it works, it just said the model will use the auto setting which will look at the image input size and decide if it should use the low or high setting.
+	// According to the official guide, "low" disable the high-res model,
+	// and only receive low-res 512px x 512px version of the image, indicating
+	// that image is treated as low-res when size is smaller than 512px x 512px,
+	// then we can assume that image size larger than 512px x 512px is treated
+	// as high-res. Then we have the following logic:
+	// if detail == "" || detail == "auto" {
+	// 	width, height, err = image.GetImageSize(url)
+	// 	if err != nil {
+	// 		return 0, err
+	// 	}
+	// 	fetchSize = false
+	// 	// not sure if this is correct
+	// 	if width > 512 || height > 512 {
+	// 		detail = "high"
+	// 	} else {
+	// 		detail = "low"
+	// 	}
+	// }
+
+	// However, in my test, it seems to be always the same as "high".
+	// The following image, which is 125x50, is still treated as high-res, taken
+	// 255 tokens in the response of non-stream chat completion api.
+	// https://upload.wikimedia.org/wikipedia/commons/1/10/18_Infantry_Division_Messina.jpg
+	if detail == "" || detail == "auto" {
+		// assume by test, not sure if this is correct
+		detail = "high"
+	}
+	switch detail {
+	case "low":
+		return lowDetailCost, nil
+	case "high":
+		if fetchSize {
+			width, height, err = image.GetImageSize(url)
+			if err != nil {
+				return 0, err
+			}
+		}
+		if width > 2048 || height > 2048 { // max(width, height) > 2048
+			ratio := float64(2048) / math.Max(float64(width), float64(height))
+			width = int(float64(width) * ratio)
+			height = int(float64(height) * ratio)
+		}
+		if width > 768 && height > 768 { // min(width, height) > 768
+			ratio := float64(768) / math.Min(float64(width), float64(height))
+			width = int(float64(width) * ratio)
+			height = int(float64(height) * ratio)
+		}
+		numSquares := int(math.Ceil(float64(width)/512) * math.Ceil(float64(height)/512))
+		result := numSquares*highDetailCostPerTile + additionalCost
+		return result, nil
+	default:
+		return 0, errors.New("invalid detail option")
+	}
+}
+
 func countTokenInput(input any, model string) int {
-	switch input.(type) {
+	switch v := input.(type) {
 	case string:
-		return countTokenText(input.(string), model)
+		return countTokenText(v, model)
 	case []string:
 		text := ""
-		for _, s := range input.([]string) {
+		for _, s := range v {
 			text += s
 		}
 		return countTokenText(text, model)
@@ -145,6 +242,19 @@ func shouldDisableChannel(err *OpenAIError, statusCode int) bool {
 	return false
 }
 
+func shouldEnableChannel(err error, openAIErr *OpenAIError) bool {
+	if !common.AutomaticEnableChannelEnabled {
+		return false
+	}
+	if err != nil {
+		return false
+	}
+	if openAIErr != nil {
+		return false
+	}
+	return true
+}
+
 func setEventStreamHeaders(c *gin.Context) {
 	c.Writer.Header().Set("Content-Type", "text/event-stream")
 	c.Writer.Header().Set("Cache-Control", "no-cache")
@@ -191,7 +301,6 @@ func getFullRequestURL(baseURL string, requestURL string, channelType int) strin
 			fullRequestURL = fmt.Sprintf("%s%s", baseURL, strings.TrimPrefix(requestURL, "/openai/deployments"))
 		}
 	}
-
 	return fullRequestURL
 }
 
@@ -216,3 +325,12 @@ func postConsumeQuota(ctx context.Context, tokenId int, quotaDelta int, totalQuo
 		common.LogError(ctx, fmt.Sprintf("totalQuota consumed is %d, something is wrong", totalQuota))
 	}
 }
+
+func GetAPIVersion(c *gin.Context) string {
+	query := c.Request.URL.Query()
+	apiVersion := query.Get("api-version")
+	if apiVersion == "" {
+		apiVersion = c.GetString("api_version")
+	}
+	return apiVersion
+}

controller/relay.go

@@ -133,18 +133,39 @@ type TextRequest struct {
 type ImageRequest struct {
 	Model          string `json:"model"`
 	Prompt         string `json:"prompt" binding:"required"`
-	N              int    `json:"n"`
+	N              int    `json:"n,omitempty"`
-	Size           string `json:"size"`
+	Size           string `json:"size,omitempty"`
-	Quality        string `json:"quality"`
+	Quality        string `json:"quality,omitempty"`
-	ResponseFormat string `json:"response_format"`
+	ResponseFormat string `json:"response_format,omitempty"`
-	Style          string `json:"style"`
+	Style          string `json:"style,omitempty"`
-	User           string `json:"user"`
+	User           string `json:"user,omitempty"`
 }
 
-type WhisperResponse struct {
+type WhisperJSONResponse struct {
 	Text string `json:"text,omitempty"`
 }
 
+type WhisperVerboseJSONResponse struct {
+	Task     string    `json:"task,omitempty"`
+	Language string    `json:"language,omitempty"`
+	Duration float64   `json:"duration,omitempty"`
+	Text     string    `json:"text,omitempty"`
+	Segments []Segment `json:"segments,omitempty"`
+}
+
+type Segment struct {
+	Id               int     `json:"id"`
+	Seek             int     `json:"seek"`
+	Start            float64 `json:"start"`
+	End              float64 `json:"end"`
+	Text             string  `json:"text"`
+	Tokens           []int   `json:"tokens"`
+	Temperature      float64 `json:"temperature"`
+	AvgLogprob       float64 `json:"avg_logprob"`
+	CompressionRatio float64 `json:"compression_ratio"`
+	NoSpeechProb     float64 `json:"no_speech_prob"`
+}
+
 type TextToSpeechRequest struct {
 	Model          string  `json:"model" binding:"required"`
 	Input          string  `json:"input" binding:"required"`

go.mod

@@ -15,7 +15,9 @@ require (
 	github.com/google/uuid v1.3.0
 	github.com/gorilla/websocket v1.5.0
 	github.com/pkoukk/tiktoken-go v0.1.5
+	github.com/stretchr/testify v1.8.3
 	golang.org/x/crypto v0.14.0
+	golang.org/x/image v0.14.0
 	gorm.io/driver/mysql v1.4.3
 	gorm.io/driver/postgres v1.5.2
 	gorm.io/driver/sqlite v1.4.3
@@ -26,6 +28,7 @@ require (
 	github.com/bytedance/sonic v1.9.1 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/dlclark/regexp2 v1.10.0 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
@@ -50,12 +53,13 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/pelletier/go-toml/v2 v2.0.8 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	github.com/ugorji/go/codec v1.2.11 // indirect
 	golang.org/x/arch v0.3.0 // indirect
 	golang.org/x/net v0.17.0 // indirect
 	golang.org/x/sys v0.13.0 // indirect
-	golang.org/x/text v0.13.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
 	google.golang.org/protobuf v1.30.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -152,6 +152,8 @@ golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
 golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
 golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
+golang.org/x/image v0.14.0 h1:tNgSxAFe3jC4uYqvZdTr84SZoM1KfwdC9SKIFrLjFn4=
+golang.org/x/image v0.14.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
 golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
@@ -168,8 +170,8 @@ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9sn
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
-golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

i18n/en.json

@@ -119,6 +119,7 @@
   " 年 ": " y ",
   "未测试": "Not tested",
   "通道 ${name} 测试成功，耗时 ${time.toFixed(2)} 秒。": "Channel ${name} test succeeded, time consumed ${time.toFixed(2)} s.",
+  "已成功开始测试所有通道，请刷新页面查看结果。": "All channels have been successfully tested, please refresh the page to view the results.",
   "已成功开始测试所有已启用通道，请刷新页面查看结果。": "All enabled channels have been successfully tested, please refresh the page to view the results.",
   "通道 ${name} 余额更新成功！": "Channel ${name} balance updated successfully!",
   "已更新完毕所有已启用通道余额！": "The balance of all enabled channels has been updated!",
@@ -139,6 +140,7 @@
   "启用": "Enable",
   "编辑": "Edit",
   "添加新的渠道": "Add a new channel",
+  "测试所有通道": "Test all channels",
   "测试所有已启用通道": "Test all enabled channels",
   "更新所有已启用通道余额": "Update the balance of all enabled channels",
   "刷新": "Refresh",

middleware/distributor.go

@@ -89,6 +89,8 @@ func Distribute() func(c *gin.Context) {
 			c.Set("api_version", channel.Other)
 		case common.ChannelTypeAIProxyLibrary:
 			c.Set("library_id", channel.Other)
+		case common.ChannelTypeAli:
+			c.Set("plugin", channel.Other)
 		}
 		c.Next()
 	}

middleware/recover.go

@@ -0,0 +1,26 @@
+package middleware
+
+import (
+	"fmt"
+	"github.com/gin-gonic/gin"
+	"net/http"
+	"one-api/common"
+)
+
+func RelayPanicRecover() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		defer func() {
+			if err := recover(); err != nil {
+				common.SysError(fmt.Sprintf("panic detected: %v", err))
+				c.JSON(http.StatusInternalServerError, gin.H{
+					"error": gin.H{
+						"message": fmt.Sprintf("Panic detected, error: %v. Please submit a issue here: https://github.com/songquanpeng/one-api", err),
+						"type":    "one_api_panic",
+					},
+				})
+				c.Abort()
+			}
+		}()
+		c.Next()
+	}
+}

model/option.go

@@ -34,6 +34,7 @@ func InitOptionMap() {
 	common.OptionMap["TurnstileCheckEnabled"] = strconv.FormatBool(common.TurnstileCheckEnabled)
 	common.OptionMap["RegisterEnabled"] = strconv.FormatBool(common.RegisterEnabled)
 	common.OptionMap["AutomaticDisableChannelEnabled"] = strconv.FormatBool(common.AutomaticDisableChannelEnabled)
+	common.OptionMap["AutomaticEnableChannelEnabled"] = strconv.FormatBool(common.AutomaticEnableChannelEnabled)
 	common.OptionMap["ApproximateTokenEnabled"] = strconv.FormatBool(common.ApproximateTokenEnabled)
 	common.OptionMap["LogConsumeEnabled"] = strconv.FormatBool(common.LogConsumeEnabled)
 	common.OptionMap["DisplayInCurrencyEnabled"] = strconv.FormatBool(common.DisplayInCurrencyEnabled)
@@ -147,6 +148,8 @@ func updateOptionMap(key string, value string) (err error) {
 			common.EmailDomainRestrictionEnabled = boolValue
 		case "AutomaticDisableChannelEnabled":
 			common.AutomaticDisableChannelEnabled = boolValue
+		case "AutomaticEnableChannelEnabled":
+			common.AutomaticEnableChannelEnabled = boolValue
 		case "ApproximateTokenEnabled":
 			common.ApproximateTokenEnabled = boolValue
 		case "LogConsumeEnabled":

router/relay-router.go

@@ -17,7 +17,7 @@ func SetRelayRouter(router *gin.Engine) {
 		modelsRouter.GET("/:model", controller.RetrieveModel)
 	}
 	relayV1Router := router.Group("/v1")
-	relayV1Router.Use(middleware.TokenAuth(), middleware.Distribute())
+	relayV1Router.Use(middleware.RelayPanicRecover(), middleware.TokenAuth(), middleware.Distribute())
 	{
 		relayV1Router.POST("/completions", controller.Relay)
 		relayV1Router.POST("/chat/completions", controller.Relay)
@@ -35,12 +35,38 @@ func SetRelayRouter(router *gin.Engine) {
 		relayV1Router.DELETE("/files/:id", controller.RelayNotImplemented)
 		relayV1Router.GET("/files/:id", controller.RelayNotImplemented)
 		relayV1Router.GET("/files/:id/content", controller.RelayNotImplemented)
-		relayV1Router.POST("/fine-tunes", controller.RelayNotImplemented)
+		relayV1Router.POST("/fine_tuning/jobs", controller.RelayNotImplemented)
-		relayV1Router.GET("/fine-tunes", controller.RelayNotImplemented)
+		relayV1Router.GET("/fine_tuning/jobs", controller.RelayNotImplemented)
-		relayV1Router.GET("/fine-tunes/:id", controller.RelayNotImplemented)
+		relayV1Router.GET("/fine_tuning/jobs/:id", controller.RelayNotImplemented)
-		relayV1Router.POST("/fine-tunes/:id/cancel", controller.RelayNotImplemented)
+		relayV1Router.POST("/fine_tuning/jobs/:id/cancel", controller.RelayNotImplemented)
-		relayV1Router.GET("/fine-tunes/:id/events", controller.RelayNotImplemented)
+		relayV1Router.GET("/fine_tuning/jobs/:id/events", controller.RelayNotImplemented)
 		relayV1Router.DELETE("/models/:model", controller.RelayNotImplemented)
 		relayV1Router.POST("/moderations", controller.Relay)
+		relayV1Router.POST("/assistants", controller.RelayNotImplemented)
+		relayV1Router.GET("/assistants/:id", controller.RelayNotImplemented)
+		relayV1Router.POST("/assistants/:id", controller.RelayNotImplemented)
+		relayV1Router.DELETE("/assistants/:id", controller.RelayNotImplemented)
+		relayV1Router.GET("/assistants", controller.RelayNotImplemented)
+		relayV1Router.POST("/assistants/:id/files", controller.RelayNotImplemented)
+		relayV1Router.GET("/assistants/:id/files/:fileId", controller.RelayNotImplemented)
+		relayV1Router.DELETE("/assistants/:id/files/:fileId", controller.RelayNotImplemented)
+		relayV1Router.GET("/assistants/:id/files", controller.RelayNotImplemented)
+		relayV1Router.POST("/threads", controller.RelayNotImplemented)
+		relayV1Router.GET("/threads/:id", controller.RelayNotImplemented)
+		relayV1Router.POST("/threads/:id", controller.RelayNotImplemented)
+		relayV1Router.DELETE("/threads/:id", controller.RelayNotImplemented)
+		relayV1Router.POST("/threads/:id/messages", controller.RelayNotImplemented)
+		relayV1Router.GET("/threads/:id/messages/:messageId", controller.RelayNotImplemented)
+		relayV1Router.POST("/threads/:id/messages/:messageId", controller.RelayNotImplemented)
+		relayV1Router.GET("/threads/:id/messages/:messageId/files/:filesId", controller.RelayNotImplemented)
+		relayV1Router.GET("/threads/:id/messages/:messageId/files", controller.RelayNotImplemented)
+		relayV1Router.POST("/threads/:id/runs", controller.RelayNotImplemented)
+		relayV1Router.GET("/threads/:id/runs/:runsId", controller.RelayNotImplemented)
+		relayV1Router.POST("/threads/:id/runs/:runsId", controller.RelayNotImplemented)
+		relayV1Router.GET("/threads/:id/runs", controller.RelayNotImplemented)
+		relayV1Router.POST("/threads/:id/runs/:runsId/submit_tool_outputs", controller.RelayNotImplemented)
+		relayV1Router.POST("/threads/:id/runs/:runsId/cancel", controller.RelayNotImplemented)
+		relayV1Router.GET("/threads/:id/runs/:runsId/steps/:stepId", controller.RelayNotImplemented)
+		relayV1Router.GET("/threads/:id/runs/:runsId/steps", controller.RelayNotImplemented)
 	}
 }

web/src/components/ChannelsTable.js

@@ -234,7 +234,7 @@ const ChannelsTable = () => {
     const res = await API.get(`/api/channel/test`);
     const { success, message } = res.data;
     if (success) {
-      showInfo('已成功开始测试所有已启用通道，请刷新页面查看结果。');
+      showInfo('已成功开始测试所有通道，请刷新页面查看结果。');
     } else {
       showError(message);
     }

web/src/components/OperationSetting.js

@@ -16,6 +16,7 @@ const OperationSetting = () => {
     ChatLink: '',
     QuotaPerUnit: 0,
     AutomaticDisableChannelEnabled: '',
+    AutomaticEnableChannelEnabled: '',
     ChannelDisableThreshold: 0,
     LogConsumeEnabled: '',
     DisplayInCurrencyEnabled: '',
@@ -269,6 +270,12 @@ const OperationSetting = () => {
               name='AutomaticDisableChannelEnabled'
               onChange={handleInputChange}
             />
+            <Form.Checkbox
+                checked={inputs.AutomaticEnableChannelEnabled === 'true'}
+                label='成功时自动启用通道'
+                name='AutomaticEnableChannelEnabled'
+                onChange={handleInputChange}
+            />
           </Form.Group>
           <Form.Button onClick={() => {
             submitConfig('monitor').then();

web/src/pages/Channel/EditChannel.js

@@ -343,6 +343,20 @@ const EditChannel = () => {
               </Form.Field>
             )
           }
+          {
+            inputs.type === 17 && (
+              <Form.Field>
+                <Form.Input
+                  label='插件参数'
+                  name='other'
+                  placeholder={'请输入插件参数，即 X-DashScope-Plugin 请求头的取值'}
+                  onChange={handleInputChange}
+                  value={inputs.other}
+                  autoComplete='new-password'
+                />
+              </Form.Field>
+            )
+          }
           <Form.Field>
             <Form.Dropdown
               label='模型'