feat: Refactor vision message handling and increase commit short SHA length

- Increase the length of the commit short SHA from 8 to 7 characters
- Update relay-aiproxy.go to handle multiple content types in the last message
- Refactor and modify countVisonTokenMessages function in relay-utils.go to handle different content types and update token counts accordingly
- Update VisionMessage struct in relay.go to have Content as a slice of OpenaiVisionMessageContent

.github/workflows/ci.yml

@@ -47,7 +47,7 @@ jobs:
           key: ${{ runner.os }}-go-mod-${{ hashFiles('**/go.sum') }}
 
       - name: Add SHORT_SHA env property with commit short sha
-        run: echo "SHORT_SHA=`echo ${GITHUB_SHA} | cut -c1-8`" >> $GITHUB_ENV
+        run: echo "SHORT_SHA=`echo ${GITHUB_SHA} | cut -c1-7`" >> $GITHUB_ENV
 
       -
         name: Build and push latest

controller/relay-aiproxy.go

@@ -54,7 +54,15 @@ func requestOpenAI2AIProxyLibrary(request GeneralOpenAIRequest) *AIProxyLibraryR
 		if msgs, err := request.TextMessages(); err == nil {
 			query = msgs[len(msgs)-1].Content
 		} else if msgs, err := request.VisionMessages(); err == nil {
-			query = msgs[len(msgs)-1].Content.Text
+			lastMsg := msgs[len(msgs)-1]
+			if len(lastMsg.Content) != 0 {
+				for i := range lastMsg.Content {
+					if lastMsg.Content[i].Type == OpenaiVisionMessageContentTypeText {
+						query = lastMsg.Content[i].Text
+						break
+					}
+				}
+			}
 		} else {
 			log.Panicf("unknown message type: %T", msgs)
 		}

controller/relay-utils.go

@@ -168,19 +168,21 @@ func countVisonTokenMessages(messages []VisionMessage, model string) (int, error
 	tokenNum := 0
 	for _, message := range messages {
 		tokenNum += tokensPerMessage
-		switch message.Content.Type {
+		for _, cnt := range message.Content {
-		case OpenaiVisionMessageContentTypeText:
+			switch cnt.Type {
-			tokenNum += getTokenNum(tokenEncoder, message.Content.Text)
+			case OpenaiVisionMessageContentTypeText:
-		case OpenaiVisionMessageContentTypeImageUrl:
+				tokenNum += getTokenNum(tokenEncoder, cnt.Text)
-			imgblob, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(message.Content.ImageUrl.URL, "data:image/jpeg;base64,"))
+			case OpenaiVisionMessageContentTypeImageUrl:
-			if err != nil {
+				imgblob, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(cnt.ImageUrl.URL, "data:image/jpeg;base64,"))
-				return 0, errors.Wrap(err, "failed to decode base64 image")
+				if err != nil {
-			}
+					return 0, errors.Wrap(err, "failed to decode base64 image")
+				}
 
-			if imgtoken, err := CountVisionImageToken(imgblob, message.Content.ImageUrl.Detail); err != nil {
+				if imgtoken, err := CountVisionImageToken(imgblob, cnt.ImageUrl.Detail); err != nil {
-				return 0, errors.Wrap(err, "failed to count vision image token")
+					return 0, errors.Wrap(err, "failed to count vision image token")
-			} else {
+				} else {
-				tokenNum += imgtoken
+					tokenNum += imgtoken
+				}
 			}
 		}
 

controller/relay.go

@@ -19,9 +19,9 @@ type Message struct {
 }
 
 type VisionMessage struct {
-	Role    string                     `json:"role"`
+	Role    string                       `json:"role"`
-	Content OpenaiVisionMessageContent `json:"content"`
+	Content []OpenaiVisionMessageContent `json:"content"`
-	Name    *string                    `json:"name,omitempty"`
+	Name    *string                      `json:"name,omitempty"`
 }
 
 // OpenaiVisionMessageContentType vision message content type