diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 055cfb30..4303cc94 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,7 +47,7 @@ jobs: key: ${{ runner.os }}-go-mod-${{ hashFiles('**/go.sum') }} - name: Add SHORT_SHA env property with commit short sha - run: echo "SHORT_SHA=`echo ${GITHUB_SHA} | cut -c1-8`" >> $GITHUB_ENV + run: echo "SHORT_SHA=`echo ${GITHUB_SHA} | cut -c1-7`" >> $GITHUB_ENV - name: Build and push latest diff --git a/controller/relay-aiproxy.go b/controller/relay-aiproxy.go index 8a664b0b..292cec15 100644 --- a/controller/relay-aiproxy.go +++ b/controller/relay-aiproxy.go @@ -54,7 +54,15 @@ func requestOpenAI2AIProxyLibrary(request GeneralOpenAIRequest) *AIProxyLibraryR if msgs, err := request.TextMessages(); err == nil { query = msgs[len(msgs)-1].Content } else if msgs, err := request.VisionMessages(); err == nil { - query = msgs[len(msgs)-1].Content.Text + lastMsg := msgs[len(msgs)-1] + if len(lastMsg.Content) != 0 { + for i := range lastMsg.Content { + if lastMsg.Content[i].Type == OpenaiVisionMessageContentTypeText { + query = lastMsg.Content[i].Text + break + } + } + } } else { log.Panicf("unknown message type: %T", msgs) } diff --git a/controller/relay-utils.go b/controller/relay-utils.go index 0538aad7..22278269 100644 --- a/controller/relay-utils.go +++ b/controller/relay-utils.go @@ -168,19 +168,21 @@ func countVisonTokenMessages(messages []VisionMessage, model string) (int, error tokenNum := 0 for _, message := range messages { tokenNum += tokensPerMessage - switch message.Content.Type { - case OpenaiVisionMessageContentTypeText: - tokenNum += getTokenNum(tokenEncoder, message.Content.Text) - case OpenaiVisionMessageContentTypeImageUrl: - imgblob, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(message.Content.ImageUrl.URL, "data:image/jpeg;base64,")) - if err != nil { - return 0, errors.Wrap(err, "failed to decode base64 image") - } + for _, cnt := range message.Content { + switch cnt.Type { + case OpenaiVisionMessageContentTypeText: + tokenNum += getTokenNum(tokenEncoder, cnt.Text) + case OpenaiVisionMessageContentTypeImageUrl: + imgblob, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(cnt.ImageUrl.URL, "data:image/jpeg;base64,")) + if err != nil { + return 0, errors.Wrap(err, "failed to decode base64 image") + } - if imgtoken, err := CountVisionImageToken(imgblob, message.Content.ImageUrl.Detail); err != nil { - return 0, errors.Wrap(err, "failed to count vision image token") - } else { - tokenNum += imgtoken + if imgtoken, err := CountVisionImageToken(imgblob, cnt.ImageUrl.Detail); err != nil { + return 0, errors.Wrap(err, "failed to count vision image token") + } else { + tokenNum += imgtoken + } } } diff --git a/controller/relay.go b/controller/relay.go index 26cbd416..a7f84a9d 100644 --- a/controller/relay.go +++ b/controller/relay.go @@ -19,9 +19,9 @@ type Message struct { } type VisionMessage struct { - Role string `json:"role"` - Content OpenaiVisionMessageContent `json:"content"` - Name *string `json:"name,omitempty"` + Role string `json:"role"` + Content []OpenaiVisionMessageContent `json:"content"` + Name *string `json:"name,omitempty"` } // OpenaiVisionMessageContentType vision message content type