From f6ebb19ab39c1dacf6b9bff408ce7a4623ca9b49 Mon Sep 17 00:00:00 2001
From: dakai <oudakai@gmail.com>
Date: Fri, 7 Feb 2025 01:56:31 +0800
Subject: [PATCH] docs: add documentation for countTokens function

---
 app/utils.ts | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/app/utils.ts b/app/utils.ts
index c5107109b..2ee1453b6 100644
--- a/app/utils.ts
+++ b/app/utils.ts
@@ -40,6 +40,12 @@ export const readFileContent = async (file: UploadFile): Promise<string> => {
   }
 };
 
+/**
+ * Estimates the token count of a text file using character-based weights.
+ * Note: This is a rough estimation as standard tokenizers cannot be used due to environment constraints.
+ * @param file - The file to analyze
+ * @returns Estimated token count in thousands (K)
+ */
 export const countTokens = async (file: UploadFile) => {
   const text = await readFileContent(file);
   let totalTokens = 0;