Implement SQLite support check for Chroma's trigram tokenizer, enhancing compatibility with cloud environments. Update README for model command syntax and add pysqlite3-binary to requirements for improved SQLite functionality.

This commit is contained in:
Mimikko-zeus
2026-03-03 13:29:05 +08:00
parent 774ea9d5e4
commit 46ff239f4c
6 changed files with 256 additions and 36 deletions

47
main.py
View File

@@ -8,6 +8,53 @@ from pathlib import Path
project_root = Path(__file__).parent
sys.path.insert(0, str(project_root))
def _sqlite_supports_trigram(sqlite_module) -> bool:
conn = None
try:
conn = sqlite_module.connect(":memory:")
conn.execute("create virtual table t using fts5(content, tokenize='trigram')")
return True
except Exception:
return False
finally:
if conn is not None:
conn.close()
def _ensure_sqlite_for_chroma():
"""
Ensure sqlite runtime supports FTS5 trigram tokenizer for Chroma.
On some cloud images, system sqlite lacks trigram support.
"""
try:
import sqlite3
except Exception:
return
if _sqlite_supports_trigram(sqlite3):
return
try:
import pysqlite3
except Exception as exc:
print(
"[WARN] sqlite3 does not support trigram tokenizer and pysqlite3 is unavailable: "
f"{exc}"
)
print("[WARN] Chroma may fail and fallback to JSON storage.")
return
if _sqlite_supports_trigram(pysqlite3):
sys.modules["sqlite3"] = pysqlite3
print("[INFO] sqlite3 switched to pysqlite3 for Chroma trigram support.")
else:
print("[WARN] pysqlite3 is installed but still lacks trigram tokenizer support.")
print("[WARN] Chroma may fail and fallback to JSON storage.")
_ensure_sqlite_for_chroma()
from src.core.bot import MyClient, build_intents
from src.core.config import Config
from src.utils.logger import setup_logger