Implement SQLite support check for Chroma's trigram tokenizer, enhancing compatibility with cloud environments. Update README for model command syntax and add pysqlite3-binary to requirements for improved SQLite functionality.
This commit is contained in:
47
main.py
47
main.py
@@ -8,6 +8,53 @@ from pathlib import Path
|
||||
project_root = Path(__file__).parent
|
||||
sys.path.insert(0, str(project_root))
|
||||
|
||||
|
||||
def _sqlite_supports_trigram(sqlite_module) -> bool:
|
||||
conn = None
|
||||
try:
|
||||
conn = sqlite_module.connect(":memory:")
|
||||
conn.execute("create virtual table t using fts5(content, tokenize='trigram')")
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
finally:
|
||||
if conn is not None:
|
||||
conn.close()
|
||||
|
||||
|
||||
def _ensure_sqlite_for_chroma():
|
||||
"""
|
||||
Ensure sqlite runtime supports FTS5 trigram tokenizer for Chroma.
|
||||
On some cloud images, system sqlite lacks trigram support.
|
||||
"""
|
||||
try:
|
||||
import sqlite3
|
||||
except Exception:
|
||||
return
|
||||
|
||||
if _sqlite_supports_trigram(sqlite3):
|
||||
return
|
||||
|
||||
try:
|
||||
import pysqlite3
|
||||
except Exception as exc:
|
||||
print(
|
||||
"[WARN] sqlite3 does not support trigram tokenizer and pysqlite3 is unavailable: "
|
||||
f"{exc}"
|
||||
)
|
||||
print("[WARN] Chroma may fail and fallback to JSON storage.")
|
||||
return
|
||||
|
||||
if _sqlite_supports_trigram(pysqlite3):
|
||||
sys.modules["sqlite3"] = pysqlite3
|
||||
print("[INFO] sqlite3 switched to pysqlite3 for Chroma trigram support.")
|
||||
else:
|
||||
print("[WARN] pysqlite3 is installed but still lacks trigram tokenizer support.")
|
||||
print("[WARN] Chroma may fail and fallback to JSON storage.")
|
||||
|
||||
|
||||
_ensure_sqlite_for_chroma()
|
||||
|
||||
from src.core.bot import MyClient, build_intents
|
||||
from src.core.config import Config
|
||||
from src.utils.logger import setup_logger
|
||||
|
||||
Reference in New Issue
Block a user