feat: 启动时就自动检查下载nltk

This commit is contained in:
martsforever
2025-10-29 14:58:59 +08:00
parent b84ad4bcb8
commit 7d2c2a371f
3 changed files with 28 additions and 9 deletions
-4
View File
@@ -72,10 +72,6 @@ class MilvusService:
embed_model=self.embeddings,
)
# 预先加载NLTK语料库以避免多线程环境中的竞争条件
# 否则在多线程环境下,parser.get_nodes_from_documents 可能会出现报错信息:'WordListCorpusReader' object has no attribute '_LazyCorpusLoader__args'
load_nltk()
from llama_index.core.node_parser import SimpleNodeParser
# parser = HierarchicalNodeParser.from_defaults(chunk_sizes=[2048, 512, 128])
parser = SimpleNodeParser()