feat: 启动时就自动检查下载nltk
This commit is contained in:
@@ -72,10 +72,6 @@ class MilvusService:
|
||||
embed_model=self.embeddings,
|
||||
)
|
||||
|
||||
# 预先加载NLTK语料库以避免多线程环境中的竞争条件
|
||||
# 否则在多线程环境下,parser.get_nodes_from_documents 可能会出现报错信息:'WordListCorpusReader' object has no attribute '_LazyCorpusLoader__args'
|
||||
load_nltk()
|
||||
|
||||
from llama_index.core.node_parser import SimpleNodeParser
|
||||
# parser = HierarchicalNodeParser.from_defaults(chunk_sizes=[2048, 512, 128])
|
||||
parser = SimpleNodeParser()
|
||||
|
||||
Reference in New Issue
Block a user