feat: load_nltk

app/utils/milvus_utils.py

@@ -7,6 +7,7 @@ from llama_index.vector_stores.milvus import MilvusVectorStore
 
 from app.config.env import env
 from app.utils.llm_utils import create_embeddings, create_llama_index_llm
+from app.utils.nltk_utils import load_nltk
 
 
 class MilvusService:
@@ -49,6 +50,11 @@ class MilvusService:
       vector_store=vector_store,
       embed_model=self.embeddings,
     )
+
+    # 预先加载NLTK语料库以避免多线程环境中的竞争条件
+    # 否则在多线程环境下，parser.get_nodes_from_documents 可能会出现报错信息：'WordListCorpusReader' object has no attribute '_LazyCorpusLoader__args'
+    load_nltk()
+
     from llama_index.core.node_parser import SimpleNodeParser
     # parser = HierarchicalNodeParser.from_defaults(chunk_sizes=[2048, 512, 128])
     parser = SimpleNodeParser()

app/utils/nltk_utils.py

@@ -0,0 +1,14 @@
+# 预先加载NLTK语料库以避免多线程环境中的竞争条件
+import nltk
+
+load_flag = False
+
+
+def load_nltk():
+  global load_flag
+  if not load_flag:
+    nltk.download('punkt')
+    nltk.download('averaged_perceptron_tagger')
+    nltk.download('maxent_ne_chunker')
+    nltk.download('words')
+    load_flag = True