From 185c16b06bf55b0a4bd2bff32af7d94f619b3b88 Mon Sep 17 00:00:00 2001
From: martsforever <martsforever@aliyun.com>
Date: Sun, 7 Sep 2025 20:16:30 +0800
Subject: [PATCH] feat: load_nltk

---
 app/utils/milvus_utils.py |  6 ++++++
 app/utils/nltk_utils.py   | 14 ++++++++++++++
 2 files changed, 20 insertions(+)
 create mode 100644 app/utils/nltk_utils.py

diff --git a/app/utils/milvus_utils.py b/app/utils/milvus_utils.py
index 10044d6..efca688 100644
--- a/app/utils/milvus_utils.py
+++ b/app/utils/milvus_utils.py
@@ -7,6 +7,7 @@ from llama_index.vector_stores.milvus import MilvusVectorStore
 
 from app.config.env import env
 from app.utils.llm_utils import create_embeddings, create_llama_index_llm
+from app.utils.nltk_utils import load_nltk
 
 
 class MilvusService:
@@ -49,6 +50,11 @@ class MilvusService:
       vector_store=vector_store,
       embed_model=self.embeddings,
     )
+
+    # 预先加载NLTK语料库以避免多线程环境中的竞争条件
+    # 否则在多线程环境下，parser.get_nodes_from_documents 可能会出现报错信息：'WordListCorpusReader' object has no attribute '_LazyCorpusLoader__args'
+    load_nltk()
+
     from llama_index.core.node_parser import SimpleNodeParser
     # parser = HierarchicalNodeParser.from_defaults(chunk_sizes=[2048, 512, 128])
     parser = SimpleNodeParser()
diff --git a/app/utils/nltk_utils.py b/app/utils/nltk_utils.py
new file mode 100644
index 0000000..12f4ffd
--- /dev/null
+++ b/app/utils/nltk_utils.py
@@ -0,0 +1,14 @@
+# 预先加载NLTK语料库以避免多线程环境中的竞争条件
+import nltk
+
+load_flag = False
+
+
+def load_nltk():
+  global load_flag
+  if not load_flag:
+    nltk.download('punkt')
+    nltk.download('averaged_perceptron_tagger')
+    nltk.download('maxent_ne_chunker')
+    nltk.download('words')
+    load_flag = True