From 8737f6bbf14abceda12b18e83af2080a7638351a Mon Sep 17 00:00:00 2001 From: liangjinglin Date: Sat, 12 Apr 2025 17:50:58 +0800 Subject: [PATCH] =?UTF-8?q?20250412=20=E8=B0=83=E6=95=B4=E6=9C=8D=E5=8A=A1?= =?UTF-8?q?=E5=90=AF=E5=8A=A8=E6=97=B6embedding=E5=8A=A0=E8=BD=BD=E7=AD=96?= =?UTF-8?q?=E7=95=A5=EF=BC=8C=E6=94=B9=E6=88=90=E9=80=9A=E8=BF=87=E8=B0=83?= =?UTF-8?q?=E7=94=A8=E6=8E=A5=E5=8F=A3(=E5=90=8E=E7=BB=AD=E5=8F=AF?= =?UTF-8?q?=E8=83=BD=E6=98=AF=E4=B8=AA=E5=AE=9A=E6=97=B6=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?)=E7=9A=84=E6=96=B9=E5=BC=8F=E5=8E=BB=E6=8F=92=E5=85=A5?= =?UTF-8?q?=E5=90=91=E9=87=8F=E6=95=B0=E6=8D=AE=E8=80=8C=E4=B8=8D=E6=98=AF?= =?UTF-8?q?=E6=AF=8F=E6=AC=A1=E5=90=AF=E5=8A=A8=E9=83=BD=E6=8F=92=E5=85=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/ai/config/EmbeddingConfig.java | 42 +++++++++++++++++++ .../java/com/ai/config/SegmentConfig.java | 29 ++++--------- .../com/ai/controller/MilvusController.java | 31 ++++++++++++++ 3 files changed, 80 insertions(+), 22 deletions(-) create mode 100644 src/main/java/com/ai/config/EmbeddingConfig.java diff --git a/src/main/java/com/ai/config/EmbeddingConfig.java b/src/main/java/com/ai/config/EmbeddingConfig.java new file mode 100644 index 0000000..8294294 --- /dev/null +++ b/src/main/java/com/ai/config/EmbeddingConfig.java @@ -0,0 +1,42 @@ +package com.ai.config; + +import dev.langchain4j.community.model.dashscope.QwenEmbeddingModel; +import dev.langchain4j.data.segment.TextSegment; +import dev.langchain4j.store.embedding.EmbeddingStore; +import dev.langchain4j.store.embedding.milvus.MilvusEmbeddingStore; +import io.milvus.common.clientenum.ConsistencyLevelEnum; +import io.milvus.param.IndexType; +import io.milvus.param.MetricType; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class EmbeddingConfig { + + @Value("${langchain4j.api-key}") + private String apiKey; + + @Bean + public QwenEmbeddingModel embeddingModel() { + return QwenEmbeddingModel.builder().apiKey(apiKey).build(); + } + + @Bean + public EmbeddingStore embeddingStore() { + return MilvusEmbeddingStore.builder() + .host("localhost") + .port(19530) + .collectionName("langchain4j_collection") + .dimension(1536) + .indexType(IndexType.FLAT) + .metricType(MetricType.COSINE) + .consistencyLevel(ConsistencyLevelEnum.EVENTUALLY) + .autoFlushOnInsert(true) + .idFieldName("id") + .textFieldName("text") + .metadataFieldName("metadata") + .vectorFieldName("vector") + .build(); + } +} diff --git a/src/main/java/com/ai/config/SegmentConfig.java b/src/main/java/com/ai/config/SegmentConfig.java index dd83f15..e3d8d08 100644 --- a/src/main/java/com/ai/config/SegmentConfig.java +++ b/src/main/java/com/ai/config/SegmentConfig.java @@ -18,6 +18,7 @@ import dev.langchain4j.store.embedding.milvus.MilvusEmbeddingStore; import io.milvus.common.clientenum.ConsistencyLevelEnum; import io.milvus.param.IndexType; import io.milvus.param.MetricType; +import jakarta.annotation.Resource; import lombok.RequiredArgsConstructor; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; @@ -38,35 +39,19 @@ public class SegmentConfig { @Value("${rag.path}") private String ragPath; + @Resource + private QwenEmbeddingModel embeddingModel; + + @Resource + private EmbeddingStore embeddingStore; + @Bean public SegmentAssist segmentAssist() { ChatLanguageModel qwenModel = QwenChatModel.builder() .apiKey(apiKey) .modelName(model) .build(); - QwenEmbeddingModel embeddingModel = QwenEmbeddingModel.builder().apiKey(apiKey).build(); // InMemoryEmbeddingStore embeddingStore = new InMemoryEmbeddingStore<>(); - EmbeddingStore embeddingStore = MilvusEmbeddingStore.builder() - .host("localhost") - .port(19530) - .collectionName("langchain4j_collection") - .dimension(1536) - .indexType(IndexType.FLAT) - .metricType(MetricType.COSINE) - .consistencyLevel(ConsistencyLevelEnum.EVENTUALLY) - .autoFlushOnInsert(true) - .idFieldName("id") - .textFieldName("text") - .metadataFieldName("metadata") - .vectorFieldName("vector") - .build(); - List documents = FileSystemDocumentLoader.loadDocuments(ragPath); - for (Document document : documents) { - DocumentByLineSplitter splitter = new DocumentByLineSplitter(300,30); - List segments = splitter.split(document); - List embeddings = embeddingModel.embedAll(segments).content(); - embeddingStore.addAll(embeddings, segments); - } ContentRetriever contentRetriever = EmbeddingStoreContentRetriever.builder() .embeddingStore(embeddingStore) diff --git a/src/main/java/com/ai/controller/MilvusController.java b/src/main/java/com/ai/controller/MilvusController.java index c30baea..5d44663 100644 --- a/src/main/java/com/ai/controller/MilvusController.java +++ b/src/main/java/com/ai/controller/MilvusController.java @@ -1,10 +1,21 @@ package com.ai.controller; import com.ai.service.SimilarService; +import dev.langchain4j.community.model.dashscope.QwenEmbeddingModel; +import dev.langchain4j.data.document.Document; +import dev.langchain4j.data.document.loader.FileSystemDocumentLoader; +import dev.langchain4j.data.document.splitter.DocumentByLineSplitter; +import dev.langchain4j.data.embedding.Embedding; +import dev.langchain4j.data.segment.TextSegment; +import dev.langchain4j.store.embedding.EmbeddingStore; +import jakarta.annotation.Resource; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; +import java.util.List; + @RestController @RequestMapping("/milvus") public class MilvusController { @@ -12,6 +23,15 @@ public class MilvusController { @Autowired private SimilarService similarService; + @Resource + private QwenEmbeddingModel embeddingModel; + + @Resource + private EmbeddingStore embeddingStore; + + @Value("${rag.path}") + private String ragPath; + @RequestMapping("/create") public String create() { similarService.createCollection(); @@ -23,4 +43,15 @@ public class MilvusController { similarService.insertProducts(); return "vector create"; } + + @RequestMapping("/import") + public void ImportShopData() { + List documents = FileSystemDocumentLoader.loadDocuments(ragPath); + for (Document document : documents) { + DocumentByLineSplitter splitter = new DocumentByLineSplitter(150,30); + List segments = splitter.split(document); + List embeddings = embeddingModel.embedAll(segments).content(); + embeddingStore.addAll(embeddings, segments); + } + } }