LiangTeam/liang-ai
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

20250412 调整服务启动时embedding加载策略，改成通过调用接口(后续可能是个定时任务)的方式去插入向量数据而不是每次启动都插入

Browse Source
This commit is contained in:
liangjinglin 2025-04-12 17:50:58 +08:00
parent 2ae4d6e425
commit 8737f6bbf1
3 changed files with 80 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
src/main/java/com/ai/config/EmbeddingConfig.java Normal file
View File

@ -0,0 +1,42 @@
package com.ai.config;
import dev.langchain4j.community.model.dashscope.QwenEmbeddingModel;
import dev.langchain4j.data.segment.TextSegment;
import dev.langchain4j.store.embedding.EmbeddingStore;
import dev.langchain4j.store.embedding.milvus.MilvusEmbeddingStore;
import io.milvus.common.clientenum.ConsistencyLevelEnum;
import io.milvus.param.IndexType;
import io.milvus.param.MetricType;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@Configuration
public class EmbeddingConfig {
@Value("${langchain4j.api-key}")
private String apiKey;
@Bean
public QwenEmbeddingModel embeddingModel() {
return QwenEmbeddingModel.builder().apiKey(apiKey).build();
}
@Bean
public EmbeddingStore<TextSegment> embeddingStore() {
return MilvusEmbeddingStore.builder()
.host("localhost")
.port(19530)
.collectionName("langchain4j_collection")
.dimension(1536)
.indexType(IndexType.FLAT)
.metricType(MetricType.COSINE)
.consistencyLevel(ConsistencyLevelEnum.EVENTUALLY)
.autoFlushOnInsert(true)
.idFieldName("id")
.textFieldName("text")
.metadataFieldName("metadata")
.vectorFieldName("vector")
.build();
}
}

29
src/main/java/com/ai/config/SegmentConfig.java
View File

@ -18,6 +18,7 @@ import dev.langchain4j.store.embedding.milvus.MilvusEmbeddingStore;
import io.milvus.common.clientenum.ConsistencyLevelEnum;
import io.milvus.param.IndexType;
import io.milvus.param.MetricType;
import jakarta.annotation.Resource;
import lombok.RequiredArgsConstructor;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
@ -38,35 +39,19 @@ public class SegmentConfig {
@Value("${rag.path}")
private String ragPath;
@Resource
private QwenEmbeddingModel embeddingModel;
@Resource
private EmbeddingStore<TextSegment> embeddingStore;
@Bean
public SegmentAssist segmentAssist() {
ChatLanguageModel qwenModel = QwenChatModel.builder()
.apiKey(apiKey)
.modelName(model)
.build();
QwenEmbeddingModel embeddingModel = QwenEmbeddingModel.builder().apiKey(apiKey).build();
// InMemoryEmbeddingStore<TextSegment> embeddingStore = new InMemoryEmbeddingStore<>();
EmbeddingStore<TextSegment> embeddingStore = MilvusEmbeddingStore.builder()
.host("localhost")
.port(19530)
.collectionName("langchain4j_collection")
.dimension(1536)
.indexType(IndexType.FLAT)
.metricType(MetricType.COSINE)
.consistencyLevel(ConsistencyLevelEnum.EVENTUALLY)
.autoFlushOnInsert(true)
.idFieldName("id")
.textFieldName("text")
.metadataFieldName("metadata")
.vectorFieldName("vector")
.build();
List<Document> documents = FileSystemDocumentLoader.loadDocuments(ragPath);
for (Document document : documents) {
DocumentByLineSplitter splitter = new DocumentByLineSplitter(300,30);
List<TextSegment> segments = splitter.split(document);
List<Embedding> embeddings = embeddingModel.embedAll(segments).content();
embeddingStore.addAll(embeddings, segments);
}
ContentRetriever contentRetriever = EmbeddingStoreContentRetriever.builder()
.embeddingStore(embeddingStore)

31
src/main/java/com/ai/controller/MilvusController.java
View File

@ -1,10 +1,21 @@
package com.ai.controller;
import com.ai.service.SimilarService;
import dev.langchain4j.community.model.dashscope.QwenEmbeddingModel;
import dev.langchain4j.data.document.Document;
import dev.langchain4j.data.document.loader.FileSystemDocumentLoader;
import dev.langchain4j.data.document.splitter.DocumentByLineSplitter;
import dev.langchain4j.data.embedding.Embedding;
import dev.langchain4j.data.segment.TextSegment;
import dev.langchain4j.store.embedding.EmbeddingStore;
import jakarta.annotation.Resource;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import java.util.List;
@RestController
@RequestMapping("/milvus")
public class MilvusController {
@ -12,6 +23,15 @@ public class MilvusController {
@Autowired
private SimilarService similarService;
@Resource
private QwenEmbeddingModel embeddingModel;
@Resource
private EmbeddingStore<TextSegment> embeddingStore;
@Value("${rag.path}")
private String ragPath;
@RequestMapping("/create")
public String create() {
similarService.createCollection();
@ -23,4 +43,15 @@ public class MilvusController {
similarService.insertProducts();
return "vector create";
}
@RequestMapping("/import")
public void ImportShopData() {
List<Document> documents = FileSystemDocumentLoader.loadDocuments(ragPath);
for (Document document : documents) {
DocumentByLineSplitter splitter = new DocumentByLineSplitter(150,30);
List<TextSegment> segments = splitter.split(document);
List<Embedding> embeddings = embeddingModel.embedAll(segments).content();
embeddingStore.addAll(embeddings, segments);
}
}
}