|
|
|
package org.jeecg.modules.airag.app.service.impl;
|
|
|
|
|
|
|
|
import com.baomidou.dynamic.datasource.annotation.DS;
|
|
|
|
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
|
|
|
|
import com.fasterxml.jackson.core.JsonProcessingException;
|
|
|
|
import org.apache.commons.lang3.StringUtils;
|
|
|
|
import org.apache.poi.hwpf.usermodel.CharacterRun;
|
|
|
|
import org.apache.poi.hwpf.HWPFDocument;
|
|
|
|
import org.apache.poi.hwpf.usermodel.Paragraph;
|
|
|
|
import org.apache.poi.hwpf.usermodel.Range;
|
|
|
|
import dev.langchain4j.data.document.Document;
|
|
|
|
import dev.langchain4j.data.document.DocumentSplitter;
|
|
|
|
import dev.langchain4j.data.document.splitter.DocumentByParagraphSplitter;
|
|
...
|
...
|
@@ -17,27 +11,20 @@ import dev.langchain4j.model.output.Response; |
|
|
|
import org.apache.commons.io.FilenameUtils;
|
|
|
|
import org.apache.poi.xwpf.usermodel.*;
|
|
|
|
import org.jeecg.common.api.vo.Result;
|
|
|
|
import org.jeecg.modules.airag.airagchatsetting.entity.AiragChatsetting;
|
|
|
|
import org.jeecg.modules.airag.app.entity.Embeddings;
|
|
|
|
import org.jeecg.modules.airag.app.entity.QuestionEmbedding;
|
|
|
|
import org.jeecg.modules.airag.app.mapper.EmbeddingsMapper;
|
|
|
|
import org.jeecg.modules.airag.app.mapper.PgVectorMapper;
|
|
|
|
import org.jeecg.modules.airag.app.mapper.QuestionEmbeddingMapper;
|
|
|
|
import org.jeecg.modules.airag.app.service.IQuestionEmbeddingService;
|
|
|
|
import org.jeecg.modules.airag.app.service.RedisChatSetService;
|
|
|
|
import org.jeecg.modules.airag.app.utils.AiModelUtils;
|
|
|
|
import org.jeecg.modules.airag.common.handler.IAIChatHandler;
|
|
|
|
import org.postgresql.util.PGobject;
|
|
|
|
import org.slf4j.Logger;
|
|
|
|
import org.slf4j.LoggerFactory;
|
|
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
|
|
import org.springframework.beans.factory.annotation.Value;
|
|
|
|
import org.springframework.stereotype.Service;
|
|
|
|
import org.springframework.web.multipart.MultipartFile;
|
|
|
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
|
|
import com.pgvector.PGvector;
|
|
|
|
import java.sql.Connection;
|
|
|
|
import java.sql.DriverManager;
|
|
|
|
import java.sql.PreparedStatement;
|
|
|
|
import java.sql.SQLException;
|
|
|
|
|
|
|
|
import java.io.File;
|
|
|
|
import java.io.FileInputStream;
|
|
...
|
...
|
@@ -69,8 +56,10 @@ public class QuestionEmbeddingServiceImpl implements IQuestionEmbeddingService { |
|
|
|
|
|
|
|
@Value("${jeecg.upload.path}")
|
|
|
|
private String uploadPath;
|
|
|
|
@Value("${jeecg.ai-chat.embedId}")
|
|
|
|
private String embedId;
|
|
|
|
@Autowired
|
|
|
|
private RedisChatSetService redisChatSetService;
|
|
|
|
// @Value("${jeecg.ai-chat.embedId}")
|
|
|
|
// private String embedId;
|
|
|
|
|
|
|
|
private static final Set<String> ALLOWED_EXTENSIONS = Set.of("txt", "doc", "docx");
|
|
|
|
private static final Pattern UUID_PATTERN = Pattern.compile("_[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}");
|
|
...
|
...
|
@@ -95,7 +84,8 @@ public class QuestionEmbeddingServiceImpl implements IQuestionEmbeddingService { |
|
|
|
@Override
|
|
|
|
public int insert(QuestionEmbedding record) {
|
|
|
|
if (StringUtils.isNotBlank(record.getQuestion())){
|
|
|
|
Response<Embedding> embedding = aiModelUtils.getEmbedding(embedId, record.getQuestion());
|
|
|
|
AiragChatsetting chatSetting = redisChatSetService.getChatSetting();
|
|
|
|
Response<Embedding> embedding = aiModelUtils.getEmbedding(chatSetting.getEmbeddingId(), record.getQuestion());
|
|
|
|
record.setEmbedding(embedding.content().vector());
|
|
|
|
}
|
|
|
|
|
|
...
|
...
|
@@ -106,7 +96,8 @@ public class QuestionEmbeddingServiceImpl implements IQuestionEmbeddingService { |
|
|
|
@Override
|
|
|
|
public int update(QuestionEmbedding record) {
|
|
|
|
if (StringUtils.isNotBlank(record.getQuestion())){
|
|
|
|
Response<Embedding> embedding = aiModelUtils.getEmbedding(embedId, record.getQuestion());
|
|
|
|
AiragChatsetting chatSetting = redisChatSetService.getChatSetting();
|
|
|
|
Response<Embedding> embedding = aiModelUtils.getEmbedding(chatSetting.getEmbeddingId(), record.getQuestion());
|
|
|
|
record.setEmbedding(embedding.content().vector());
|
|
|
|
}
|
|
|
|
return questionEmbeddingMapper.update(record);
|
|
...
|
...
|
@@ -123,8 +114,8 @@ public class QuestionEmbeddingServiceImpl implements IQuestionEmbeddingService { |
|
|
|
}
|
|
|
|
|
|
|
|
@Override
|
|
|
|
public List<QuestionEmbedding> similaritySearchByQuestion(String question, int limit, Double minSimilarity) {
|
|
|
|
Response<Embedding> embedding = aiModelUtils.getEmbedding(embedId, question);
|
|
|
|
public List<QuestionEmbedding> similaritySearchByQuestion(String question, int limit, Double minSimilarity, AiragChatsetting chatSetting) {
|
|
|
|
Response<Embedding> embedding = aiModelUtils.getEmbedding(chatSetting.getEmbeddingId(), question);
|
|
|
|
return questionEmbeddingMapper.similaritySearchByQuestion(embedding.content().vector(), limit, minSimilarity);
|
|
|
|
}
|
|
|
|
|
|
...
|
...
|
@@ -202,7 +193,7 @@ public class QuestionEmbeddingServiceImpl implements IQuestionEmbeddingService { |
|
|
|
}
|
|
|
|
private void saveToEmbeddingsTable(List<String> segments, String originalFileName, String storedFileName, String knowledgeId) {
|
|
|
|
if (segments.isEmpty()) return;
|
|
|
|
|
|
|
|
AiragChatsetting chatSetting = redisChatSetService.getChatSetting();
|
|
|
|
String displayFileName = removeUuidSuffix(originalFileName);
|
|
|
|
displayFileName = FilenameUtils.removeExtension(displayFileName);
|
|
|
|
|
|
...
|
...
|
@@ -218,7 +209,7 @@ public class QuestionEmbeddingServiceImpl implements IQuestionEmbeddingService { |
|
|
|
String answer = segment.trim();
|
|
|
|
|
|
|
|
// 获取 embedding
|
|
|
|
Response<Embedding> embeddingResponse = aiModelUtils.getEmbedding(embedId, answer);
|
|
|
|
Response<Embedding> embeddingResponse = aiModelUtils.getEmbedding(chatSetting.getEmbeddingId(), answer);
|
|
|
|
float[] embeddingVector = embeddingResponse.content().vector();
|
|
|
|
|
|
|
|
Map<String, Object> metadata = new HashMap<>();
|
|
...
|
...
|
@@ -390,8 +381,8 @@ public class QuestionEmbeddingServiceImpl implements IQuestionEmbeddingService { |
|
|
|
|
|
|
|
|
|
|
|
log.info("保存分段: title={}, content_length={}", question, segment.length());
|
|
|
|
|
|
|
|
Response<Embedding> embeddingResponse = aiModelUtils.getEmbedding(embedId, record.getQuestion());
|
|
|
|
AiragChatsetting chatSetting = redisChatSetService.getChatSetting();
|
|
|
|
Response<Embedding> embeddingResponse = aiModelUtils.getEmbedding(chatSetting.getEmbeddingId(), record.getQuestion());
|
|
|
|
record.setEmbedding(embeddingResponse.content().vector());
|
|
|
|
record.setKnowledgeId(knowledgeId);
|
|
|
|
insert(record);
|
...
|
...
|
|