李想 / aiqasystem · 提交

转到一个项目

作者 lixiang
提交 c59448bfd9a485bf43a2f40a09c5c59ecf4cd7d4 1 个父辈 d0c419cc

智能助手修改

内嵌 并排对比
正在显示 11 个修改的文件 包含 427 行增加 和 143 行删除
... ... @@ -20,6 +20,7 @@ import org.springframework.web.servlet.ModelAndView;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.util.Arrays;
import java.util.List;
/**
* @Description: 按钮表单
... ... @@ -58,6 +59,22 @@ public class AiragButtonController extends JeecgController<AiragButton, IAiragBu
return Result.OK(pageList);
}
/**
* 分页列表查询
*
* @return
*/
@Operation(summary="按钮表单-智能助手按钮列表")
@GetMapping(value = "/buttonList")
public Result<List<AiragButton>> buttonList() {
QueryWrapper<AiragButton> queryWrapper = new QueryWrapper<>();
queryWrapper.eq("button_switch", "Y");
List<AiragButton> list = airagButtonService.list(queryWrapper);
return Result.OK(list);
}
/**
* 添加
*
... ...
package org.jeecg.modules.airag.app.controller;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.jeecg.common.api.vo.Result;
import org.jeecg.modules.airag.app.entity.QuestionEmbedding;
import org.jeecg.modules.airag.app.service.IQuestionEmbeddingService;
import org.jeecg.modules.airag.app.utils.JsonUtils;
import org.jeecg.modules.airag.llm.entity.AiragKnowledge;
import org.jeecg.modules.airag.llm.service.IAiragKnowledgeService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@RestController
@RequestMapping("/question/embedding")
@Slf4j
public class QuestionEmbeddingController {
@Autowired
private IQuestionEmbeddingService questionEmbeddingService;
@Autowired
private IAiragKnowledgeService airagKnowledgeService;
@GetMapping("/list")
public Result<List<QuestionEmbedding>> findAll() {
List<QuestionEmbedding> list = questionEmbeddingService.findAll();
Map<String, String> airagKnowledgeMap = airagKnowledgeService.list()
.stream()
.collect(Collectors.toMap(AiragKnowledge::getId, AiragKnowledge::getName));
list.forEach(item -> {
String metadata = item.getMetadata();
if (StringUtils.isNotBlank(metadata)) {
Map<String, String> jsonMap = JsonUtils.jsonUtils(metadata);
if (jsonMap.containsKey("knowledgeId")) {
item.setKnowledgeName(airagKnowledgeMap.get(jsonMap.get("knowledgeId")));
item.setKnowledgeId(jsonMap.get("knowledgeId"));
}
}
});
return Result.OK(list);
}
... ... @@ -51,8 +76,41 @@ public class QuestionEmbeddingController {
@PostMapping("/uploadZip")
@Transactional(rollbackFor = {Exception.class})
public Result<?> uploadZip(@RequestParam("file") MultipartFile file) {
return questionEmbeddingService.processZipUpload(file);
public Result<?> uploadZip(
@RequestParam("file") MultipartFile file,
@RequestParam("knowledgeId") String knowledgeId) {
// 增强日志记录
log.info("收到上传请求 - 文件名: {}, 大小: {} bytes, 知识库ID: {}",
file.getOriginalFilename(),
file.getSize(),
knowledgeId);
if (file.isEmpty()) {
log.warn("上传文件为空");
return Result.error("请上传有效的ZIP文件");
}
try {
// 验证文件类型
if (!"application/zip".equals(file.getContentType()) &&
!"application/x-zip-compressed".equals(file.getContentType())) {
log.warn("不支持的文件类型: {}", file.getContentType());
return Result.error("仅支持ZIP格式文件");
}
// 处理文件
return questionEmbeddingService.processZipUpload(file, knowledgeId);
} catch (Exception e) {
log.error("处理ZIP文件上传失败", e);
return Result.error("处理ZIP文件失败: " + e.getMessage());
}
}
public static void main(String[] args) {
String str = "{\"docId\": \"1930453193934766081\", \"docName\": \"在校学生身份证换证、补办手续办理\", \"knowledgeId\": \"1930453193934766080\"}";
Map<String, String> jsonMap = JsonUtils.jsonUtils(str);
System.out.println(jsonMap.get("docName"));
}
}
\ No newline at end of file
... ...
package org.jeecg.modules.airag.app.entity;
import com.baomidou.mybatisplus.annotation.TableField;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
... ... @@ -10,11 +11,44 @@ import java.util.Map;
@AllArgsConstructor
@NoArgsConstructor
public class QuestionEmbedding {
/**
* 主键id
*/
private String id;
/**
* 原文
*/
private String text;
/**
* 问题
*/
private String question;
/**
* 回答
*/
private String answer;
/**
* 元数据
*/
private String metadata;
/**
* 向量
*/
private float[] embedding;
/**
* 相似度
*/
private Double similarity;
/**
* 知识库名称
*/
@TableField(exist = false)
private String knowledgeName;
/**
* 知识库id
*/
@TableField(exist = false)
private String knowledgeId;
}
\ No newline at end of file
... ...
... ... @@ -20,7 +20,7 @@ import java.util.*;
public class PgVectorMapper {
// PostgreSQL连接参数（实际项目中应从配置读取）
private static final String URL = "jdbc:postgresql://192.168.100.103:5432/postgres";
private static final String URL = "jdbc:postgresql://192.168.100.104:5432/postgres";
private static final String USER = "postgres";
private static final String PASSWORD = "postgres";
... ...
package org.jeecg.modules.airag.app.mapper;
import com.alibaba.fastjson2.JSONObject;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
... ... @@ -25,7 +26,7 @@ public class QuestionEmbeddingMapper {
private AiModelUtils aiModelUtils;
// PostgreSQL连接参数（应与项目配置一致）
private static final String URL = "jdbc:postgresql://192.168.100.103:5432/postgres";
private static final String URL = "jdbc:postgresql://192.168.100.104:5432/postgres";
private static final String USER = "postgres";
private static final String PASSWORD = "postgres";
... ... @@ -86,7 +87,11 @@ public class QuestionEmbeddingMapper {
stmt.setString(4, record.getAnswer());
PGobject jsonObject = new PGobject();
jsonObject.setType("json");
jsonObject.setValue("{\"name\":\"John\", \"age\":30}");
// JSONObject mataData = new JSONObject();
// mataData.put("knowledgeId",record.getKnowledgeId());
jsonObject.setValue(record.getMetadata());
stmt.setObject(5, jsonObject);
Response<Embedding> embedding = aiModelUtils.getEmbedding("1925730210204721154", record.getQuestion());
stmt.setObject(6, embedding.content().vector());
... ... @@ -109,7 +114,11 @@ public class QuestionEmbeddingMapper {
stmt.setString(3, record.getAnswer());
PGobject jsonObject = new PGobject();
jsonObject.setType("json");
jsonObject.setValue("{\"name\":\"John\", \"age\":30}");
JSONObject mataData = new JSONObject();
// mataData.put("knowledgeId",record.getKnowledgeId());
jsonObject.setValue(mataData.toJSONString());
stmt.setObject(4, jsonObject);
Response<Embedding> embedding = aiModelUtils.getEmbedding("1925730210204721154", record.getQuestion());
... ... @@ -240,7 +249,7 @@ public class QuestionEmbeddingMapper {
String metadataJson = rs.getString("metadata");
if (StringUtils.isNotBlank(metadataJson)) {
record.setMetadata("");
record.setMetadata(metadataJson);
}
return record;
... ...
... ... @@ -15,5 +15,5 @@ public interface IQuestionEmbeddingService {
List<QuestionEmbedding> similaritySearchByQuestion(String question, int limit, Double minSimilarity);
List<QuestionEmbedding> similaritySearch(float[] vector, int limit);
Result<?> processZipUpload(MultipartFile file);
Result<?> processZipUpload(MultipartFile file, String knowledgeId);
}
\ No newline at end of file
... ...
... ... @@ -6,13 +6,9 @@ import org.apache.poi.hwpf.usermodel.Paragraph;
import org.apache.poi.hwpf.usermodel.Range;
import dev.langchain4j.data.document.Document;
import dev.langchain4j.data.document.DocumentSplitter;
import dev.langchain4j.data.document.loader.FileSystemDocumentLoader;
import dev.langchain4j.data.document.parser.TextDocumentParser;
import dev.langchain4j.data.document.splitter.DocumentByParagraphSplitter;
import dev.langchain4j.data.document.splitter.DocumentSplitters;
import dev.langchain4j.data.embedding.Embedding;
import dev.langchain4j.data.segment.TextSegment;
import dev.langchain4j.model.embedding.EmbeddingModel;
import dev.langchain4j.model.output.Response;
import org.apache.commons.io.FilenameUtils;
import org.apache.poi.xwpf.usermodel.IBodyElement;
... ... @@ -20,7 +16,6 @@ import org.apache.poi.xwpf.usermodel.XWPFDocument;
import org.apache.poi.xwpf.usermodel.XWPFParagraph;
import org.apache.poi.xwpf.usermodel.XWPFTable;
import org.jeecg.common.api.vo.Result;
import org.jeecg.common.util.CommonUtils;
import org.jeecg.modules.airag.app.entity.QuestionEmbedding;
import org.jeecg.modules.airag.app.mapper.QuestionEmbeddingMapper;
import org.jeecg.modules.airag.app.service.IQuestionEmbeddingService;
... ... @@ -36,6 +31,7 @@ import org.springframework.web.multipart.MultipartFile;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
... ... @@ -65,6 +61,7 @@ public class QuestionEmbeddingServiceImpl implements IQuestionEmbeddingService {
private static final Set<String> ALLOWED_EXTENSIONS = Set.of("txt", "doc", "docx");
private static final Pattern SPECIAL_CHARS_PATTERN = Pattern.compile("[^a-zA-Z0-9\\u4e00-\\u9fa5\\s]");
private static final Pattern UUID_PATTERN = Pattern.compile("_[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}");
@Override
public List<QuestionEmbedding> findAll() {
... ... @@ -101,7 +98,8 @@ public class QuestionEmbeddingServiceImpl implements IQuestionEmbeddingService {
return questionEmbeddingMapper.similaritySearch(vector, limit);
}
public Result<?> processZipUpload(MultipartFile zipFile) {
@Override
public Result<?> processZipUpload(MultipartFile zipFile, String knowledgeId) {
try {
Path tempDir = Files.createTempDirectory("zip_upload_");
List<Path> validFiles = extractAndFilterZip(zipFile, tempDir);
... ... @@ -111,7 +109,7 @@ public class QuestionEmbeddingServiceImpl implements IQuestionEmbeddingService {
}
for (Path filePath : validFiles) {
processSingleFile(filePath);
processSingleFile(filePath, knowledgeId);
}
return Result.OK("文件上传和处理成功");
... ... @@ -124,7 +122,7 @@ public class QuestionEmbeddingServiceImpl implements IQuestionEmbeddingService {
private List<Path> extractAndFilterZip(MultipartFile zipFile, Path tempDir) throws IOException {
List<Path> validFiles = new ArrayList<>();
try (ZipInputStream zipIn = new ZipInputStream(zipFile.getInputStream())) {
try (ZipInputStream zipIn = new ZipInputStream(zipFile.getInputStream(), Charset.forName("GBK"))) {
ZipEntry entry;
while ((entry = zipIn.getNextEntry()) != null) {
if (!entry.isDirectory()) {
... ... @@ -144,11 +142,13 @@ public class QuestionEmbeddingServiceImpl implements IQuestionEmbeddingService {
return validFiles;
}
private void processSingleFile(Path filePath) throws Exception {
private void processSingleFile(Path filePath, String knowledgeId) throws Exception {
String originalFileName = filePath.getFileName().toString();
String fileExt = FilenameUtils.getExtension(originalFileName);
String newFileName = FilenameUtils.removeExtension(originalFileName) + "_" + UUID.randomUUID() + "." + fileExt;
Path targetPath = Paths.get(uploadPath, newFileName);
// 生成带UUID的文件名用于存储
String storedFileName = generateStoredFileName(originalFileName);
Path targetPath = Paths.get(uploadPath, storedFileName);
Files.move(filePath, targetPath, StandardCopyOption.REPLACE_EXISTING);
List<String> segments;
... ... @@ -160,7 +160,13 @@ public class QuestionEmbeddingServiceImpl implements IQuestionEmbeddingService {
segments = splitWordDocument(targetPath.toString());
}
saveSegmentsToDatabase(segments, originalFileName, newFileName);
saveSegmentsToDatabase(segments, originalFileName, storedFileName, knowledgeId);
}
private String generateStoredFileName(String originalFileName) {
String baseName = FilenameUtils.removeExtension(originalFileName);
String ext = FilenameUtils.getExtension(originalFileName);
return baseName + "_" + UUID.randomUUID() + "." + ext;
}
private String readFileContent(Path filePath) throws IOException {
... ... @@ -168,10 +174,44 @@ public class QuestionEmbeddingServiceImpl implements IQuestionEmbeddingService {
}
private String cleanText(String text) {
text = SPECIAL_CHARS_PATTERN.matcher(text).replaceAll("");
// 保留基本的标点符号，包括 . : - 等
Pattern preservedCharsPattern = Pattern.compile("[^a-zA-Z0-9\\u4e00-\\u9fa5\\s.,:、，：；。;-]");
text = preservedCharsPattern.matcher(text).replaceAll("");
// 将多个换行符缩减为一个换行符
text = text.replaceAll("(\r?\n){2,}", "\n");
// 处理其他空白字符
return text.replaceAll("\\s+", " ").trim();
}
// 修改isHeading方法中的判断条件，不再排除包含.的文本
private static boolean isHeading(Paragraph para, Range range) {
int styleIndex = para.getStyleIndex();
if (styleIndex >= 1 && styleIndex <= 9) {
return true;
}
try {
CharacterRun run = para.getCharacterRun(0);
if (run.isBold() || run.getFontSize() > 12) {
return true;
}
} catch (Exception e) {
log.warn("获取字符格式失败", e);
}
String text = para.text().trim();
return text.toUpperCase().equals(text) &&
text.length() < 100 &&
!text.contains("\t"); // 移除了 !text.contains(".") 的判断
}
private String removeUuidSuffix(String fileName) {
// 移除UUID后缀部分
return UUID_PATTERN.matcher(fileName).replaceFirst("");
}
private List<String> splitTxtDocument(String content) {
DocumentSplitter splitter = new DocumentByParagraphSplitter(1000, 200);
Document document = Document.from(content);
... ... @@ -185,8 +225,10 @@ public class QuestionEmbeddingServiceImpl implements IQuestionEmbeddingService {
List<String> result = new ArrayList<>();
String ext = FilenameUtils.getExtension(filePath).toLowerCase();
StringBuilder fullContent = new StringBuilder();
String fileName = new File(filePath).getName();
fileName = fileName.substring(0, fileName.lastIndexOf('.')); // 去掉后缀
// 获取无UUID的文件名用于显示
String displayFileName = removeUuidSuffix(new File(filePath).getName());
displayFileName = FilenameUtils.removeExtension(displayFileName);
if (ext.equals("docx")) {
try (XWPFDocument doc = new XWPFDocument(new FileInputStream(filePath))) {
... ... @@ -207,7 +249,7 @@ public class QuestionEmbeddingServiceImpl implements IQuestionEmbeddingService {
String style = para.getStyle();
if (style != null && style.matches("Heading\\d")) {
if (currentSection.length() > 0) {
if (!currentSection.isEmpty()) {
result.add(currentSection.toString().trim());
}
currentSection = new StringBuilder(text).append("\n");
... ... @@ -219,7 +261,7 @@ public class QuestionEmbeddingServiceImpl implements IQuestionEmbeddingService {
fullContent.append(tableContent).append("\n");
if (!isTableSection) {
if (currentSection.length() > 0) {
if (!currentSection.isEmpty()) {
result.add(currentSection.toString().trim());
}
currentSection = new StringBuilder();
... ... @@ -229,7 +271,7 @@ public class QuestionEmbeddingServiceImpl implements IQuestionEmbeddingService {
}
}
if (currentSection.length() > 0) {
if (!currentSection.isEmpty()) {
result.add(currentSection.toString().trim());
}
}
... ... @@ -246,7 +288,7 @@ public class QuestionEmbeddingServiceImpl implements IQuestionEmbeddingService {
if (para.isInTable()) {
if (!isTableSection) {
if (currentSection.length() > 0) {
if (!currentSection.isEmpty()) {
result.add(currentSection.toString().trim());
}
currentSection = new StringBuilder();
... ... @@ -261,7 +303,7 @@ public class QuestionEmbeddingServiceImpl implements IQuestionEmbeddingService {
}
if (isHeading(para, range)) {
if (currentSection.length() > 0) {
if (!currentSection.isEmpty()) {
result.add(currentSection.toString().trim());
}
currentSection = new StringBuilder(text).append("\n");
... ... @@ -271,59 +313,49 @@ public class QuestionEmbeddingServiceImpl implements IQuestionEmbeddingService {
}
}
if (currentSection.length() > 0) {
if (!currentSection.isEmpty()) {
result.add(currentSection.toString().trim());
}
}
}
if (fullContent.length() < 1000) {
return Collections.singletonList(fileName + "\n" + fullContent.toString().trim());
return Collections.singletonList(displayFileName + "\n" + fullContent.toString().trim());
}
return result;
}
private String extractTableContent(XWPFTable table) {
StringBuilder tableContent = new StringBuilder();
StringBuilder tableContent = new StringBuilder("\n"); // 表格前加换行
table.getRows().forEach(row -> {
row.getTableCells().forEach(cell -> {
tableContent.append("| ").append(cleanText(cell.getText())).append(" ");
// 处理单元格内容中的多个换行
String cellText = cleanText(cell.getText()).replaceAll("(\r?\n){2,}", "\n");
tableContent.append("| ").append(cellText).append(" ");
});
tableContent.append("|\n");
});
return tableContent.toString();
}
private static boolean isHeading(Paragraph para, Range range) {
int styleIndex = para.getStyleIndex();
if (styleIndex >= 1 && styleIndex <= 9) {
return true;
}
try {
CharacterRun run = para.getCharacterRun(0);
if (run.isBold() || run.getFontSize() > 12) {
return true;
}
} catch (Exception e) {
log.warn("获取字符格式失败", e);
}
String text = para.text().trim();
return text.toUpperCase().equals(text) &&
text.length() < 100 &&
!text.contains(".") &&
!text.contains("\t");
}
private void saveSegmentsToDatabase(List<String> segments, String originalFileName, String storedFileName) {
private void saveSegmentsToDatabase(List<String> segments, String originalFileName, String storedFileName, String knowledgeId) {
if (segments.isEmpty()) {
return;
}
String fileNameWithoutExt = originalFileName.substring(0, originalFileName.lastIndexOf('.'));
String question = segments.size() == 1 ? fileNameWithoutExt : null;
// 从存储文件名中提取UUID部分
String uuid = storedFileName.substring(
storedFileName.lastIndexOf('_') + 1,
storedFileName.lastIndexOf('.')
);
// 获取无UUID和扩展名的文件名用于显示
String displayFileName = removeUuidSuffix(originalFileName);
displayFileName = FilenameUtils.removeExtension(displayFileName);
// 判断是否是单一段落
boolean isSingleSegment = segments.size() == 1;
for (String segment : segments) {
if (segment.trim().isEmpty()) {
... ... @@ -333,20 +365,26 @@ public class QuestionEmbeddingServiceImpl implements IQuestionEmbeddingService {
QuestionEmbedding record = new QuestionEmbedding();
record.setId(UUID.randomUUID().toString());
if (question != null) {
record.setQuestion(question);
if (isSingleSegment) {
record.setQuestion(displayFileName);
record.setAnswer(segment.trim());
} else {
String firstLine = segment.lines().findFirst().orElse("未命名问题");
record.setQuestion(cleanText(firstLine));
String firstLine = segment.lines().findFirst().orElse("");
record.setQuestion(displayFileName + ": " + cleanText(firstLine));
record.setAnswer(segment.trim());
}
record.setAnswer(segment.trim());
record.setText("");
record.setMetadata("{\"fileName\":\"" + storedFileName + "\"}");
record.setMetadata(String.format(
"{\"docId\":\"%s\",\"docName\":\"%s\",\"knowledgeId\":\"%s\"}",
uuid,
originalFileName,
knowledgeId
));
Response<Embedding> embeddingResponse = aiModelUtils.getEmbedding("1925730210204721154", record.getQuestion());
record.setEmbedding(embeddingResponse.content().vector());
record.setKnowledgeId(knowledgeId);
questionEmbeddingMapper.insert(record);
}
}
... ...
package org.jeecg.modules.airag.app.utils;
import lombok.extern.slf4j.Slf4j;
import me.zhyd.oauth.log.Log;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Base64;
@Slf4j
public class FileToBase64Util {
/**
* 将文件转换为Base64字符串
* @param filePath 文件路径
* @return Base64编码的字符串
* @throws IOException 如果文件读取失败
*/
public static String fileToBase64(String filePath) throws IOException {
// 1. 参数校验
if (filePath == null || filePath.trim().isEmpty()) {
log.error("文件路径为空");
return null;
}
// 2. 检查文件是否存在
Path path = Paths.get(filePath);
if (!Files.exists(path)) {
String errorMsg = String.format("文件不存在: %s", filePath);
log.error(errorMsg);
return null;
}
// 读取文件字节
byte[] fileBytes = Files.readAllBytes(path);
// 使用Base64编码器进行编码
return Base64.getEncoder().encodeToString(fileBytes);
}
/**
* 测试方法
*/
public static void main(String[] args) {
try {
// 示例：转换一个文件为Base64
String base64String = fileToBase64("D:\\upload\\新生落户手续办理_f75c658b-6a8c-4b19-94ff-28d3c469be9a.docx");
System.out.println("Base64编码结果:");
System.out.println(base64String);
} catch (IOException e) {
System.err.println("文件转换失败: " + e.getMessage());
}
}
}
\ No newline at end of file
... ...
package org.jeecg.modules.airag.app.utils;
import java.sql.*;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class PureJdbcVectorQuery {
// PostgreSQL连接参数
private static final String URL = "jdbc:postgresql://192.168.100.103:5432/postgres";
private static final String USER = "postgres";
private static final String PASSWORD = "postgres";
public static List<Map<String, Object>> queryVectors() {
List<Map<String, Object>> results = new ArrayList<>();
try (Connection conn = DriverManager.getConnection(URL, USER, PASSWORD);
PreparedStatement stmt = conn.prepareStatement("SELECT * FROM embeddings");
ResultSet rs = stmt.executeQuery()) {
ResultSetMetaData metaData = rs.getMetaData();
int columnCount = metaData.getColumnCount();
while (rs.next()) {
Map<String, Object> row = new HashMap<>();
for (int i = 1; i <= columnCount; i++) {
String columnName = metaData.getColumnName(i);
Object value = "embedding".equals(columnName)
? rs.getBytes(i) // 向量字段特殊处理
: rs.getObject(i);
row.put(columnName, value);
}
results.add(row);
}
} catch (SQLException e) {
e.printStackTrace();
}
return results;
}
public static void main(String[] args) {
List<Map<String, Object>> maps = queryVectors();
for (Map<String, Object> map : maps) {
System.out.println(map);
}
}
}
\ No newline at end of file
//package org.jeecg.modules.airag.app.utils;
//
//import java.sql.*;
//import java.util.ArrayList;
//import java.util.HashMap;
//import java.util.List;
//import java.util.Map;
//
//public class PureJdbcVectorQuery {
//
// // PostgreSQL连接参数
// private static final String URL = "jdbc:postgresql://192.168.100.103:5432/postgres";
// private static final String USER = "postgres";
// private static final String PASSWORD = "postgres";
//
// public static List<Map<String, Object>> queryVectors() {
// List<Map<String, Object>> results = new ArrayList<>();
//
// try (Connection conn = DriverManager.getConnection(URL, USER, PASSWORD);
// PreparedStatement stmt = conn.prepareStatement("SELECT * FROM embeddings");
// ResultSet rs = stmt.executeQuery()) {
//
// ResultSetMetaData metaData = rs.getMetaData();
// int columnCount = metaData.getColumnCount();
//
// while (rs.next()) {
// Map<String, Object> row = new HashMap<>();
// for (int i = 1; i <= columnCount; i++) {
// String columnName = metaData.getColumnName(i);
// Object value = "embedding".equals(columnName)
// ? rs.getBytes(i) // 向量字段特殊处理
// : rs.getObject(i);
// row.put(columnName, value);
// }
// results.add(row);
// }
// } catch (SQLException e) {
// e.printStackTrace();
// }
// return results;
// }
//
//
// public static void main(String[] args) {
// List<Map<String, Object>> maps = queryVectors();
// for (Map<String, Object> map : maps) {
// System.out.println(map);
// }
// }
//}
\ No newline at end of file
... ...
package org.jeecg.modules.airag.llm.handler;
import com.alibaba.fastjson.JSONObject;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Lists;
import dev.langchain4j.data.document.Document;
import dev.langchain4j.data.document.DocumentSplitter;
... ... @@ -251,6 +253,22 @@ public class EmbeddingHandler implements IEmbeddingHandler {
data.put("content", matchRes.embedded().text());
Metadata metadata = matchRes.embedded().metadata();
data.put("chunk", metadata.getInteger("index"));
Map<String, Object> metadataMap = new HashMap<>();
metadataMap.put("docId", metadata.getString("docId")); // 假设metadata中有docId字段
metadataMap.put("docName", metadata.getString(EMBED_STORE_METADATA_DOCNAME));
metadataMap.put("index", metadata.getInteger("index"));
ObjectMapper objectMapper = new ObjectMapper();
try {
data.put("metadata", objectMapper.writeValueAsString(metadataMap));
} catch (JsonProcessingException e) {
throw new RuntimeException(e);
}
data.put(EMBED_STORE_METADATA_DOCNAME, metadata.getString(EMBED_STORE_METADATA_DOCNAME));
return data;
}).collect(Collectors.toList());
... ...
package org.jeecg.modules.airag.zdyrag.controller;
import cn.hutool.core.collection.CollectionUtil;
import com.fasterxml.jackson.databind.ObjectMapper;
import dev.langchain4j.data.message.ChatMessage;
import dev.langchain4j.data.message.UserMessage;
import dev.langchain4j.service.TokenStream;
import dev.langchain4j.store.embedding.pgvector.PgVectorEmbeddingStore;
import io.swagger.v3.oas.annotations.Operation;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.jeecg.ai.handler.AIParams;
import org.jeecg.ai.handler.LLMHandler;
import org.jeecg.common.api.vo.Result;
import org.jeecg.modules.airag.app.entity.QuestionEmbedding;
import org.jeecg.modules.airag.app.service.IQuestionEmbeddingService;
import org.jeecg.modules.airag.app.utils.FileToBase64Util;
import org.jeecg.modules.airag.common.handler.IAIChatHandler;
import org.jeecg.modules.airag.llm.handler.EmbeddingHandler;
import org.jeecg.modules.airag.llm.service.IAiragKnowledgeService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
... ... @@ -25,35 +32,50 @@ import java.util.Map;
@RestController
@RequestMapping("/airag/zdyRag")
@Slf4j
public class ZdyRagController {
@Autowired
private EmbeddingHandler embeddingHandler;
@Autowired
IAIChatHandler aiChatHandler;
@Autowired
LLMHandler llmHandler;
@Autowired
private IQuestionEmbeddingService questionEmbeddingService;
@Value("${jeecg.upload.path}")
private String uploadPath;
@Operation(summary = "send")
@GetMapping("send")
public Result<Map<String, Object>> send(String questionText) {
public Result<Map<String, Object>> send(String questionText) throws Exception {
String knowId = "1926872137990148098";
// String text = "身份证丢失办理流程";
Integer topNumber = 3;
Integer topNumber = 1;
Double similarity = 0.8;
HashMap<String, Object> resMap = new HashMap<>();
//根据问题相似度进行查询
List<QuestionEmbedding> questionEmbeddings = questionEmbeddingService.similaritySearchByQuestion(questionText, 1,0.8);
for (QuestionEmbedding questionEmbedding : questionEmbeddings) {
resMap.put("question", questionEmbedding.getQuestion());
resMap.put("question", questionText);
resMap.put("answer", questionEmbedding.getAnswer());
resMap.put("similarity", similarity);
System.out.println("questionEmbedding.getQuestion() = " + questionEmbedding.getQuestion());
System.out.println("questionEmbedding.getAnswer() = " + questionEmbedding.getAnswer());
System.out.println("questionEmbedding.getSimilarity() = " + questionEmbedding.getSimilarity());
System.out.println("-------------------------------------------------------------");
resMap.put("similarity", questionEmbedding.getSimilarity());
ObjectMapper objectMapper = new ObjectMapper();
Map<String, String> metadata = objectMapper.readValue(questionEmbedding.getMetadata(), Map.class);
// 获取docName和docId
if (metadata != null) {
String docName = metadata.get("docName");
resMap.put("fileName", docName);
String fileName = generateFilePath(questionEmbedding.getMetadata());
if (StringUtils.isNotBlank(fileName)) {
resMap.put("fileBase64", FileToBase64Util.fileToBase64(uploadPath + fileName));
}
}
log.info("questionEmbedding.getMetadata() = " + questionEmbedding.getMetadata());
log.info("questionEmbedding.getQuestion() = " + questionEmbedding.getQuestion());
log.info("questionEmbedding.getAnswer() = " + questionEmbedding.getAnswer());
log.info("questionEmbedding.getSimilarity() = " + questionEmbedding.getSimilarity());
log.info("-------------------------------------------------------------");
}
//返回问题库命中的问题
if (!questionEmbeddings.isEmpty()) {
... ... @@ -61,43 +83,78 @@ public class ZdyRagController {
}
List<Map<String, Object>> maps = embeddingHandler.searchEmbedding(knowId, questionText, topNumber, similarity);
if (CollectionUtil.isEmpty(maps)) {
resMap.put("answer", "该问题未记录在知识库中");
//未记录在知识库中的问题插入未记录问题表
return Result.OK(resMap);
}
StringBuilder content = new StringBuilder();
for (Map<String, Object> map : maps) {
if (Double.parseDouble(map.get("score").toString()) > similarity){
System.out.println("score = " + map.get("score").toString());
System.out.println("content = " + map.get("content").toString());
log.info("score = " + map.get("score").toString());
log.info("content = " + map.get("content").toString());
content.append(map.get("content").toString()).append("\n");
}
}
List<ChatMessage> messages = new ArrayList<>();
String questin = "请整理出与用户所提出的问题相关的信息，舍弃掉与问题无关的内容，进行整理，回答用户的问题" +
String questin = "请整理出与用户所提出的问题相关的信息，参考内容中含有与问题无关信息，舍弃掉与问题无关的内容，进行整理，回答用户的问题" +
"问题如下：" + questionText +
"文本信息如下：" + content
;
"参考信息如下：" + content ;
messages.add(new UserMessage("user", questin));
// AIParams aiParams = new AIParams();
// aiParams.setBaseUrl("http://localhost:11434");
// aiParams.setModelName("EntropyYue/chatglm3");
// aiParams.setProvider("OLLAMA");
String chat = aiChatHandler.completions("1926875898187878401", messages, null);
resMap.put("question", questionText);
resMap.put("answer", chat);
resMap.put("similarity", maps.get(0).get("score").toString());
String fileName = generateFilePath(maps.get(0).get("metadata").toString());
resMap.put("fileName", fileName);
resMap.put("fileBase64",FileToBase64Util.fileToBase64(uploadPath + fileName));
return Result.OK(resMap);
}
private String generateFilePath(String metadataJson) throws Exception {
if (StringUtils.isEmpty(metadataJson)) {
return "";
}
ObjectMapper objectMapper = new ObjectMapper();
// 解析JSON字符串
Map<String, String> metadata = objectMapper.readValue(metadataJson, Map.class);
// 获取docName和docId
String docName = metadata.get("docName");
String docId = metadata.get("docId");
// 分离文件名和扩展名
if(StringUtils.isEmpty(docName)){
return null;
}
int dotIndex = docName.lastIndexOf('.');
String baseName = (dotIndex > 0) ? docName.substring(0, dotIndex) : docName;
String extension = (dotIndex > 0) ? docName.substring(dotIndex) : "";
// 组合成新文件名
return baseName + "_" + docId + extension;
}
public static void main(String[] args) {
List<ChatMessage> messages = new ArrayList<>();
messages.add(new UserMessage("user", "你好，你是谁？"));
LLMHandler llmHandler1 = new LLMHandler();
AIParams aiParams = new AIParams();
aiParams.setBaseUrl("http://localhost:11434");
aiParams.setModelName("EntropyYue/chatglm3");
aiParams.setProvider("OLLAMA");
TokenStream chat = llmHandler1.chat(messages, aiParams);
System.out.println("chat = " + chat);
String s = "学生户口复印_efde055d-1207-4b6f-8d46-79eb557ca711.docx";
String s1 = StringUtils.substringBefore(s, ".");
log.info("s1 = " + s1);
String[] split = s.split("_");
for (String string : split) {
log.info("string = " + string);
}
}
... ...