李想 / aiqasystem · 提交

转到一个项目

作者 dong
提交 e09114866e71bf146403c655123011e4af6a64d7 1 个父辈 1c4b1a7f

资料管理界面后端增删改查及导入功能

内嵌 并排对比
正在显示 5 个修改的文件 包含 370 行增加 和 203 行删除
package org.jeecg.modules.airag.app.controller;
import java.lang.reflect.Field;
import java.util.Arrays;
import java.util.List;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.sql.DataSource;
import java.sql.Connection; // PostgreSQL JDBC 连接
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import dev.langchain4j.store.embedding.EmbeddingStore; // LangChain4j 的 EmbeddingStore 接口
import dev.langchain4j.store.embedding.pgvector.PgVectorEmbeddingStore;
import dev.langchain4j.data.embedding.Embedding;
import dev.langchain4j.data.segment.TextSegment;
import dev.langchain4j.model.embedding.EmbeddingModel;
import dev.langchain4j.model.output.Response;
import dev.langchain4j.store.embedding.EmbeddingMatch;
import dev.langchain4j.store.embedding.EmbeddingSearchRequest;
import dev.langchain4j.store.embedding.EmbeddingSearchResult;
import dev.langchain4j.store.embedding.EmbeddingStore;
import dev.langchain4j.store.embedding.pgvector.PgVectorEmbeddingStore;
import org.jeecg.common.api.vo.Result;
import org.jeecg.common.system.query.QueryGenerator;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import cn.hutool.core.lang.generator.SnowflakeGenerator;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import lombok.extern.slf4j.Slf4j;
//import org.jeecg.modules.airag.app.config.PostgreEmbeddingStore;
import org.jeecg.modules.airag.app.entity.Embeddings;
import org.apache.poi.xwpf.usermodel.*;
import org.apache.shiro.authz.annotation.RequiresPermissions;
import org.jeecg.common.api.vo.Result;
import org.jeecg.common.aspect.annotation.AutoLog;
import org.jeecg.modules.airag.app.entity.Embeddings;
import org.jeecg.modules.airag.app.service.IEmbeddingsService;
import org.jeecg.common.system.base.controller.JeecgController;
import org.jeecg.modules.airag.app.utils.AiModelUtils;
import org.jeecg.modules.airag.llm.entity.AiragModel;
import org.jeecg.modules.airag.llm.handler.EmbeddingHandler;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.jdbc.DataSourceBuilder;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.servlet.ModelAndView;
import io.swagger.v3.oas.annotations.tags.Tag;
import io.swagger.v3.oas.annotations.Operation;
import org.jeecg.common.aspect.annotation.AutoLog;
import org.apache.shiro.authz.annotation.RequiresPermissions;
/**
* @Description: Embeddings
* @Author: jeecg-boot
* @Date: 2025-05-26
* @Version: V1.0
*/
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.sql.SQLException;
import java.util.*;
/**
* @Description: Embeddings
* @Author: jeecg-boot
* @Date: 2025-05-26
* @Version: V1.0
*/
@Tag(name="Embeddings")
@RestController
@RequestMapping("/embeddings/embeddings")
@Slf4j
public class EmbeddingsController {
@Autowired
private IEmbeddingsService embeddingsService;
@Autowired
private AiModelUtils aiModelUtils;
@Autowired
private EmbeddingHandler embeddingHandler;
/**
* 分页列表查询
*
* @param Embeddings
* @param pageNo
* @param pageSize
* @param req
* @return
*/
//@AutoLog(value = "Embeddings-分页列表查询")
@Operation(summary="Embeddings-分页列表查询")
@GetMapping(value = "/list")
public Result<IPage<Embeddings>> queryPageList(Embeddings Embeddings,
@RequestParam(name="pageNo", defaultValue="1") Integer pageNo,
@RequestParam(name="pageSize", defaultValue="10") Integer pageSize,
HttpServletRequest req) throws NoSuchFieldException, IllegalAccessException, SQLException {
// AiragModel airagModel = new AiragModel();
// airagModel.setId("1925730210204721154");
// airagModel.setProvider("OLLAMA");
// airagModel.setModelName("nomic-embed-text");
// airagModel.setBaseUrl("http://localhost:11434");
// EmbeddingStore<TextSegment> embedStore = embeddingHandler.getEmbedStore(airagModel);
// embeddingHandler.searchEmbedding()
Response<Embedding> embedding = aiModelUtils.getEmbedding("1925730210204721154", "33333");
List<Embeddings> records = embeddingsService.findAll();
for (Embeddings record : records) {
System.out.println("record = " + record);
}
Page<Embeddings> page = new Page<Embeddings>(pageNo, pageSize);
page.setRecords(records);
return Result.OK(page);
}
/**
* 添加
*
* @param Embeddings
* @return
*/
@AutoLog(value = "Embeddings-添加")
@Operation(summary="Embeddings-添加")
@RequiresPermissions("Embeddings:Embeddings:add")
@PostMapping(value = "/add")
public Result<String> add(@RequestBody Embeddings Embeddings) {
@Autowired
private IEmbeddingsService embeddingsService;
/*@Autowired
private AiModelUtils aiModelUtils;
*/
/**
* 分页列表查询
*
* @param embeddings
* @param pageNo
* @param pageSize
* @param req
* @return
*/
//@AutoLog(value = "Embeddings-分页列表查询")
@Operation(summary = "Embeddings-分页列表查询")
@GetMapping(value = "/list")
public Result<IPage<Embeddings>> queryPageList(Embeddings embeddings,
@RequestParam(name = "pageNo", defaultValue = "1") Integer pageNo,
@RequestParam(name = "pageSize", defaultValue = "10") Integer pageSize,
HttpServletRequest req) throws NoSuchFieldException, IllegalAccessException, SQLException {
//Response<Embedding> embedding = aiModelUtils.getEmbedding("1925730210204721154", "33333");
List<Embeddings> records = embeddingsService.findAll(embeddings);
for (Embeddings record : records) {
System.out.println("record = " + record);
}
Page<Embeddings> page = new Page<Embeddings>(pageNo, pageSize);
page.setRecords(records);
return Result.OK(page);
}
/**
* 添加
*
* @param embeddings
* @return
*/
@AutoLog(value = "Embeddings-添加")
@Operation(summary = "Embeddings-添加")
@RequiresPermissions("embeddings:embeddings:add")
@PostMapping(value = "/add")
public Result<String> add(@RequestBody Embeddings embeddings) {
// embeddingsService.save(Embeddings);
return Result.OK("添加成功！");
}
/**
* 编辑
*
* @param Embeddings
* @return
*/
@AutoLog(value = "Embeddings-编辑")
@Operation(summary="Embeddings-编辑")
@RequiresPermissions("Embeddings:Embeddings:edit")
@RequestMapping(value = "/edit", method = {RequestMethod.PUT,RequestMethod.POST})
public Result<String> edit(@RequestBody Embeddings Embeddings) {
// 1. 构建完整的metadata
Map<String, Object> metadata = new HashMap<>();
SnowflakeGenerator snowflakeGenerator = new SnowflakeGenerator();
String knowledgeId = String.valueOf(snowflakeGenerator.next());
metadata.put("knowledgeId", knowledgeId); // 使用前端传入的知识库ID
metadata.put("docName", embeddings.getDocName());
String docId = String.valueOf(snowflakeGenerator.next());
metadata.put("docId", docId); // 自动生成唯一文档ID
metadata.put("index", "0"); // 默认索引位置为0
// 2. 设置到embeddings对象
embeddings.setMetadata(metadata);
/*// 3. 生成向量嵌入（实际项目中应调用嵌入模型API）
embeddings.setEmbedding(generateEmbedding(embeddings.getText()));
*/
System.out.println(new SnowflakeGenerator().next());
embeddingsService.insert(embeddings);
return Result.OK("添加成功！");
}
/**
* 编辑
*
* @param embeddings
* @return
*/
@AutoLog(value = "Embeddings-编辑")
@Operation(summary = "Embeddings-编辑")
@RequiresPermissions("embeddings:embeddings:edit")
@RequestMapping(value = "/edit", method = {RequestMethod.PUT, RequestMethod.POST})
public Result<String> edit(@RequestBody Embeddings embeddings) {
// embeddingsService.updateById(Embeddings);
return Result.OK("编辑成功!");
}
/**
* 通过id删除
*
* @param id
* @return
*/
@AutoLog(value = "Embeddings-通过id删除")
@Operation(summary="Embeddings-通过id删除")
@RequiresPermissions("Embeddings:Embeddings:delete")
@DeleteMapping(value = "/delete")
public Result<String> delete(@RequestParam(name="id",required=true) String id) {
// embeddingsService.removeById(id);
return Result.OK("删除成功!");
}
/**
* 批量删除
*
* @param ids
* @return
*/
@AutoLog(value = "Embeddings-批量删除")
@Operation(summary="Embeddings-批量删除")
@RequiresPermissions("Embeddings:Embeddings:deleteBatch")
@DeleteMapping(value = "/deleteBatch")
public Result<String> deleteBatch(@RequestParam(name="ids",required=true) String ids) {
embeddingsService.update(embeddings);
return Result.OK("编辑成功!");
}
/**
* 通过id删除
*
* @param id
* @return
*/
@AutoLog(value = "Embeddings-通过id删除")
@Operation(summary = "Embeddings-通过id删除")
@RequiresPermissions("embeddings:embeddings:delete")
@DeleteMapping(value = "/delete")
public Result<String> delete(@RequestParam(name = "id", required = true) String id) {
//embeddingsService.removeById(id);
embeddingsService.deleteById(id);
return Result.OK("删除成功!");
}
/**
* 批量删除
*
* @param ids
* @return
*/
@AutoLog(value = "Embeddings-批量删除")
@Operation(summary = "Embeddings-批量删除")
@RequiresPermissions("embeddings:embeddings:deleteBatch")
@DeleteMapping(value = "/deleteBatch")
public Result<String> deleteBatch(@RequestParam(name = "ids", required = true) String ids) {
// this.embeddingsService.removeByIds(Arrays.asList(ids.split(",")));
return Result.OK("批量删除成功!");
}
/**
* 通过id查询
*
* @param id
* @return
*/
//@AutoLog(value = "Embeddings-通过id查询")
@Operation(summary="Embeddings-通过id查询")
@GetMapping(value = "/queryById")
public Result<Embeddings> queryById(@RequestParam(name="id",required=true) String id) {
return Result.OK("批量删除成功!");
}
/**
* 通过id查询
*
* @param id
* @return
*/
//@AutoLog(value = "Embeddings-通过id查询")
@Operation(summary = "Embeddings-通过id查询")
@GetMapping(value = "/queryById")
public Result<Embeddings> queryById(@RequestParam(name = "id", required = true) String id) {
// Embeddings Embeddings = embeddingsService.getById(id);
// if(Embeddings==null) {
// return Result.error("未找到对应数据");
// }
return Result.OK();
}
embeddingsService.findById(id);
return Result.OK();
}
/**
* 导出excel
*
* @param request
* @param Embeddings
*/
* 导出excel
*
* @param request
* @param Embeddings
*/
@RequiresPermissions("Embeddings:Embeddings:exportXls")
@RequestMapping(value = "/exportXls")
public ModelAndView exportXls(HttpServletRequest request, Embeddings Embeddings) {
... ... @@ -199,16 +178,118 @@ public class EmbeddingsController {
}
/**
* 通过excel导入数据
*
* @param request
* @param response
* @return
*/
* 通过excel导入数据
*
* @param request
* @param response
* @return
*/
@RequiresPermissions("Embeddings:Embeddings:importExcel")
@RequestMapping(value = "/importExcel", method = RequestMethod.POST)
public Result<?> importExcel(HttpServletRequest request, HttpServletResponse response) {
return null;
}
// EmbeddingsController.java 新增方法
@AutoLog(value = "Embeddings-导入Word文档")
@Operation(summary = "Embeddings-导入Word文档")
@RequiresPermissions("embeddings:embeddings:importWord")
@PostMapping(value = "/importWord")
public Result<Map<String, Object>> importWord(@RequestParam(value = "file", required = false) MultipartFile file) {
try {
// 1. 验证文件
if (file.isEmpty()) {
return Result.error("上传文件为空");
}
String fileName = file.getOriginalFilename();
String fileExtension = fileName.substring(fileName.lastIndexOf(".") + 1).toLowerCase();
if (!"doc".equals(fileExtension) && !"docx".equals(fileExtension)) {
return Result.error("仅支持.doc和.docx格式的Word文档");
}
if (fileName != null) {
// 去掉后缀
int dotIndex = fileName.lastIndexOf('.');
fileName = fileName.substring(0, dotIndex);
}
// 2. 解析Word文档
String fullContent = readFullWordContent(file);
if (fullContent.isEmpty()) {
return Result.error("文档内容为空，无法导入");
}
// 3. 准备元数据
SnowflakeGenerator snowflakeGenerator = new SnowflakeGenerator();
String knowledgeId = String.valueOf(snowflakeGenerator.next());
String docId = String.valueOf(snowflakeGenerator.next());
String docName = fileName;
Map<String, Object> metadata = new HashMap<>();
metadata.put("knowledgeId", knowledgeId);
metadata.put("docName", docName);
metadata.put("docId", docId);
// 4. 创建并保存Embeddings对象
Embeddings embeddings = new Embeddings();
embeddings.setText(fullContent);
embeddings.setId(UUID.randomUUID().toString());
embeddings.setMetadata(metadata);
// 设置向量嵌入（实际项目中应调用模型API）
embeddings.setEmbedding(generateRandomEmbedding());
// 保存到数据库
embeddingsService.insert(embeddings);
return Result.OK("导入成功");
} catch (Exception e) {
log.error("Word导入失败", e);
return Result.error("导入失败: " + e.getMessage());
}
}
// 读取整个Word内容作为单个字符串
private String readFullWordContent(MultipartFile file) throws Exception {
try (XWPFDocument doc = new XWPFDocument(file.getInputStream())) {
StringBuilder fullContent = new StringBuilder();
// 读取所有段落文本
for (XWPFParagraph p : doc.getParagraphs()) {
fullContent.append(p.getText()).append("\n");
}
// 读取所有表格文本
for (XWPFTable table : doc.getTables()) {
for (XWPFTableRow row : table.getRows()) {
for (XWPFTableCell cell : row.getTableCells()) {
fullContent.append(cell.getText()).append("\t");
}
fullContent.append("\n");
}
}
return fullContent.toString();
}
}
// 生成随机向量（1536维）
private float[] generateRandomEmbedding() {
float[] embedding = new float[768];
Random random = new Random();
for (int i = 0; i < embedding.length; i++) {
embedding[i] = random.nextFloat() * 2 - 1;
}
return embedding;
}
}
... ...
... ... @@ -19,4 +19,11 @@ public class Embeddings {
private Map<String, Object> metadata;
private Double similarity;
// 添加 docName 字段（仅用于接收前端数据）
private String docName;
private String knowledgeId; // 新增知识库ID字段
private String docId; // 新增文档ID字段
private String index; // 新增索引位置字段
}
\ No newline at end of file
... ...
package org.jeecg.modules.airag.app.mapper;
import cn.hutool.core.lang.generator.SnowflakeGenerator;
import com.alibaba.fastjson2.JSONObject;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.pgvector.PGvector;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.shiro.dao.DataAccessException;
import org.jeecg.modules.airag.app.entity.Embeddings;
import org.postgresql.util.PGobject;
import org.springframework.stereotype.Component;
import java.sql.*;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.*;
@Component
@Slf4j
... ... @@ -31,9 +30,16 @@ public class PgVectorMapper {
}
// 查询所有向量记录
public List<Embeddings> findAll() {
public List<Embeddings> findAll(Embeddings embeddings) {
List<Embeddings> results = new ArrayList<>();
String sql = "SELECT * FROM embeddings";
if (StringUtils.isNotBlank(embeddings.getId())){
sql += " WHERE embedding_id = '" + embeddings.getId() + "'";
}
if(StringUtils.isNotBlank(embeddings.getText())){
sql += " where text = '" + embeddings.getText() + "'";
}
try (Connection conn = getConnection();
PreparedStatement stmt = conn.prepareStatement(sql);
... ... @@ -50,13 +56,13 @@ public class PgVectorMapper {
}
// 根据ID查询单个向量记录
public Embeddings findById(Long id) {
String sql = "SELECT * FROM embeddings WHERE id = ?";
public Embeddings findById(String id) {
String sql = "SELECT * FROM embeddings WHERE embedding_id = ?";
try (Connection conn = getConnection();
PreparedStatement stmt = conn.prepareStatement(sql)) {
stmt.setLong(1, id);
stmt.setString(1, id);
try (ResultSet rs = stmt.executeQuery()) {
if (rs.next()) {
return mapRowToEmbeddings(rs);
... ... @@ -71,14 +77,30 @@ public class PgVectorMapper {
// 插入新向量记录
public int insert(Embeddings record) {
String sql = "INSERT INTO embeddings (id, embedding, metadata) VALUES (?, ?, ?::jsonb)";
/* Map<String, Object> metadata = new LinkedHashMap<>();
// 按固定顺序添加字段
metadata.put("docId", UUID.randomUUID().toString());
metadata.put("knowledgeId", getKnowledgeId(record)); // 使用统一方法获取
metadata.put("docName", record.getDocName());
metadata.put("index", 0); // 确保是整数
record.setMetadata(metadata);
// 自动生成向量（这里需要调用嵌入模型）
float[] embedding = generateEmbedding(record.getText());
record.setEmbedding(embedding);*/
String sql = "INSERT INTO embeddings (embedding_id, embedding, text, metadata) VALUES (?, ?, ?, ?::jsonb)";
try (Connection conn = getConnection();
PreparedStatement stmt = conn.prepareStatement(sql)) {
stmt.setString(1, record.getId());
stmt.setString(1, UUID.randomUUID().toString());
// stmt.setObject(2, new PGvector(record.getEmbedding()));
stmt.setObject(2, new PGvector(record.getEmbedding()));
stmt.setString(3, toJson(record.getMetadata()));
stmt.setObject(3, record.getText());
stmt.setObject(4, toJson(record.getMetadata()));
return stmt.executeUpdate();
} catch (SQLException e) {
... ... @@ -89,14 +111,27 @@ public class PgVectorMapper {
// 更新向量记录
public int update(Embeddings record) {
String sql = "UPDATE embeddings SET embedding = ?, metadata = ?::jsonb WHERE id = ?";
String sql = "UPDATE embeddings SET embedding = ?, metadata = ?, text = ?::jsonb WHERE embedding_id = ?";
try (Connection conn = getConnection();
PreparedStatement stmt = conn.prepareStatement(sql)) {
JSONObject mataData = new JSONObject();
SnowflakeGenerator snowflakeGenerator = new SnowflakeGenerator();
String knowledgeId = String.valueOf(snowflakeGenerator.next());
mataData.put("knowledgeId", knowledgeId); // 使用前端传入的知识库ID
mataData.put("docName", record.getDocName());
String docId = String.valueOf(snowflakeGenerator.next());
mataData.put("docId", docId); // 自动生成唯一文档ID
mataData.put("index", "0");
PGobject jsonObject = new PGobject();
jsonObject.setType("json");
jsonObject.setValue(mataData.toJSONString());
stmt.setObject(1, new PGvector(record.getEmbedding()));
stmt.setString(2, toJson(record.getMetadata()));
stmt.setString(3, record.getId());
stmt.setObject(2, jsonObject);
stmt.setObject(3, record.getText());
stmt.setString(4, record.getId());
return stmt.executeUpdate();
} catch (SQLException e) {
... ... @@ -106,13 +141,13 @@ public class PgVectorMapper {
}
// 根据ID删除向量记录
public int deleteById(Long id) {
String sql = "DELETE FROM embeddings WHERE id = ?";
public int deleteById(String id) {
String sql = "DELETE FROM embeddings WHERE embedding_id = ?";
try (Connection conn = getConnection();
PreparedStatement stmt = conn.prepareStatement(sql)) {
stmt.setLong(1, id);
stmt.setString(1, id);
return stmt.executeUpdate();
} catch (SQLException e) {
log.error("删除向量记录失败, ID: {}", id, e);
... ... @@ -176,4 +211,31 @@ public class PgVectorMapper {
return Collections.emptyMap();
}
}
/*// 自动生成嵌入向量的方法（需根据您的嵌入模型实现）
private float[] generateEmbedding(String text) {
// 改为生成 768 维向量
float[] embedding = new float[768]; // OpenAI 标准维度是 1536，这里改为 768
// 实际项目中应调用嵌入模型 API
// 例如：return embeddingClient.generate(text, 768);
// 临时实现：生成随机向量（仅用于演示）
log.warn("使用随机向量生成 - 实际项目中应替换为真实模型调用");
Random random = new Random();
for (int i = 0; i < embedding.length; i++) {
embedding[i] = random.nextFloat() * 2 - 1;
}
return embedding;
}
private String getKnowledgeId(Embeddings record) {
// 1. 优先使用前端传入的knowledgeId（如果存在）
if (record.getMetadata() != null && record.getMetadata().containsKey("knowledgeId")) {
return String.valueOf(record.getMetadata().get("knowledgeId"));
}
// 2. 使用配置的默认值
return "default_knowledge"; // 实际应从配置读取
}*/
}
\ No newline at end of file
... ...
package org.jeecg.modules.airag.app.service;
//import org.jeecg.modules.demo.test.entity.Test;
import org.jeecg.modules.airag.app.entity.Test;
import com.baomidou.mybatisplus.extension.service.IService;
import org.jeecg.modules.airag.app.entity.Embeddings;
import java.util.List;
... ... @@ -14,7 +12,11 @@ import java.util.List;
* @Date: 2025-05-26
* @Version: V1.0
*/
public interface IEmbeddingsService {
public interface IEmbeddingsService {
List<Embeddings> findAll();
List<Embeddings> findAll(Embeddings embeddings);
int deleteById(String id);
int insert(Embeddings record);
int update(Embeddings record);
Embeddings findById(String id);
}
... ...
package org.jeecg.modules.airag.app.service.impl;
import org.jeecg.modules.airag.app.entity.Test;
import org.jeecg.modules.airag.app.entity.Embeddings;
import org.jeecg.modules.airag.app.mapper.EmbeddingsMapper;
import org.jeecg.modules.airag.app.mapper.PgVectorMapper;
import org.jeecg.modules.airag.app.service.IEmbeddingsService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import java.util.List;
/**
... ... @@ -22,8 +18,27 @@ import java.util.List;
public class IEmbeddingsServiceImpl implements IEmbeddingsService {
@Autowired
private PgVectorMapper pgVectorMapper;
@Override
public List<Embeddings> findAll(Embeddings embeddings) {
return pgVectorMapper.findAll(embeddings);
}
public int deleteById(String id) {
return pgVectorMapper.deleteById(id);
}
public int insert(Embeddings record) {
return pgVectorMapper.insert(record);
}
@Override
public int update(Embeddings record) {
return pgVectorMapper.update(record);
}
@Override
public List<Embeddings> findAll() {
return pgVectorMapper.findAll();
public Embeddings findById(String id) {
return pgVectorMapper.findById(id);
}
}
... ...