作者 dong

知识库列表没有进行分页,问题库列表没有及进行分页处理,存入问题库增加重复校验,日志管理增加回答方式,是否存入问题库,五个需要已实现

正在显示 13 个修改的文件 包含 212 行增加35 行删除
... ... @@ -12,8 +12,11 @@ import org.jeecg.common.api.vo.Result;
import org.jeecg.common.aspect.annotation.AutoLog;
import org.jeecg.common.system.base.controller.JeecgController;
import org.jeecg.common.system.query.QueryGenerator;
import org.jeecg.modules.airag.app.config.DataSourceConfig;
import org.jeecg.modules.airag.app.entity.AiragLog;
import org.jeecg.modules.airag.app.entity.QuestionEmbedding;
import org.jeecg.modules.airag.app.service.IAiragLogService;
import org.jeecg.modules.airag.app.service.IQuestionEmbeddingService;
import org.jeecg.modules.airag.llm.entity.AiragKnowledge;
import org.jeecg.modules.airag.llm.entity.AiragModel;
import org.jeecg.modules.airag.llm.service.IAiragModelService;
... ... @@ -24,10 +27,7 @@ import org.springframework.web.servlet.ModelAndView;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.sql.SQLException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.*;
/**
* @Description: 日志管理
... ... @@ -45,7 +45,13 @@ public class AiragLogController extends JeecgController<AiragLog, IAiragLogServi
@Autowired
private IAiragModelService airagModelService;
/**
@Autowired
private DataSourceConfig dataSourceConfig;
@Autowired
private IQuestionEmbeddingService questionEmbeddingService;
/**
* 分页列表查询
*
* @param airagLog
... ... @@ -61,6 +67,8 @@ public class AiragLogController extends JeecgController<AiragLog, IAiragLogServi
@RequestParam(name="pageNo", defaultValue="1") Integer pageNo,
@RequestParam(name="pageSize", defaultValue="10") Integer pageSize,
HttpServletRequest req) {
QueryWrapper<AiragLog> queryWrapper = QueryGenerator.initQueryWrapper(airagLog, req.getParameterMap());
Page<AiragLog> page = new Page<AiragLog>(pageNo, pageSize);
// IPage<AiragLog> pageList = airagLogService.page(page, queryWrapper);
... ... @@ -127,6 +135,15 @@ public class AiragLogController extends JeecgController<AiragLog, IAiragLogServi
@RequiresPermissions("airaglog:airag_log:saveToQuestionLibrary")
@PostMapping(value = "/saveToQuestionLibrary")
public Result<String> saveToQuestionLibrary(@RequestBody AiragLog airagLog) {
QuestionEmbedding questionEmbedding = new QuestionEmbedding();
questionEmbedding.setQuestion(airagLog.getQuestion());
List<QuestionEmbedding> list = questionEmbeddingService.findQuestion(questionEmbedding);
for (QuestionEmbedding questionEmbedding1 : list){
if(questionEmbedding1.getQuestion().equals(airagLog.getQuestion())){
return Result.error("问题库已有相同问题,不可存入");
}
}
airagLog.setIsStorage(1);
airagLogService.saveToQuestionLibrary(airagLog);
return Result.OK("存入问题库成功!");
}
... ...
... ... @@ -59,19 +59,14 @@ public class EmbeddingsController {
//@AutoLog(value = "Embeddings-分页列表查询")
@Operation(summary = "Embeddings-分页列表查询")
@GetMapping(value = "/list")
public Result<IPage<Embeddings>> queryPageList(Embeddings embeddings,
public Result<Page<Embeddings>> queryPageList(Embeddings embeddings,
@RequestParam(name = "pageNo", defaultValue = "1") Integer pageNo,
@RequestParam(name = "pageSize", defaultValue = "10") Integer pageSize,
HttpServletRequest req) throws NoSuchFieldException, IllegalAccessException, SQLException {
//Response<Embedding> embedding = aiModelUtils.getEmbedding("1925730210204721154", "33333");
List<Embeddings> records = embeddingsService.findAll(embeddings);
for (Embeddings record : records) {
System.out.println("record = " + record);
}
Page<Embeddings> page = new Page<Embeddings>(pageNo, pageSize);
page.setRecords(records);
return Result.OK(page);
Page<Embeddings> records = embeddingsService.findAll(embeddings,pageNo,pageSize);
return Result.OK(records);
}
/**
* 查询知识库ID
... ...
package org.jeecg.modules.airag.app.controller;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import dev.langchain4j.internal.Json;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.jeecg.common.api.vo.Result;
import org.jeecg.modules.airag.app.entity.AiragLog;
import org.jeecg.modules.airag.app.entity.QuestionEmbedding;
import org.jeecg.modules.airag.app.service.IQuestionEmbeddingService;
import org.jeecg.modules.airag.app.utils.JsonUtils;
import org.jeecg.modules.airag.llm.entity.AiragKnowledge;
import org.jeecg.modules.airag.llm.entity.AiragModel;
import org.jeecg.modules.airag.llm.service.IAiragKnowledgeService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.transaction.annotation.Transactional;
... ... @@ -17,7 +16,6 @@ import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
... ... @@ -31,13 +29,13 @@ public class QuestionEmbeddingController {
private IAiragKnowledgeService airagKnowledgeService;
@GetMapping("/list")
public Result<List<QuestionEmbedding>> findAll(QuestionEmbedding questionEmbedding) {
List<QuestionEmbedding> list = questionEmbeddingService.findAll(questionEmbedding);
public Result<Page<QuestionEmbedding>> findAll(QuestionEmbedding questionEmbedding, @RequestParam(defaultValue = "1") int pageNo, @RequestParam(defaultValue = "10") int pageSize) {
Page<QuestionEmbedding> page = questionEmbeddingService.findAll(questionEmbedding,pageNo,pageSize);
Map<String, String> airagKnowledgeMap = airagKnowledgeService.list()
.stream()
.collect(Collectors.toMap(AiragKnowledge::getId, AiragKnowledge::getName));
list.forEach(item -> {
page.getRecords().forEach(item -> {
String metadata = item.getMetadata();
if (StringUtils.isNotBlank(metadata)) {
Map<String, String> jsonMap = JsonUtils.jsonUtils(metadata);
... ... @@ -48,8 +46,7 @@ public class QuestionEmbeddingController {
}
});
return Result.OK(list);
return Result.OK(page);
}
@GetMapping("/queryById")
... ...
... ... @@ -77,8 +77,32 @@ public class AiragLog implements Serializable {
@Excel(name = "模型ID", width = 15)
@Schema(description = "模型ID")
private String modelId;
/**
* 回答方式
*/
@Excel(name = "回答方式", width = 15)
@TableField("answer_type")
@Schema(description = "回答方式:1:问题库回答 2:模型回答 3:未命中")
private int answerType;
/**
* 回答方式
*/
@Excel(name = "是否存入问题库", width = 15)
@TableField("is_storage")
@Schema(description = "是否存入问题库 0:否 1:是")
private int isStorage;
// 新增:临时字段(非数据库字段)
@TableField(exist = false) // MyBatis-Plus 标记该字段不存在于数据库表中
private String name;
// 新增:临时字段(非数据库字段)
@TableField(exist = false) // MyBatis-Plus 标记该字段不存在于数据库表中
private String createTime_begin;
// 新增:临时字段(非数据库字段)
@TableField(exist = false) // MyBatis-Plus 标记该字段不存在于数据库表中
private String createTime_end;
@TableField(exist = false) // MyBatis-Plus 标记该字段不存在于数据库表中
private String createTimeStr;
}
... ...
... ... @@ -18,4 +18,7 @@ public interface AiragLogMapper extends BaseMapper<AiragLog> {
IPage<AiragLog> list1(@Param("param1")AiragLog airagLog, Page<AiragLog> page);
IPage<AiragLog> pageList(@Param("param1") AiragLog airagLog, Page<AiragLog> page);
int updataIsStorage(@Param("param1") int isStorage);
}
... ...
... ... @@ -3,6 +3,7 @@ package org.jeecg.modules.airag.app.mapper;
import ch.qos.logback.core.net.SyslogOutputStream;
import cn.hutool.core.lang.generator.SnowflakeGenerator;
import com.alibaba.fastjson2.JSONObject;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
... ... @@ -31,22 +32,32 @@ public class PgVectorMapper {
}
// 查询所有向量记录
public List<Embeddings> findAll(Embeddings embeddings) {
public Page<Embeddings> findAll(Embeddings embeddings,int pageNo,int pageSize) {
List<Embeddings> results = new ArrayList<>();
StringBuilder sql = new StringBuilder("SELECT * FROM embeddings WHERE 1=1");
StringBuilder countSql = new StringBuilder("SELECT COUNT(1) FROM embeddings WHERE 1=1");
List<Object> params = new ArrayList<>(); // 存储参数值
List<Object> countParams = new ArrayList<>(); // 存储参数值
// 动态构建查询条件
if (StringUtils.isNotBlank(embeddings.getKnowledgeId())) {
sql.append(" AND metadata ->> 'knowledgeId' = ?");
countSql.append(" AND metadata ->> 'knowledgeId' = ?");
params.add(embeddings.getKnowledgeId());
countParams.add(embeddings.getKnowledgeId());
}
if (StringUtils.isNotBlank(embeddings.getText())) {
sql.append(" AND text ILIKE ?"); // 使用 ILIKE 进行不区分大小写的模糊匹配
countSql.append(" AND text ILIKE ?"); // 使用 ILIKE 进行不区分大小写的模糊匹配
params.add("%" + embeddings.getText() + "%");
countParams.add("%" + embeddings.getText() + "%");
}
// 添加分页
sql.append(" LIMIT ? OFFSET ?");
params.add(pageSize);
params.add((pageNo - 1) * pageSize);
System.out.println("SQL: " + sql.toString());
... ... @@ -67,7 +78,31 @@ public class PgVectorMapper {
log.error("查询所有向量记录失败", e);
throw new RuntimeException("查询向量数据时发生数据库错误", e);
}
return results;
// 执行计数查询
int total = 0;
try(Connection conn = getConnection();
PreparedStatement stmt = conn.prepareStatement(countSql.toString())){
// 设置参数值
for (int i = 0; i < countParams.size(); i++) {
stmt.setObject(i + 1, countParams.get(i));
}
try (ResultSet rs = stmt.executeQuery()) {
if (rs.next()) {
total = rs.getInt(1); // 直接获取count值
}
}
} catch (SQLException e) {
log.error("查询记录总数失败", e);
throw new RuntimeException("查询记录总数时发生数据库错误", e);
}
Page<Embeddings> page = new Page<>();
page.setRecords(results);
page.setTotal(total);
return page;
}
// 根据ID查询单个向量记录
... ...
... ... @@ -2,6 +2,7 @@ package org.jeecg.modules.airag.app.mapper;
import cn.hutool.core.lang.generator.SnowflakeGenerator;
import com.alibaba.fastjson2.JSONObject;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
... ... @@ -38,22 +39,93 @@ public class QuestionEmbeddingMapper {
}
// 查询所有记录
public List<QuestionEmbedding> findAll(QuestionEmbedding questionEmbedding) {
public Page<QuestionEmbedding> findAll(QuestionEmbedding questionEmbedding, int pageNo, int pageSize) {
List<QuestionEmbedding> results = new ArrayList<>();
StringBuilder sql = new StringBuilder("select * from question_embedding where 1 = 1");
StringBuilder countSql = new StringBuilder("select count(1) from question_embedding where 1 = 1");
List<Object> params = new ArrayList<>();
List<Object> countParams = new ArrayList<>();
if (StringUtils.isNotBlank(questionEmbedding.getKnowledgeId())) {
sql.append(" AND metadata ->> 'knowledgeId' = ?");
countSql.append(" AND metadata ->> 'knowledgeId' = ?");
params.add(questionEmbedding.getKnowledgeId());
countParams.add(questionEmbedding.getKnowledgeId());
}
if(StringUtils.isNotBlank(questionEmbedding.getQuestion())){
sql.append(" and question like '%").append(questionEmbedding.getQuestion()).append("%'");
sql.append(" AND question ILIKE ?"); // 使用 ILIKE 进行不区分大小写的模糊匹配
countSql.append(" AND question ILIKE ?"); // 使用 ILIKE 进行不区分大小写的模糊匹配
params.add("%" + questionEmbedding.getQuestion() + "%");
countParams.add("%" + questionEmbedding.getQuestion() + "%");
}
if(StringUtils.isNotBlank(questionEmbedding.getAnswer())){
sql.append(" and answer like '%").append(questionEmbedding.getAnswer()).append("%'");
sql.append(" AND answer ILIKE ?"); // 使用 ILIKE 进行不区分大小写的模糊匹配
countSql.append(" AND answer ILIKE ?"); // 使用 ILIKE 进行不区分大小写的模糊匹配
params.add("%" + questionEmbedding.getAnswer() + "%");
countParams.add("%" + questionEmbedding.getAnswer() + "%");
}
// 添加分页
sql.append(" LIMIT ? OFFSET ?");
params.add(pageSize);
params.add((pageNo - 1) * pageSize);
try(Connection conn = getConnection();
PreparedStatement stmt = conn.prepareStatement(sql.toString())){
// 设置参数值
for (int i = 0; i < params.size(); i++) {
stmt.setObject(i + 1, params.get(i));
}
try (ResultSet rs = stmt.executeQuery()) {
while (rs.next()) {
results.add(mapRowToQuestionEmbedding(rs));
}
}
} catch (SQLException e) {
log.error("查询所有记录失败", e);
throw new RuntimeException("查询数据时发生数据库错误", e);
}
// 执行计数查询
long total = 0;
try(Connection conn = getConnection();
PreparedStatement stmt = conn.prepareStatement(countSql.toString())){
// 设置参数值
for (int i = 0; i < countParams.size(); i++) {
stmt.setObject(i + 1, countParams.get(i));
}
try (ResultSet rs = stmt.executeQuery()) {
if (rs.next()) {
total = rs.getLong(1); // 直接获取count值
}
}
} catch (SQLException e) {
log.error("查询记录总数失败", e);
throw new RuntimeException("查询记录总数时发生数据库错误", e);
}
Page<QuestionEmbedding> page = new Page<>();
page.setRecords(results);
page.setTotal(total);
return page;
}
// 查询所有记录
public List<QuestionEmbedding> findQuestion(QuestionEmbedding questionEmbedding) {
List<QuestionEmbedding> results = new ArrayList<>();
StringBuilder sql = new StringBuilder("select * from question_embedding where 1 = 1");
List<Object> params = new ArrayList<>();
if(StringUtils.isNotBlank(questionEmbedding.getQuestion())){
sql.append(" AND question = ?"); // 使用 ILIKE 进行不区分大小写的模糊匹配
params.add(questionEmbedding.getQuestion());
}
try(Connection conn = getConnection();
PreparedStatement stmt = conn.prepareStatement(sql.toString())){
// 设置参数值
... ... @@ -70,6 +142,8 @@ public class QuestionEmbeddingMapper {
log.error("查询所有记录失败", e);
throw new RuntimeException("查询数据时发生数据库错误", e);
}
return results;
}
... ...
... ... @@ -9,13 +9,35 @@
</select>
<select id="pageList" resultType="org.jeecg.modules.airag.app.entity.AiragLog">
select * from airag_log
select
id, create_by, create_time,
update_by, update_time, question,
answer, model_id, answer_type, is_storage
from airag_log
where 1=1
<if test="param1.isStorage != null and param1.isStorage != -1">
and is_storage = #{param1.isStorage}
</if>
<if test="param1.name != '' and param1.name != null">
and model_id = #{param1.name}
</if>
<if test="param1.question != '' and param1.question != null">
and question LIKE CONCAT('%', #{param1.question}, '%')
</if>
<if test="param1.answerType != '' and param1.answerType != null">
and answer_type = #{param1.answerType}
</if>
<if test="param1.createTime_begin != '' and param1.createTime_begin != null and param1.createTime_end != null and param1.createTime_end != null">
and create_time between #{param1.createTime_begin} and #{param1.createTime_end}
</if>
</select>
<update id="updataIsStorage">
update airag_log
set is_storage = #{isStorage}
</update>
</mapper>
\ No newline at end of file
... ...
... ... @@ -2,6 +2,7 @@ package org.jeecg.modules.airag.app.service;
//import org.jeecg.modules.demo.test.entity.Test;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import org.jeecg.modules.airag.app.entity.Embeddings;
import java.util.List;
... ... @@ -14,7 +15,7 @@ import java.util.List;
*/
public interface IEmbeddingsService {
List<Embeddings> findAll(Embeddings embeddings);
Page<Embeddings> findAll(Embeddings embeddings,int pageNo,int pageSize);
int deleteById(String id);
int insert(Embeddings record);
int update(Embeddings record);
... ...
package org.jeecg.modules.airag.app.service;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import org.jeecg.common.api.vo.Result;
import org.jeecg.modules.airag.app.entity.QuestionEmbedding;
import org.springframework.web.multipart.MultipartFile;
... ... @@ -7,7 +8,8 @@ import org.springframework.web.multipart.MultipartFile;
import java.util.List;
public interface IQuestionEmbeddingService {
List<QuestionEmbedding> findAll(QuestionEmbedding questionEmbedding);
Page<QuestionEmbedding> findAll(QuestionEmbedding questionEmbedding, Integer pageNo, Integer pageSize);
List<QuestionEmbedding> findQuestion(QuestionEmbedding questionEmbedding);
QuestionEmbedding findById(String id);
int insert(QuestionEmbedding record);
int update(QuestionEmbedding record);
... ...
... ... @@ -56,9 +56,9 @@ public class AiragLogServiceImpl extends ServiceImpl<AiragLogMapper, AiragLog> i
QuestionEmbedding questionEmbedding = new QuestionEmbedding();
questionEmbedding.setQuestion(log.getQuestion());
questionEmbedding.setAnswer(log.getAnswer());
questionEmbeddingMapper.insert(questionEmbedding);
airagLogMapper.updataIsStorage(log.getIsStorage());
}
@Override
... ...
package org.jeecg.modules.airag.app.service.impl;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import org.jeecg.modules.airag.app.entity.Embeddings;
import org.jeecg.modules.airag.app.mapper.PgVectorMapper;
import org.jeecg.modules.airag.app.service.IEmbeddingsService;
... ... @@ -20,8 +21,8 @@ public class IEmbeddingsServiceImpl implements IEmbeddingsService {
private PgVectorMapper pgVectorMapper;
@Override
public List<Embeddings> findAll(Embeddings embeddings) {
return pgVectorMapper.findAll(embeddings);
public Page<Embeddings> findAll(Embeddings embeddings, int pageNo, int pageSize) {
return pgVectorMapper.findAll(embeddings,pageNo,pageSize);
}
public int deleteById(String id) {
... ...
package org.jeecg.modules.airag.app.service.impl;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import org.apache.poi.hwpf.usermodel.CharacterRun;
import org.apache.poi.hwpf.HWPFDocument;
import org.apache.poi.hwpf.usermodel.Paragraph;
... ... @@ -64,8 +65,13 @@ public class QuestionEmbeddingServiceImpl implements IQuestionEmbeddingService {
private static final Pattern UUID_PATTERN = Pattern.compile("_[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}");
@Override
public List<QuestionEmbedding> findAll(QuestionEmbedding questionEmbedding) {
return questionEmbeddingMapper.findAll(questionEmbedding);
public Page<QuestionEmbedding> findAll(QuestionEmbedding questionEmbedding, Integer pageNo, Integer pageSize) {
return questionEmbeddingMapper.findAll(questionEmbedding,pageNo,pageSize);
}
@Override
public List<QuestionEmbedding> findQuestion(QuestionEmbedding questionEmbedding) {
return questionEmbeddingMapper.findQuestion(questionEmbedding);
}
@Override
... ...