Compare commits

..

13 Commits

Author SHA1 Message Date
xiongxiaoyang
84a90bbc34 v5.2.3 发布 2025-07-19 18:22:25 +08:00
xiongxiaoyang
b2d8fd8c66 feat(AI): 更新默认AI对话模型
- 原默认AI对话模型 deepseek-ai/DeepSeek-R1-Distill-Llama-8B 已从硅基流动模型广场下线。
- 此次更新将项目中使用的默认AI对话模型更改为新的可用模型 deepseek-ai/DeepSeek-R1-0528-Qwen3-8B。
2025-07-19 17:27:45 +08:00
xiongxiaoyang
11d9d6f6e8 refactor: 重构排序参数处理代码 2025-07-19 17:27:21 +08:00
xiongxiaoyang
8c7b891af2 build(AI): Spring AI 升级到 1.0.0 2025-07-18 21:24:24 +08:00
xiongxiaoyang
bb1a87e337 chore(sql): 内置海外专用源 2025-07-18 20:41:00 +08:00
xiongxiaoyang
1cd8a49fd4 perf: 优化排序参数校验 2025-07-18 16:21:35 +08:00
xiongxiaoyang
773ce159f7 v5.2.2 发布 2025-07-17 21:14:24 +08:00
xiongxiaoyang
91e7d2712b refactor: 重构sort和order参数校验功能 2025-07-17 20:53:41 +08:00
xiongxiaoyang
3db8828384 fix: 修复sort和order参数的SQL注入漏洞 2025-07-17 19:03:58 +08:00
xiongxiaoyang
54bd194b98 feat(novel-crawl): 增加爬虫源采集章节数量监控功能
可以监测到爬虫源在当前环境下是否可用
2025-07-16 19:52:07 +08:00
xiongxiaoyang
3d41cf3ebb perf(novel-crawl): 优化爬虫源列表排序
按照更新时间倒序
2025-07-15 18:53:31 +08:00
xiongxiaoyang
720711414c v5.2.1 发布 2025-07-14 22:00:34 +08:00
xiongxiaoyang
522bb7c739 fix(novel-front): 修复评论回复中的XSS漏洞 2025-07-14 21:02:13 +08:00
47 changed files with 337 additions and 82 deletions

View File

@@ -81,7 +81,7 @@ novel-plus 5.x 已集成 Spring 官方最新发布的 Spring AI 框架,并推
我们将持续关注 AI 技术的发展并致力于将其与小说创作场景深度融合为用户带来更智能更便捷的创作工具
novel-plus 项目默认使用的是第三方大模型服务平台[硅基流动](https://cloud.siliconflow.cn/i/DOgMRH9S)提供的 API兼容 OpenAI 的相关接口,可直接通过 Spring AI 框架调用),采用的 AI 模型有对话模型`deepseek-ai/DeepSeek-R1-Distill-Llama-8B`DeepSeek-R1 的蒸馏版本,免费使用)和生图模型`Kwai-Kolors/Kolors`(快手 Kolors 团队开发的文本到图像生成模型,免费使用)。只需注册一个硅基流动账号,创建一个 API 密钥,并将其添加到 novel-plus 项目 novel-front 模块的 yaml 配置文件中,即可体验 novel-plus 项目的 AI 写作功能。
novel-plus 项目默认使用的是第三方大模型服务平台[硅基流动](https://cloud.siliconflow.cn/i/DOgMRH9S)提供的 API兼容 OpenAI 的相关接口,可直接通过 Spring AI 框架调用),采用的 AI 模型有对话模型`deepseek-ai/DeepSeek-R1-0528-Qwen3-8B`DeepSeek-R1 的蒸馏版本,免费使用)和生图模型`Kwai-Kolors/Kolors`(快手 Kolors 团队开发的文本到图像生成模型,免费使用)。只需注册一个硅基流动账号,创建一个 API 密钥,并将其添加到 novel-plus 项目 novel-front 模块的 yaml 配置文件中,即可体验 novel-plus 项目的 AI 写作功能。
```yaml
spring:
@@ -98,7 +98,7 @@ spring:
base-url: https://api.siliconflow.cn
chat:
options:
model: deepseek-ai/DeepSeek-R1-Distill-Llama-8B
model: deepseek-ai/DeepSeek-R1-0528-Qwen3-8B
```
novel-plus 项目默认使用的都是免费 AI 模型生成效果有限如果对生成内容有更高的要求建议选用付费的 AI 模型

View File

@@ -3174,4 +3174,43 @@ CREATE TABLE `book_comment_reply`
`create_user_id` bigint(20) DEFAULT NULL COMMENT '回复时间',
PRIMARY KEY (`id`)
) ENGINE = InnoDB
DEFAULT CHARSET = utf8mb4 COMMENT ='小说评论回复表';
DEFAULT CHARSET = utf8mb4 COMMENT ='小说评论回复表';
INSERT INTO crawl_source (source_name, crawl_rule, source_status, create_time
, update_time)
VALUES ('飘天文学网海外专用', '{
"bookListUrl": "https://www.piaotia.com/booksort{catId}/{page}.html",
"catIdRule": {
"catId1": "1/0",
"catId2": "2/0",
"catId3": "3/0",
"catId4": "4/0",
"catId5": "6/0",
"catId6": "5/0"
},
"bookIdPatten": "href=\\"https://www.piaotia.com/bookinfo/(\\\\d+/\\\\d+).html\\"",
"pagePatten": "<em\\\\s+id=\\"pagestats\\">(\\\\d+)/\\\\d+</em>",
"totalPagePatten": "<em\\\\s+id=\\"pagestats\\">\\\\d+/(\\\\d+)</em>",
"bookDetailUrl": "https://www.piaotia.com/bookinfo/{bookId}.html",
"bookNamePatten": "<h1>([^/]+)</h1>",
"authorNamePatten": "<td\\\\s+width=\\"\\\\d+%\\">作&nbsp;&nbsp;&nbsp; 者:([^/]+)<",
"picUrlPatten": "<img\\\\s+src=\\"(https://www.piaotia.com/files/article/image/[^\\"]+)\\"",
"statusPatten": "<td>文章状态:([^/]+)</td>",
"bookStatusRule": {
"连载中": 0,
"已完成": 1
},
"descStart": " <span class=\\"hottext\\">内容简介:</span><br />",
"descEnd": "</td>",
"filterDesc": "",
"bookIndexUrl": "https://www.piaotia.com/html/{bookId}/index.html",
"indexIdPatten": "<li><a href=\\"(\\\\d+).html\\">[^/]+</a></li>",
"indexNamePatten": "<li><a href=\\"\\\\d+.html\\">([^/]+)</a></li>",
"bookContentUrl": "https://www.piaotia.com/html/{bookId}/{indexId}.html",
"contentStart": "<br>",
"contentEnd": "</div>",
"filterContent": "",
"charset": "gbk"
}', 0, '2025-07-13 18:57:39'
, '2025-07-13 18:57:39');

View File

@@ -5,7 +5,7 @@
<groupId>com.java2nb</groupId>
<artifactId>novel-admin</artifactId>
<version>5.2.0</version>
<version>5.2.3</version>
<packaging>jar</packaging>
<name>novel-admin</name>

View File

@@ -0,0 +1,14 @@
package com.java2nb.common.annotation;
import java.lang.annotation.*;
/**
* 标记某个方法参数需要进行 Map 字段的清理和标准化处理。
*
* <p>通常用于 DAO 接口中 list 方法的 Map 参数,用于防止非法排序字段或排序顺序。</p>
*/
@Target(ElementType.PARAMETER)
@Retention(RetentionPolicy.RUNTIME)
@Documented
public @interface SanitizeMap {
}

View File

@@ -0,0 +1,69 @@
package com.java2nb.common.aspect;
import com.java2nb.common.annotation.SanitizeMap;
import com.java2nb.common.utils.SortWhitelistUtil;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import org.aspectj.lang.ProceedingJoinPoint;
import org.aspectj.lang.annotation.Around;
import org.aspectj.lang.annotation.Aspect;
import org.aspectj.lang.reflect.MethodSignature;
import org.springframework.stereotype.Component;
import java.lang.annotation.Annotation;
import java.lang.reflect.Method;
import java.util.Arrays;
import java.util.Map;
/**
* 拦截所有 Mapper 接口的 list* 方法,对带有 @SanitizeMap 注解的 Map 参数进行排序字段和顺序的规范化处理。
*
* <p>主要防止 SQL 注入或非法排序字段、非法排序顺序的问题。
* 例如对 sort 和 order 字段进行白名单过滤和标准化处理。</p>
*/
@Aspect
@Component
@RequiredArgsConstructor
public class MapSortValidationAspect {
/**
* 拦截所有 Mapper 接口的 list* 方法(如 list(), listByPage 等)。
* 对带有 @SanitizeMap 注解的 Map 参数进行处理。
*
* <p>执行逻辑:</p>
* <ol>
* <li>获取方法参数及注解信息</li>
* <li>遍历所有参数,检查是否带有 @SanitizeMap 注解</li>
* <li>如果参数是 Map 类型且有注解,则进行字段清理</li>
* </ol>
*
* @param joinPoint 切点信息
* @return 方法执行结果
*/
@SneakyThrows
@Around("execution(* com.java2nb.*.dao.*Dao.list*(..))")
public Object sanitizeMapParameters(ProceedingJoinPoint joinPoint) {
Object[] args = joinPoint.getArgs();
MethodSignature signature = (MethodSignature) joinPoint.getSignature();
Method method = signature.getMethod();
Annotation[][] parameterAnnotations = method.getParameterAnnotations();
for (int i = 0; i < parameterAnnotations.length; i++) {
boolean hasAnnotation = Arrays.stream(parameterAnnotations[i])
.anyMatch(a -> a.annotationType().equals(SanitizeMap.class));
if (hasAnnotation && args[i] instanceof Map map) {
if (map.get("sort") instanceof String sortStr) {
map.put("sort", SortWhitelistUtil.sanitizeColumn(sortStr));
}
if (map.get("order") instanceof String orderStr) {
map.put("order", SortWhitelistUtil.sanitizeOrder(orderStr));
}
}
}
return joinPoint.proceed(args);
}
}

View File

@@ -1,5 +1,6 @@
package com.java2nb.common.dao;
import com.java2nb.common.annotation.SanitizeMap;
import com.java2nb.common.domain.DictDO;
import java.util.List;
@@ -19,7 +20,7 @@ public interface DictDao {
DictDO get(Long id);
List<DictDO> list(Map<String, Object> map);
List<DictDO> list(@SanitizeMap Map<String, Object> map);
int count(Map<String, Object> map);

View File

@@ -1,5 +1,6 @@
package com.java2nb.common.dao;
import com.java2nb.common.annotation.SanitizeMap;
import com.java2nb.common.domain.FileDO;
import java.util.List;
@@ -18,7 +19,7 @@ public interface FileDao {
FileDO get(Long id);
List<FileDO> list(Map<String,Object> map);
List<FileDO> list(@SanitizeMap Map<String,Object> map);
int count(Map<String,Object> map);

View File

@@ -3,6 +3,7 @@ package com.java2nb.common.dao;
import java.util.List;
import java.util.Map;
import com.java2nb.common.annotation.SanitizeMap;
import com.java2nb.common.domain.GenColumnsDO;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
@@ -19,7 +20,7 @@ public interface GenColumnsDao {
GenColumnsDO get(Long id);
List<GenColumnsDO> list(Map<String,Object> map);
List<GenColumnsDO> list(@SanitizeMap Map<String,Object> map);
int count(Map<String,Object> map);

View File

@@ -1,5 +1,6 @@
package com.java2nb.common.dao;
import com.java2nb.common.annotation.SanitizeMap;
import com.java2nb.common.domain.LogDO;
import java.util.List;
@@ -18,7 +19,7 @@ public interface LogDao {
LogDO get(Long id);
List<LogDO> list(Map<String,Object> map);
List<LogDO> list(@SanitizeMap Map<String,Object> map);
int count(Map<String,Object> map);

View File

@@ -0,0 +1,43 @@
package com.java2nb.common.utils;
import lombok.experimental.UtilityClass;
import java.util.Set;
/**
* 排序字段和排序顺序的白名单工具类。
*/
@UtilityClass
public class SortWhitelistUtil {
// 白名单字段
private static final Set<String> ALLOWED_COLUMNS = Set.of("id", "name", "order_num");
// 白名单排序方式
private static final Set<String> ALLOWED_ORDERS = Set.of("asc", "desc");
/**
* 对排序字段进行白名单过滤和标准化。
*
* @param column 原始字段名
* @return 安全的字段名,若非法则返回 null
*/
public static String sanitizeColumn(String column) {
if (column == null) return null;
String lower = column.trim().toLowerCase();
return ALLOWED_COLUMNS.contains(lower) ? lower : null;
}
/**
* 对排序方式进行白名单过滤和标准化。
*
* @param order 原始排序方式
* @return 安全的排序方式("asc" 或 "desc"),若非法则返回 null
*/
public static String sanitizeOrder(String order) {
if (order == null) return null;
String lower = order.trim().toLowerCase();
return ALLOWED_ORDERS.contains(lower) ? lower : null;
}
}

View File

@@ -1,5 +1,7 @@
package com.java2nb.novel.dao;
import com.java2nb.common.annotation.SanitizeMap;
import com.java2nb.novel.domain.AuthorCodeDO;
import java.util.List;
@@ -18,7 +20,7 @@ public interface AuthorCodeDao {
AuthorCodeDO get(Long id);
List<AuthorCodeDO> list(Map<String,Object> map);
List<AuthorCodeDO> list(@SanitizeMap Map<String,Object> map);
int count(Map<String,Object> map);

View File

@@ -1,5 +1,6 @@
package com.java2nb.novel.dao;
import com.java2nb.common.annotation.SanitizeMap;
import com.java2nb.novel.domain.AuthorDO;
import java.util.Date;
@@ -19,7 +20,7 @@ public interface AuthorDao {
AuthorDO get(Long id);
List<AuthorDO> list(Map<String,Object> map);
List<AuthorDO> list(@SanitizeMap Map<String,Object> map);
int count(Map<String,Object> map);

View File

@@ -1,5 +1,7 @@
package com.java2nb.novel.dao;
import com.java2nb.common.annotation.SanitizeMap;
import com.java2nb.novel.domain.BookCommentDO;
import java.util.List;
@@ -18,7 +20,7 @@ public interface BookCommentDao {
BookCommentDO get(Long id);
List<BookCommentDO> list(Map<String,Object> map);
List<BookCommentDO> list(@SanitizeMap Map<String,Object> map);
int count(Map<String,Object> map);

View File

@@ -1,5 +1,7 @@
package com.java2nb.novel.dao;
import com.java2nb.common.annotation.SanitizeMap;
import com.java2nb.novel.domain.BookContentDO;
import org.apache.ibatis.annotations.Mapper;
@@ -18,7 +20,7 @@ public interface BookContentDao {
BookContentDO get(Long id);
List<BookContentDO> list(Map<String, Object> map);
List<BookContentDO> list(@SanitizeMap Map<String, Object> map);
int count(Map<String, Object> map);

View File

@@ -1,5 +1,7 @@
package com.java2nb.novel.dao;
import com.java2nb.common.annotation.SanitizeMap;
import com.java2nb.novel.domain.BookDO;
import org.apache.ibatis.annotations.Mapper;
@@ -19,7 +21,7 @@ public interface BookDao {
BookDO get(Long id);
List<BookDO> list(Map<String, Object> map);
List<BookDO> list(@SanitizeMap Map<String, Object> map);
int count(Map<String, Object> map);

View File

@@ -1,5 +1,7 @@
package com.java2nb.novel.dao;
import com.java2nb.common.annotation.SanitizeMap;
import com.java2nb.novel.domain.BookIndexDO;
import org.apache.ibatis.annotations.Mapper;
@@ -18,7 +20,7 @@ public interface BookIndexDao {
BookIndexDO get(Long id);
List<BookIndexDO> list(Map<String, Object> map);
List<BookIndexDO> list(@SanitizeMap Map<String, Object> map);
int count(Map<String, Object> map);

View File

@@ -1,5 +1,7 @@
package com.java2nb.novel.dao;
import com.java2nb.common.annotation.SanitizeMap;
import com.java2nb.novel.domain.BookSettingDO;
import java.util.List;
@@ -18,7 +20,7 @@ public interface BookSettingDao {
BookSettingDO get(Long id);
List<BookSettingDO> list(Map<String,Object> map);
List<BookSettingDO> list(@SanitizeMap Map<String,Object> map);
int count(Map<String,Object> map);

View File

@@ -1,5 +1,7 @@
package com.java2nb.novel.dao;
import com.java2nb.common.annotation.SanitizeMap;
import com.java2nb.novel.domain.CategoryDO;
import java.util.List;
@@ -18,7 +20,7 @@ public interface CategoryDao {
CategoryDO get(Integer id);
List<CategoryDO> list(Map<String,Object> map);
List<CategoryDO> list(@SanitizeMap Map<String,Object> map);
int count(Map<String,Object> map);

View File

@@ -1,5 +1,7 @@
package com.java2nb.novel.dao;
import com.java2nb.common.annotation.SanitizeMap;
import com.java2nb.novel.domain.FriendLinkDO;
import java.util.List;
@@ -18,7 +20,7 @@ public interface FriendLinkDao {
FriendLinkDO get(Integer id);
List<FriendLinkDO> list(Map<String,Object> map);
List<FriendLinkDO> list(@SanitizeMap Map<String,Object> map);
int count(Map<String,Object> map);

View File

@@ -1,5 +1,7 @@
package com.java2nb.novel.dao;
import com.java2nb.common.annotation.SanitizeMap;
import com.java2nb.novel.domain.NewsDO;
import java.util.List;
@@ -18,7 +20,7 @@ public interface NewsDao {
NewsDO get(Long id);
List<NewsDO> list(Map<String,Object> map);
List<NewsDO> list(@SanitizeMap Map<String,Object> map);
int count(Map<String,Object> map);

View File

@@ -1,5 +1,7 @@
package com.java2nb.novel.dao;
import com.java2nb.common.annotation.SanitizeMap;
import com.java2nb.novel.domain.PayDO;
import java.util.Date;
@@ -19,7 +21,7 @@ public interface PayDao {
PayDO get(Long id);
List<PayDO> list(Map<String,Object> map);
List<PayDO> list(@SanitizeMap Map<String,Object> map);
int count(Map<String,Object> map);

View File

@@ -1,5 +1,7 @@
package com.java2nb.novel.dao;
import com.java2nb.common.annotation.SanitizeMap;
import com.java2nb.novel.domain.UserDO;
import org.apache.ibatis.annotations.Mapper;
@@ -17,7 +19,7 @@ public interface UserDao {
UserDO get(Long id);
List<UserDO> list(Map<String, Object> map);
List<UserDO> list(@SanitizeMap Map<String, Object> map);
int count(Map<String, Object> map);

View File

@@ -1,5 +1,7 @@
package com.java2nb.novel.dao;
import com.java2nb.common.annotation.SanitizeMap;
import com.java2nb.novel.domain.UserFeedbackDO;
import java.util.List;
@@ -18,7 +20,7 @@ public interface UserFeedbackDao {
UserFeedbackDO get(Long id);
List<UserFeedbackDO> list(Map<String,Object> map);
List<UserFeedbackDO> list(@SanitizeMap Map<String,Object> map);
int count(Map<String,Object> map);

View File

@@ -1,5 +1,7 @@
package com.java2nb.novel.dao;
import com.java2nb.common.annotation.SanitizeMap;
import com.java2nb.novel.domain.WebsiteInfoDO;
import java.util.List;
@@ -18,7 +20,7 @@ public interface WebsiteInfoDao {
WebsiteInfoDO get(Long id);
List<WebsiteInfoDO> list(Map<String,Object> map);
List<WebsiteInfoDO> list(@SanitizeMap Map<String,Object> map);
int count(Map<String,Object> map);

View File

@@ -1,5 +1,7 @@
package com.java2nb.system.dao;
import com.java2nb.common.annotation.SanitizeMap;
import com.java2nb.system.domain.DataPermDO;
import java.util.List;
@@ -19,7 +21,7 @@ public interface DataPermDao {
DataPermDO get(Long id);
List<DataPermDO> list(Map<String,Object> map);
List<DataPermDO> list(@SanitizeMap Map<String,Object> map);
int count(Map<String,Object> map);

View File

@@ -1,5 +1,7 @@
package com.java2nb.system.dao;
import com.java2nb.common.annotation.SanitizeMap;
import com.java2nb.system.domain.DeptDO;
import java.util.List;
@@ -19,7 +21,7 @@ public interface DeptDao {
DeptDO get(Long deptId);
List<DeptDO> list(Map<String,Object> map);
List<DeptDO> list(@SanitizeMap Map<String,Object> map);
int count(Map<String,Object> map);

View File

@@ -1,5 +1,7 @@
package com.java2nb.system.dao;
import com.java2nb.common.annotation.SanitizeMap;
import com.java2nb.system.domain.MenuDO;
import java.util.List;
@@ -18,7 +20,7 @@ public interface MenuDao {
MenuDO get(Long menuId);
List<MenuDO> list(Map<String,Object> map);
List<MenuDO> list(@SanitizeMap Map<String,Object> map);
int count(Map<String,Object> map);

View File

@@ -1,5 +1,7 @@
package com.java2nb.system.dao;
import com.java2nb.common.annotation.SanitizeMap;
import com.java2nb.system.domain.RoleDO;
import java.util.List;
@@ -18,7 +20,7 @@ public interface RoleDao {
RoleDO get(Long roleId);
List<RoleDO> list(Map<String,Object> map);
List<RoleDO> list(@SanitizeMap Map<String,Object> map);
int count(Map<String,Object> map);

View File

@@ -1,5 +1,7 @@
package com.java2nb.system.dao;
import com.java2nb.common.annotation.SanitizeMap;
import com.java2nb.system.domain.RoleDataPermDO;
import java.util.List;
@@ -18,7 +20,7 @@ public interface RoleDataPermDao {
RoleDataPermDO get(Long id);
List<RoleDataPermDO> list(Map<String,Object> map);
List<RoleDataPermDO> list(@SanitizeMap Map<String,Object> map);
int count(Map<String,Object> map);

View File

@@ -1,5 +1,7 @@
package com.java2nb.system.dao;
import com.java2nb.common.annotation.SanitizeMap;
import com.java2nb.system.domain.RoleMenuDO;
import java.util.List;
@@ -18,7 +20,7 @@ public interface RoleMenuDao {
RoleMenuDO get(Long id);
List<RoleMenuDO> list(Map<String,Object> map);
List<RoleMenuDO> list(@SanitizeMap Map<String,Object> map);
int count(Map<String,Object> map);

View File

@@ -1,15 +1,15 @@
package com.java2nb.system.dao;
import com.java2nb.common.annotation.SanitizeMap;
import com.java2nb.system.domain.UserDO;
import java.util.List;
import java.util.Map;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
/**
*
* @author xiongxy
* @email 1179705413@qq.com
* @date 2019-10-03 09:45:11
@@ -17,23 +17,23 @@ import org.apache.ibatis.annotations.Param;
@Mapper
public interface SysUserDao {
UserDO get(Long userId);
List<UserDO> list(Map<String,Object> map);
int count(Map<String,Object> map);
int save(UserDO user);
int update(UserDO user);
int remove(Long userId);
int batchRemove(Long[] userIds);
Long[] listAllDept();
UserDO get(Long userId);
List<UserDO> listByPerm(Map<String, Object> map);
List<UserDO> list(@SanitizeMap Map<String, Object> map);
int countByPerm(Map<String,Object> map);
int count(Map<String, Object> map);
int save(UserDO user);
int update(UserDO user);
int remove(Long userId);
int batchRemove(Long[] userIds);
Long[] listAllDept();
List<UserDO> listByPerm(@SanitizeMap Map<String, Object> map);
int countByPerm(Map<String, Object> map);
}

View File

@@ -1,5 +1,7 @@
package com.java2nb.system.dao;
import com.java2nb.common.annotation.SanitizeMap;
import com.java2nb.system.domain.UserRoleDO;
import java.util.List;
@@ -19,7 +21,7 @@ public interface UserRoleDao {
UserRoleDO get(Long id);
List<UserRoleDO> list(Map<String, Object> map);
List<UserRoleDO> list(@SanitizeMap Map<String, Object> map);
int count(Map<String, Object> map);

View File

@@ -18,7 +18,7 @@ public interface ${className}Dao {
${className}DO get(${pk.javaType} ${pk.attrname});
List<${className}DO> list(Map<String,Object> map);
List<${className}DO> list(@SanitizeMap Map<String,Object> map);
int count(Map<String,Object> map);

View File

@@ -44,7 +44,7 @@ public interface ${className}Mapper {
"limit #{offset}, #{limit}" +
"</if>"+
"</script>")
List<${className}DO> list(Map<String,Object> map);
List<${className}DO> list(@SanitizeMap Map<String,Object> map);
@Select("<script>" +
"select count(*) from ${tableName} " +

View File

@@ -5,7 +5,7 @@
<parent>
<artifactId>novel</artifactId>
<groupId>com.java2nb</groupId>
<version>5.2.0</version>
<version>5.2.3</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -22,6 +22,12 @@
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<!-- Aop 相关 -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-aop</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-thymeleaf</artifactId>

View File

@@ -5,7 +5,7 @@
<parent>
<artifactId>novel</artifactId>
<groupId>com.java2nb</groupId>
<version>5.2.0</version>
<version>5.2.3</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@@ -12,6 +12,8 @@ import io.github.xxyopen.util.IdWorker;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Component;
import java.text.ParseException;
@@ -34,6 +36,13 @@ public class CrawlParser {
private final CrawlHttpClient crawlHttpClient;
private final StringRedisTemplate stringRedisTemplate;
/**
* 爬虫源采集章节数量缓存key
*/
private static final String CRAWL_SOURCE_CHAPTER_COUNT_CACHE_KEY = "crawlSource:chapterCount:";
/**
* 爬虫任务进度
*/
@@ -53,6 +62,20 @@ public class CrawlParser {
crawlTaskProgress.remove(taskId);
}
/**
* 获取爬虫源采集的章节数量
*/
public Long getCrawlSourceChapterCount(Integer sourceId) {
return Optional.ofNullable(
stringRedisTemplate.opsForValue().get(CRAWL_SOURCE_CHAPTER_COUNT_CACHE_KEY + sourceId)).map(v -> {
try {
return Long.parseLong(v);
} catch (NumberFormatException e) {
return 0L;
}
}).orElse(0L);
}
public void parseBook(RuleBean ruleBean, String bookId, CrawlBookHandler handler)
throws InterruptedException {
Book book = new Book();
@@ -182,7 +205,7 @@ public class CrawlParser {
handler.handle(book);
}
public boolean parseBookIndexAndContent(String sourceBookId, Book book, RuleBean ruleBean,
public boolean parseBookIndexAndContent(String sourceBookId, Book book, RuleBean ruleBean, Integer sourceId,
Map<Integer, BookIndex> existBookIndexMap, CrawlBookChapterHandler handler, CrawlSingleTask task)
throws InterruptedException {
@@ -314,10 +337,12 @@ public class CrawlParser {
bookIndex.setUpdateTime(currentDate);
if (task != null) {
// 更新采集进度
// 更新单本任务采集进度
crawlTaskProgress.put(task.getId(), indexList.size());
}
// 更新爬虫源采集章节数量
stringRedisTemplate.opsForValue().increment(CRAWL_SOURCE_CHAPTER_COUNT_CACHE_KEY + sourceId);
}

View File

@@ -74,7 +74,7 @@ public class StarterListener implements ServletContextInitializer {
needUpdateBook.getId());
//解析章节目录
crawlParser.parseBookIndexAndContent(needUpdateBook.getCrawlBookId(), book,
ruleBean, existBookIndexMap,
ruleBean, needUpdateBook.getCrawlSourceId(), existBookIndexMap,
chapter -> bookService.updateBookAndIndexAndContent(book,
chapter.getBookIndexList(),
chapter.getBookContentList(), existBookIndexMap), null);

View File

@@ -100,14 +100,19 @@ public class CrawlServiceImpl implements CrawlService {
PageHelper.startPage(page, pageSize);
SelectStatementProvider render = select(id, sourceName, sourceStatus, createTime, updateTime)
.from(crawlSource)
.orderBy(updateTime)
.orderBy(updateTime.descending())
.build()
.render(RenderingStrategies.MYBATIS3);
List<CrawlSource> crawlSources = crawlSourceMapper.selectMany(render);
crawlSources.forEach(crawlSource -> crawlSource.setSourceStatus(
Optional.ofNullable(crawlSourceStatusMap.get(crawlSource.getId())).orElse((byte) 0)));
PageBean<CrawlSource> pageBean = PageBuilder.build(crawlSources);
pageBean.setList(BeanUtil.copyList(crawlSources, CrawlSourceVO.class));
List<CrawlSourceVO> crawlSourceVOS = BeanUtil.copyList(crawlSources, CrawlSourceVO.class);
crawlSourceVOS.forEach(crawlSource -> {
crawlSource.setSourceStatus(
Optional.ofNullable(crawlSourceStatusMap.get(crawlSource.getId())).orElse((byte) 0));
crawlSource.setChapterCount(crawlParser.getCrawlSourceChapterCount(crawlSource.getId()));
}
);
pageBean.setList(crawlSourceVOS);
return pageBean;
}
@@ -386,7 +391,7 @@ public class CrawlServiceImpl implements CrawlService {
book.setCrawlLastTime(new Date());
book.setId(idWorker.nextId());
//解析章节目录
boolean parseIndexContentResult = crawlParser.parseBookIndexAndContent(bookId, book, ruleBean,
boolean parseIndexContentResult = crawlParser.parseBookIndexAndContent(bookId, book, ruleBean, sourceId,
new HashMap<>(0), chapter -> {
bookService.saveBookAndIndexAndContent(book, chapter.getBookIndexList(),
chapter.getBookContentList());

View File

@@ -20,7 +20,7 @@ public class CrawlSourceVO extends CrawlSource{
@JsonFormat(timezone = "GMT+8", pattern = "yyyy-MM-dd HH:mm")
private Date updateTime;
private Long chapterCount;
@Override
public String toString() {

View File

@@ -43,7 +43,7 @@
<th class="style">
序号
</th>
<th class="chapter">
<th class="name">
爬虫源
</th>
<th class="name">
@@ -52,6 +52,9 @@
<th class="name">
更新时间
</th>
<th class="goread">
采集数量
</th>
<th class="goread">
状态
</th>
@@ -111,11 +114,17 @@
<script src="/javascript/header.js" type="text/javascript"></script>
<script src="/javascript/user.js" type="text/javascript"></script>
<script language="javascript" type="text/javascript">
search(1, 10);
let curr = 1;
let limit = 10;
search();
setInterval(function(){
search();
}, 10000);
var pageCrawlSourceList = null;
function search(curr, limit) {
function search() {
$.ajax({
type: "get",
@@ -134,13 +143,15 @@
" <td class=\"style bookclass\">\n" +
" [" + (i + 1) + "]\n" +
" </td>\n" +
" <td class=\"chapter\">\n" +
" <td class=\"name\">\n" +
" " + crawlSource.sourceName + "</td>\n" +
" <td class=\"name\" valsc=\"291|2037554|1\">"
+ crawlSource.createTime + "</td>\n" +
" <td class=\"name\">\n" +
" " + crawlSource.updateTime + "\n" +
" </td>\n" +
" <td class=\"goread\">\n" +
" " + crawlSource.chapterCount + "章</td>\n" +
" <td class=\"goread\" id='sourceStatus" + crawlSource.id + "'>" + (crawlSource.sourceStatus == 0 ? '停止运行' : '正在运行') +
" </td>\n" +
@@ -169,7 +180,9 @@
//首次不执行
if (!first) {
search(obj.curr, obj.limit);
curr = obj.curr;
limit = obj.limit;
search();
} else {
}
@@ -216,11 +229,11 @@
if (status == 0) {
//开启
$("#sourceStatus" + sourceId).html("正在运行");
$("#opt" + sourceId).html("<a href='javascript:openOrStopCrawl(" + sourceId + "," + 1 + ")'>关闭</a>");
$("#opt" + sourceId).html("<a href='javascript:openOrStopCrawl(" + sourceId + "," + 1 + ")'>关闭 </a>"+"<a href='javascript:updateCrawlSource(" + sourceId + ")'>修改 </a>");
} else {
//关闭
$("#sourceStatus" + sourceId).html("停止运行");
$("#opt" + sourceId).html("<a href='javascript:openOrStopCrawl(" + sourceId + "," + 0 + ")'>开启</a>");
$("#opt" + sourceId).html("<a href='javascript:openOrStopCrawl(" + sourceId + "," + 0 + ")'>开启 </a>"+"<a href='javascript:updateCrawlSource(" + sourceId + ")'>修改 </a>");
}

View File

@@ -5,7 +5,7 @@
<parent>
<artifactId>novel</artifactId>
<groupId>com.java2nb</groupId>
<version>5.2.0</version>
<version>5.2.3</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -59,7 +59,7 @@
<!-- AI -->
<dependency>
<groupId>org.springframework.ai</groupId>
<artifactId>spring-ai-openai-spring-boot-starter</artifactId>
<artifactId>spring-ai-starter-model-openai</artifactId>
</dependency>
<dependency>

View File

@@ -15,6 +15,7 @@ import io.github.xxyopen.model.resp.RestResult;
import jakarta.servlet.http.HttpServletRequest;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.*;
import java.util.HashMap;
@@ -82,7 +83,7 @@ public class BookController extends BaseController {
* 分页搜索
*/
@GetMapping("searchByPage")
public RestResult<?> searchByPage(BookSpVO bookSP, @RequestParam(value = "curr", defaultValue = "1") int page,
public RestResult<?> searchByPage(@Validated BookSpVO bookSP, @RequestParam(value = "curr", defaultValue = "1") int page,
@RequestParam(value = "limit", defaultValue = "20") int pageSize) {
return RestResult.ok(bookService.searchByPage(bookSP, page, pageSize));
}

View File

@@ -1,12 +1,7 @@
package com.java2nb.novel.mapper;
import com.java2nb.novel.entity.Book;
import com.java2nb.novel.vo.BookSpVO;
import com.java2nb.novel.vo.BookVO;
import org.apache.ibatis.annotations.Param;
import java.util.List;
/**
* @author Administrator
*/

View File

@@ -1,5 +1,6 @@
package com.java2nb.novel.vo;
import jakarta.validation.constraints.Pattern;
import lombok.Data;
import java.util.Date;
@@ -29,9 +30,8 @@ public class BookSpVO {
private Long updatePeriod;
@Pattern(regexp = "^(last_index_update_time|word_count|visit_count)$")
private String sort;
}

View File

@@ -23,7 +23,7 @@ xss:
# 排除链接多个用逗号分隔
excludes: /system/notice/*
# 匹配链接 多个用逗号分隔
urlPatterns: /book/addBookComment,/user/addFeedBack,/author/addBook,/author/addBookContent,/author/updateBookContent,/author/register.html
urlPatterns: /book/addCommentReply,/book/addBookComment,/user/addFeedBack,/author/addBook,/author/addBookContent,/author/updateBookContent,/author/register.html
author:
@@ -59,6 +59,6 @@ spring:
base-url: https://api.siliconflow.cn
chat:
options:
model: deepseek-ai/DeepSeek-R1-Distill-Llama-8B
model: deepseek-ai/DeepSeek-R1-0528-Qwen3-8B

View File

@@ -5,7 +5,7 @@
<groupId>com.java2nb</groupId>
<artifactId>novel</artifactId>
<version>5.2.0</version>
<version>5.2.3</version>
<modules>
<module>novel-common</module>
<module>novel-front</module>
@@ -56,7 +56,7 @@
<dependency>
<groupId>org.springframework.ai</groupId>
<artifactId>spring-ai-bom</artifactId>
<version>1.0.0-M6</version>
<version>1.0.0</version>
<type>pom</type>
<scope>import</scope>
</dependency>