From 0279a86e565cbb6501a2338b26a38524b5e20564 Mon Sep 17 00:00:00 2001
From: xiongxiaoyang <1179705413@qq.com>
Date: Tue, 14 Oct 2025 10:17:47 +0800
Subject: [PATCH 1/7] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E4=B8=AD=E5=9B=BD?=
=?UTF-8?q?=E9=83=A8=E5=88=86=20IP=20=E6=97=A0=E7=9C=81=E4=BB=BD=E4=BF=A1?=
=?UTF-8?q?=E6=81=AF=E5=AF=BC=E8=87=B4=E5=9C=B0=E7=90=86=E4=BD=8D=E7=BD=AE?=
=?UTF-8?q?=E6=98=BE=E7=A4=BA=E4=B8=BA=E2=80=9C0=E2=80=9D=E7=9A=84?=
=?UTF-8?q?=E9=97=AE=E9=A2=98?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
部分中国 IP（如 46.248.24.0-46.248.25.255）缺少省份信息，返回值为“0”。
现优化为直接显示国家名称“中国”。
---
.../novel/service/impl/IpLocationServiceImpl.java | 15 ++++++++++++---
1 file changed, 12 insertions(+), 3 deletions(-)
diff --git a/novel-front/src/main/java/com/java2nb/novel/service/impl/IpLocationServiceImpl.java b/novel-front/src/main/java/com/java2nb/novel/service/impl/IpLocationServiceImpl.java
index e3c3241..572e9a7 100644
--- a/novel-front/src/main/java/com/java2nb/novel/service/impl/IpLocationServiceImpl.java
+++ b/novel-front/src/main/java/com/java2nb/novel/service/impl/IpLocationServiceImpl.java
@@ -26,6 +26,7 @@ public class IpLocationServiceImpl implements IpLocationService {
try {
// 示例返回："中国|0|湖北省|武汉市|电信"
String region = searcher.search(ip);
+ log.info("IP：{}，区域：{}", ip, region);
String[] regions = region.split("\\|");
if (regions.length > 0) {
// 国家
@@ -38,9 +39,17 @@ public class IpLocationServiceImpl implements IpLocationService {
return getLocation(publicIp);
}
} else if ("中国".equals(country)) {
- // 是中国，则返回省份（第三个字段）
- String province = regions.length > 2 ? regions[2] : "未知地区";
- // 去掉最后一个“省”字
+ // 若为中国，则尝试返回省份（regions[2]）
+ if (regions.length <= 2) {
+ // 数据不足，直接返回国家
+ return country;
+ }
+ String province = regions[2];
+ if (!StringUtils.hasText(province) || "0".equals(province)) {
+ // 省份字段为空或未知，返回国家
+ return country;
+ }
+ // 去掉末尾的“省”字
return province.endsWith("省") ? province.substring(0, province.length() - 1) : province;
} else {
// 非中国，返回国家名
From 938ae8571dbc6ea93c7d4dd553a738b2de7cdb42 Mon Sep 17 00:00:00 2001
From: xiongxiaoyang <1179705413@qq.com>
Date: Fri, 24 Oct 2025 19:04:50 +0800
Subject: [PATCH 2/7] =?UTF-8?q?feat(novel-crawl):=20=E6=96=B0=E5=A2=9E?=
=?UTF-8?q?=E7=A3=81=E7=9B=98=E4=BF=9D=E6=8A=A4=E6=9C=BA=E5=88=B6=EF=BC=8C?=
=?UTF-8?q?=E6=94=AF=E6=8C=81=E9=82=AE=E4=BB=B6=E5=91=8A=E8=AD=A6=E4=B8=8E?=
=?UTF-8?q?=E8=87=AA=E5=8A=A8=E7=86=94=E6=96=AD?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
- 实现定时检测磁盘使用率，避免爬虫耗尽磁盘资源
- 当磁盘使用率达到 85%、90%、95% 时，分别发送告警邮件
- 当使用率达到 95% 时，强制停止当前爬虫进程
---
novel-crawl/pom.xml | 4 +
.../com/java2nb/novel/core/bean/DiskInfo.java | 28 ++++
.../core/config/DiskMonitorProperties.java | 25 ++++
.../core/schedule/DiskMonitorSchedule.java | 138 ++++++++++++++++++
.../java2nb/novel/service/EmailService.java | 14 ++
.../novel/service/impl/EmailServiceImpl.java | 62 ++++++++
.../src/main/resources/application.yml | 37 ++++-
.../main/resources/templates/disk_alert.html | 44 ++++++
8 files changed, 351 insertions(+), 1 deletion(-)
create mode 100644 novel-crawl/src/main/java/com/java2nb/novel/core/bean/DiskInfo.java
create mode 100644 novel-crawl/src/main/java/com/java2nb/novel/core/config/DiskMonitorProperties.java
create mode 100644 novel-crawl/src/main/java/com/java2nb/novel/core/schedule/DiskMonitorSchedule.java
create mode 100644 novel-crawl/src/main/java/com/java2nb/novel/service/EmailService.java
create mode 100644 novel-crawl/src/main/java/com/java2nb/novel/service/impl/EmailServiceImpl.java
create mode 100644 novel-crawl/src/main/resources/templates/disk_alert.html
diff --git a/novel-crawl/pom.xml b/novel-crawl/pom.xml
index 3cfe319..c8e78ca 100644
--- a/novel-crawl/pom.xml
+++ b/novel-crawl/pom.xml
@@ -37,6 +37,10 @@
jackson-databind
+
+ org.springframework.boot
+ spring-boot-starter-mail
+
diff --git a/novel-crawl/src/main/java/com/java2nb/novel/core/bean/DiskInfo.java b/novel-crawl/src/main/java/com/java2nb/novel/core/bean/DiskInfo.java
new file mode 100644
index 0000000..416ecbd
--- /dev/null
+++ b/novel-crawl/src/main/java/com/java2nb/novel/core/bean/DiskInfo.java
@@ -0,0 +1,28 @@
+package com.java2nb.novel.core.bean;
+
+import lombok.Data;
+
+/**
+ * @author xiongxiaoyang
+ * @date 2025/10/24
+ */
+@Data
+public class DiskInfo {
+
+ private String path;
+ private double totalGB;
+ private double freeGB;
+ private double usage;
+
+ public DiskInfo(String path, long totalBytes, long freeBytes, double usage) {
+ this.path = path;
+ this.totalGB = bytesToGB(totalBytes);
+ this.freeGB = bytesToGB(freeBytes);
+ this.usage = usage;
+ }
+
+ private double bytesToGB(long bytes) {
+ return bytes / (1024.0 * 1024.0 * 1024.0);
+ }
+
+}
diff --git a/novel-crawl/src/main/java/com/java2nb/novel/core/config/DiskMonitorProperties.java b/novel-crawl/src/main/java/com/java2nb/novel/core/config/DiskMonitorProperties.java
new file mode 100644
index 0000000..36a5fa9
--- /dev/null
+++ b/novel-crawl/src/main/java/com/java2nb/novel/core/config/DiskMonitorProperties.java
@@ -0,0 +1,25 @@
+package com.java2nb.novel.core.config;
+
+import lombok.Data;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.stereotype.Component;
+
+import java.util.List;
+
+/**
+ * @author xiongxiaoyang
+ * @date 2025/10/24
+ */
+@Component
+@ConfigurationProperties(prefix = "disk-monitor")
+@Data
+public class DiskMonitorProperties {
+
+ private boolean enabled;
+ private int intervalMinutes;
+ private double warningThreshold;
+ private double severeThreshold;
+ private double criticalThreshold;
+ private List recipients;
+
+}
diff --git a/novel-crawl/src/main/java/com/java2nb/novel/core/schedule/DiskMonitorSchedule.java b/novel-crawl/src/main/java/com/java2nb/novel/core/schedule/DiskMonitorSchedule.java
new file mode 100644
index 0000000..c7207fa
--- /dev/null
+++ b/novel-crawl/src/main/java/com/java2nb/novel/core/schedule/DiskMonitorSchedule.java
@@ -0,0 +1,138 @@
+package com.java2nb.novel.core.schedule;
+
+import com.java2nb.novel.core.bean.DiskInfo;
+import com.java2nb.novel.core.config.DiskMonitorProperties;
+import com.java2nb.novel.service.EmailService;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Service;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * @author xiongxiaoyang
+ * @date 2025/10/24
+ */
+@ConditionalOnProperty(prefix = "disk-monitor", name = "enabled", havingValue = "true")
+@Service
+@RequiredArgsConstructor
+@Slf4j
+public class DiskMonitorSchedule {
+
+ private final DiskMonitorProperties properties;
+
+ private final EmailService emailService;
+
+ private final AtomicBoolean criticalAlertSent = new AtomicBoolean(false);
+
+ @Scheduled(fixedDelayString = "#{1000 * 60 * ${disk-monitor.interval-minutes}}")
+ public void checkDiskUsage() {
+ log.info("🔍 开始检查磁盘使用情况...");
+
+ File[] roots = File.listRoots();
+ List diskInfos = new ArrayList<>();
+ boolean criticalDetected = false;
+
+ for (File root : roots) {
+ String path = root.getAbsolutePath().trim();
+ if (path.isEmpty()) continue;
+ long total = root.getTotalSpace();
+ if (total == 0) continue;
+ long free = root.getFreeSpace();
+ double usage = (double)(total - free) / total * 100;
+ diskInfos.add(new DiskInfo(path, total, free, usage));
+ if (usage >= properties.getCriticalThreshold()) {
+ criticalDetected = true;
+ }
+ }
+
+ if (criticalDetected) {
+ if (criticalAlertSent.compareAndSet(false, true)) {
+ sendAlertEmail("CRITICAL", diskInfos);
+ log.error("🚨 磁盘使用率 ≥ 95%，10秒后终止进程...");
+ try {
+ Thread.sleep(10000);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ System.exit(1);
+ }
+ } else {
+ criticalAlertSent.set(false);
+ double maxUsage = diskInfos.stream().mapToDouble(DiskInfo::getUsage).max().orElse(0);
+ if (maxUsage >= properties.getSevereThreshold() && maxUsage < properties.getCriticalThreshold()) {
+ sendAlertEmail("WARNING", diskInfos);
+ } else if (maxUsage >= properties.getWarningThreshold() && maxUsage < properties.getSevereThreshold()) {
+ sendAlertEmail("INFO", diskInfos);
+ }
+ }
+ }
+
+ private void sendAlertEmail(String level, List diskInfos) {
+ Map model = new HashMap<>();
+ model.put("diskInfos", diskInfos);
+ model.put("alertLevel", getAlertLevelText(level));
+ model.put("icon", getIcon(level));
+ model.put("actionText", getActionText(level));
+ model.put("levelColor", getLevelColor(level));
+
+ String subject = getSubject(level);
+ emailService.sendHtmlEmail(subject, "disk_alert", model, properties.getRecipients());
+ }
+
+ private String getAlertLevelText(String level) {
+ return switch (level) {
+ case "CRITICAL" -> "严重";
+ case "WARNING" -> "警告";
+ case "INFO" -> "提示";
+ default -> "通知";
+ };
+ }
+
+ private String getIcon(String level) {
+ return switch (level) {
+ case "CRITICAL" -> "🚨";
+ case "WARNING" -> "⚠️";
+ case "INFO" -> "💡";
+ default -> "📢";
+ };
+ }
+
+ private String getActionText(String level) {
+ return switch (level) {
+ case "CRITICAL" ->
+ "⚠️ 检测到任一磁盘使用率 ≥ 95%，为防止系统崩溃，系统已自动关闭爬虫程序。";
+ case "WARNING" ->
+ "📌 建议：请立即暂停爬虫程序，防止磁盘写满导致服务中断。";
+ case "INFO" ->
+ "📌 提示：磁盘使用率已较高，请关注爬虫数据写入情况。";
+ default -> "";
+ };
+ }
+
+ private String getLevelColor(String level) {
+ return switch (level) {
+ case "CRITICAL" -> "#d32f2f";
+ case "WARNING" -> "#f57c00";
+ case "INFO" -> "#388e3c";
+ default -> "#1976d2";
+ };
+ }
+
+ private String getSubject(String level) {
+ return switch (level) {
+ case "CRITICAL" -> "🚨 严重告警：磁盘使用率超 95%，爬虫已关闭";
+ case "WARNING" -> "⚠️ 警告：磁盘使用率超 90%，请暂停爬虫";
+ case "INFO" -> "💡 提示：磁盘使用率超 85%";
+ default -> "磁盘使用率告警";
+ };
+ }
+
+}
diff --git a/novel-crawl/src/main/java/com/java2nb/novel/service/EmailService.java b/novel-crawl/src/main/java/com/java2nb/novel/service/EmailService.java
new file mode 100644
index 0000000..474da6d
--- /dev/null
+++ b/novel-crawl/src/main/java/com/java2nb/novel/service/EmailService.java
@@ -0,0 +1,14 @@
+package com.java2nb.novel.service;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author xiongxiaoyang
+ * @date 2025/10/24
+ */
+public interface EmailService {
+
+ void sendHtmlEmail(String subject, String templateName, Map templateModel, List to);
+
+}
diff --git a/novel-crawl/src/main/java/com/java2nb/novel/service/impl/EmailServiceImpl.java b/novel-crawl/src/main/java/com/java2nb/novel/service/impl/EmailServiceImpl.java
new file mode 100644
index 0000000..c3bf15e
--- /dev/null
+++ b/novel-crawl/src/main/java/com/java2nb/novel/service/impl/EmailServiceImpl.java
@@ -0,0 +1,62 @@
+package com.java2nb.novel.service.impl;
+
+import com.java2nb.novel.service.EmailService;
+import jakarta.mail.internet.InternetAddress;
+import jakarta.mail.internet.MimeMessage;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.mail.javamail.JavaMailSender;
+import org.springframework.mail.javamail.MimeMessageHelper;
+import org.springframework.stereotype.Service;
+import org.thymeleaf.TemplateEngine;
+import org.thymeleaf.context.Context;
+
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author xiongxiaoyang
+ * @date 2025/10/24
+ */
+@Slf4j
+@Service
+@RequiredArgsConstructor
+public class EmailServiceImpl implements EmailService {
+
+ private final JavaMailSender javaMailSender;
+ private final TemplateEngine templateEngine;
+
+ @Value("${spring.mail.username}")
+ private String fromEmail;
+
+ @Value("${spring.mail.nickname}")
+ private String nickName;
+
+ @Override
+ public void sendHtmlEmail(String subject, String templateName, Map templateModel, List to) {
+ try {
+ MimeMessage message = javaMailSender.createMimeMessage();
+ MimeMessageHelper helper = new MimeMessageHelper(
+ message,
+ MimeMessageHelper.MULTIPART_MODE_MIXED_RELATED,
+ StandardCharsets.UTF_8.name());
+
+ Context context = new Context();
+ context.setVariables(templateModel);
+
+ String htmlBody = templateEngine.process(templateName, context);
+
+ helper.setFrom(new InternetAddress(fromEmail, nickName, "UTF-8"));
+ helper.setTo(to.toArray(new String[0]));
+ helper.setSubject(subject);
+ helper.setText(htmlBody, true);
+
+ javaMailSender.send(message);
+ } catch (Exception e) {
+ log.error("发送邮件失败");
+ log.error(e.getMessage(), e);
+ }
+ }
+}
diff --git a/novel-crawl/src/main/resources/application.yml b/novel-crawl/src/main/resources/application.yml
index b4908fe..de787fb 100644
--- a/novel-crawl/src/main/resources/application.yml
+++ b/novel-crawl/src/main/resources/application.yml
@@ -28,6 +28,41 @@ crawl:
max: 500
-
+---
+
+#邮箱服务器
+spring:
+ mail:
+ host: smtp.163.com
+ #发件人昵称
+ nickname: novel-plus
+ #邮箱账户
+ username: xxyopen@163.com
+ #邮箱第三方授权码
+ password: novel123456
+ #编码类型
+ default-encoding: UTF-8
+ port: 465
+ properties:
+ mail:
+ smtp:
+ auth: true
+ starttls:
+ enable: true
+ required: rue
+ socketFactory:
+ port: 465
+ class: javax.net.ssl.SSLSocketFactory
+ fallback: false
+
+# 磁盘监控配置
+disk-monitor:
+ enabled: false
+ interval-minutes: 5
+ warning-threshold: 85.0
+ severe-threshold: 90.0
+ critical-threshold: 95.0
+ # 告警邮件接收人列表（支持多个邮箱）
+ recipients: xxyopen@foxmail.com,12345678@qq.com
diff --git a/novel-crawl/src/main/resources/templates/disk_alert.html b/novel-crawl/src/main/resources/templates/disk_alert.html
new file mode 100644
index 0000000..9629139
--- /dev/null
+++ b/novel-crawl/src/main/resources/templates/disk_alert.html
@@ -0,0 +1,44 @@
+
+
+
+
+
+
+
+
+
+
+
时间：
+
+
+
+
📊 磁盘使用详情
+
+
+ | 磁盘路径 | 总空间 (GB) | 空闲空间 (GB) | 使用率 |
+
+
+ |
+ |
+ |
+
+ |
+
+
+
+
+
';
var r = ' ';
- return r;
+ return d + r;
}
}
@@ -249,4 +253,9 @@ function batchRemove() {
}, function () {
});
+}
+
+function downloadBook(bookId, bookName) {
+ window.open(prefix + '/download?bookId='+bookId+'&bookName='+bookName);
+
}
\ No newline at end of file
From 3d1b952e1aa62ae2430138e9bf772dddb7b3cae7 Mon Sep 17 00:00:00 2001
From: xiongxiaoyang <1179705413@qq.com>
Date: Sat, 25 Oct 2025 19:19:45 +0800
Subject: [PATCH 6/7] =?UTF-8?q?perf(novel-crawl):=20=E4=BC=98=E5=8C=96?=
=?UTF-8?q?=E9=87=87=E9=9B=86=E7=9B=91=E6=8E=A7=E9=A1=B5=E5=BC=82=E5=B8=B8?=
=?UTF-8?q?=E6=8F=90=E7=A4=BA=E4=BD=93=E9=AA=8C?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
在爬虫采集监控页面，针对网络或服务异常情况，避免频繁弹窗提示，提升用户体验。
---
.../templates/crawl/crawlSingleTask_list.html | 16 +++++++++++++---
.../templates/crawl/crawlSource_list.html | 19 +++++++++++--------
2 files changed, 24 insertions(+), 11 deletions(-)
diff --git a/novel-crawl/src/main/resources/templates/crawl/crawlSingleTask_list.html b/novel-crawl/src/main/resources/templates/crawl/crawlSingleTask_list.html
index f0bca00..a80bbcc 100644
--- a/novel-crawl/src/main/resources/templates/crawl/crawlSingleTask_list.html
+++ b/novel-crawl/src/main/resources/templates/crawl/crawlSingleTask_list.html
@@ -118,9 +118,10 @@