shen/CloudFlare-AI-Insight-Daily
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(数据源): 添加新数据源

Browse Source 
- 在wrangler.toml中添加新数据源的环境变量配置
- 新增三个数据源模块(qbit.js/xinzhiyuan.js/jiqizhixin.js)实现数据获取和转换
- 更新dataFetchers.js整合新数据源到论文分类
- 更新README.md说明新增的科技大V社交媒体内容来源
This commit is contained in:
justlovemaki
2025-06-20 17:41:53 +08:00
parent 53dc5124d1
commit b80312aed4
8 changed files with 456 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
.github/workflows/build-daily-book.yml vendored
View File

@@ -112,6 +112,34 @@ jobs:
mkdir -p today
mv book today/
- name: Add Clarity tracking to book.js
run: |
TOC_FILE="today/book/book.js"
FORWARD_CODE='${{ vars.FORWARD_CODE }}'
CLARITY_CODE='${{ vars.CLARITY_CODE }}'
if [ -f "$TOC_FILE" ]; then
echo "Appending Forward tracking code to $TOC_FILE"
# Add a newline first
echo "" >> "$TOC_FILE"
# Append the Forward script
echo "$FORWARD_CODE" >> "$TOC_FILE"
echo "Forward code appended."
echo "Appending Clarity tracking code to $TOC_FILE"
# Add a newline first
echo "" >> "$TOC_FILE"
# Append the Clarity script
echo "$CLARITY_CODE" >> "$TOC_FILE"
echo "Clarity code appended."
# tail -1000 "$TOC_FILE"
else
echo "Warning: $TOC_FILE not found. Skipping Clarity code injection."
# Depending on your needs, you might want to fail the build if toc.js is essential
# and not found, e.g., by uncommenting the next line:
# exit 1
fi
- name: Download RSS Feed
run: |
if [ -z "${{ vars.RSS_FEED_URL }}" ]; then

13
README.md
View File

@@ -2,7 +2,7 @@
> 您的每日 AI 信息整合,分析,日报,播客内容生成平台。
**AI 洞察日报** 是一个基于 **Cloudflare Workers** 驱动的内容聚合与生成平台。它每日为您精选 AI 领域的最新动态，包括行业新闻、热门开源项目和前沿学术论文，并通过 **Google Gemini** 模型进行智能处理与摘要生成，最终自动发布到 GitHub Pages。
**AI 洞察日报** 是一个基于 **Cloudflare Workers** 驱动的内容聚合与生成平台。它每日为您精选 AI 领域的最新动态，包括行业新闻、热门开源项目、前沿学术论文、科技大V社交媒体言论，并通过 **Google Gemini** 模型进行智能处理与摘要生成，最终自动发布到 GitHub Pages 生成 AI 日报。
我们的目标是成为您在瞬息万变的 AI 浪潮中保持领先的得力助手，让您高效获取最有价值的信息。
@@ -61,18 +61,13 @@
#### 💻 网页直达
无需安装任何应用，直接在浏览器中打开，即刻阅读。
无需安装任何应用，直接在浏览器中打开，即刻阅读，支持pc和移动端。
* **主站点 (GitHub Pages)**
> [https://justlovemaki.github.io/CloudFlare-AI-Insight-Daily/today/book/](https://justlovemaki.github.io/CloudFlare-AI-Insight-Daily/today/book/)
* **唯一主站点 (GitHub Pages)**
> [https://ai.hubtoday.app/](https://ai.hubtoday.app/)
>
> `✅ 推荐` `🚀 访问速度快`
* **备用站点 (Cloudflare)**
> [https://ai-today.justlikemaki.vip/](https://ai-today.justlikemaki.vip/)
>
> `💡 当主站点访问受阻时，请使用此链接`
---
#### 📡 RSS 订阅

5
src/dataFetchers.js
View File

@@ -2,6 +2,9 @@
import AibaseDataSource from './dataSources/aibase.js';
import GithubTrendingDataSource from './dataSources/github-trending.js';
import HuggingfacePapersDataSource from './dataSources/huggingface-papers.js';
import XinZhiYuanDataSource from './dataSources/xinzhiyuan.js';
import QBitDataSource from './dataSources/qbit.js';
import JiqizhixinDataSource from './dataSources/jiqizhixin.js';
import XiaohuDataSource from './dataSources/xiaohu.js';
import TwitterDataSource from './dataSources/twitter.js';
@@ -9,7 +12,7 @@ import TwitterDataSource from './dataSources/twitter.js';
export const dataSources = {
news: { name: '新闻', sources: [AibaseDataSource, XiaohuDataSource] },
project: { name: '项目', sources: [GithubTrendingDataSource] },
paper: { name: '论文', sources: [HuggingfacePapersDataSource] },
paper: { name: '论文', sources: [HuggingfacePapersDataSource, XinZhiYuanDataSource, QBitDataSource, JiqizhixinDataSource] },
socialMedia: { name: '社交平台', sources: [TwitterDataSource] },
// Add new data sources here as arrays, e.g.,
// newType: { name: '新类型', sources: [NewTypeDataSource1, NewTypeDataSource2] },

137
src/dataSources/jiqizhixin.js Normal file
View File

@@ -0,0 +1,137 @@
import { getRandomUserAgent, sleep, isDateWithinLastDays, stripHtml, formatDateToChineseWithTime, escapeHtml } from '../helpers.js';
const JiqizhixinDataSource = {
fetch: async (env, foloCookie) => {
const feedId = env.JIQIZHIXIN_FEED_ID;
const fetchPages = parseInt(env.JIQIZHIXIN_FETCH_PAGES || '3', 10);
const allJiqizhixinItems = [];
const filterDays = parseInt(env.FOLO_FILTER_DAYS || '3', 10);
if (!feedId) {
console.error('JIQIZHIXIN_FEED_ID is not set in environment variables.');
return {
version: "https://jsonfeed.org/version/1.1",
title: "Jiqizhixin.AI Daily Feeds",
home_page_url: "https://www.jiqizhixin.ai",
description: "Aggregated Jiqizhixin.AI Daily feeds",
language: "zh-cn",
items: []
};
}
let publishedAfter = null;
for (let i = 0; i < fetchPages; i++) {
const userAgent = getRandomUserAgent();
const headers = {
'User-Agent': userAgent,
'Content-Type': 'application/json',
'accept': 'application/json',
'accept-language': 'zh-CN,zh;q=0.9',
'baggage': 'sentry-environment=stable,sentry-release=5251fa921ef6cbb6df0ac4271c41c2b4a0ce7c50,sentry-public_key=e5bccf7428aa4e881ed5cb713fdff181,sentry-trace_id=2da50ca5ad944cb794670097d876ada8,sentry-sampled=true,sentry-sample_rand=0.06211835167903246,sentry-sample_rate=1',
'origin': 'https://app.follow.is',
'priority': 'u=1, i',
'sec-ch-ua': '"Google Chrome";v="135", "Not-A.Brand";v="8", "Chromium";v="135"',
'sec-ch-ua-mobile': '?1',
'sec-ch-ua-platform': '"Android"',
'sec-fetch-dest': 'empty',
'sec-fetch-mode': 'cors',
'sec-fetch-site': 'same-site',
'x-app-name': 'Folo Web',
'x-app-version': '0.4.9',
};
// 直接使用传入的 foloCookie
if (foloCookie) {
headers['Cookie'] = foloCookie;
}
const body = {
feedId: feedId,
view: 1,
withContent: true,
};
if (publishedAfter) {
body.publishedAfter = publishedAfter;
}
try {
console.log(`Fetching Jiqizhixin.AI data, page ${i + 1}...`);
const response = await fetch(env.FOLO_DATA_API, {
method: 'POST',
headers: headers,
body: JSON.stringify(body),
});
if (!response.ok) {
console.error(`Failed to fetch Jiqizhixin.AI data, page ${i + 1}: ${response.statusText}`);
break;
}
const data = await response.json();
if (data && data.data && data.data.length > 0) {
const filteredItems = data.data.filter(entry => isDateWithinLastDays(entry.entries.publishedAt, filterDays));
allJiqizhixinItems.push(...filteredItems.map(entry => ({
id: entry.entries.id,
url: entry.entries.url,
title: entry.entries.title,
content_html: entry.entries.content,
date_published: entry.entries.publishedAt,
authors: [{ name: entry.entries.author }],
source: `机器之心`,
})));
publishedAfter = data.data[data.data.length - 1].entries.publishedAt;
} else {
console.log(`No more data for Jiqizhixin.AI, page ${i + 1}.`);
break;
}
} catch (error) {
console.error(`Error fetching Jiqizhixin.AI data, page ${i + 1}:`, error);
break;
}
// Random wait time between 0 and 5 seconds to avoid rate limiting
await sleep(Math.random() * 5000);
}
return {
version: "https://jsonfeed.org/version/1.1",
title: "Jiqizhixin.AI Daily Feeds",
home_page_url: "https://www.jiqizhixin.ai",
description: "Aggregated Jiqizhixin.AI Daily feeds",
language: "zh-cn",
items: allJiqizhixinItems
};
},
transform: (rawData, sourceType) => {
const unifiedNews = [];
if (rawData && Array.isArray(rawData.items)) {
rawData.items.forEach((item) => {
unifiedNews.push({
id: item.id,
type: sourceType,
url: item.url,
title: item.title,
description: stripHtml(item.content_html || ""),
published_date: item.date_published,
authors: item.authors ? item.authors.map(a => a.name).join(', ') : 'Unknown',
source: item.source || '机器之心',
details: {
content_html: item.content_html || ""
}
});
});
}
return unifiedNews;
},
generateHtml: (item) => {
return `
<strong>${escapeHtml(item.title)}</strong><br>
<small>来源: ${escapeHtml(item.source || '未知')} | 发布日期: ${formatDateToChineseWithTime(item.published_date)}</small>
<div class="content-html">${item.details.content_html || '无内容。'}</div>
<a href="${escapeHtml(item.url)}" target="_blank" rel="noopener noreferrer">阅读更多</a>
`;
}
};
export default JiqizhixinDataSource;

137
src/dataSources/qbit.js Normal file
View File

@@ -0,0 +1,137 @@
import { getRandomUserAgent, sleep, isDateWithinLastDays, stripHtml, formatDateToChineseWithTime, escapeHtml } from '../helpers.js';
const QBitDataSource = {
fetch: async (env, foloCookie) => {
const feedId = env.QBIT_FEED_ID;
const fetchPages = parseInt(env.QBIT_FETCH_PAGES || '3', 10);
const allQBitItems = [];
const filterDays = parseInt(env.FOLO_FILTER_DAYS || '3', 10);
if (!feedId) {
console.error('QBIT_FEED_ID is not set in environment variables.');
return {
version: "https://jsonfeed.org/version/1.1",
title: "QBit.AI Daily Feeds",
home_page_url: "https://www.qbit.ai",
description: "Aggregated QBit.AI Daily feeds",
language: "zh-cn",
items: []
};
}
let publishedAfter = null;
for (let i = 0; i < fetchPages; i++) {
const userAgent = getRandomUserAgent();
const headers = {
'User-Agent': userAgent,
'Content-Type': 'application/json',
'accept': 'application/json',
'accept-language': 'zh-CN,zh;q=0.9',
'baggage': 'sentry-environment=stable,sentry-release=5251fa921ef6cbb6df0ac4271c41c2b4a0ce7c50,sentry-public_key=e5bccf7428aa4e881ed5cb713fdff181,sentry-trace_id=2da50ca5ad944cb794670097d876ada8,sentry-sampled=true,sentry-sample_rand=0.06211835167903246,sentry-sample_rate=1',
'origin': 'https://app.follow.is',
'priority': 'u=1, i',
'sec-ch-ua': '"Google Chrome";v="135", "Not-A.Brand";v="8", "Chromium";v="135"',
'sec-ch-ua-mobile': '?1',
'sec-ch-ua-platform': '"Android"',
'sec-fetch-dest': 'empty',
'sec-fetch-mode': 'cors',
'sec-fetch-site': 'same-site',
'x-app-name': 'Folo Web',
'x-app-version': '0.4.9',
};
// 直接使用传入的 foloCookie
if (foloCookie) {
headers['Cookie'] = foloCookie;
}
const body = {
feedId: feedId,
view: 1,
withContent: true,
};
if (publishedAfter) {
body.publishedAfter = publishedAfter;
}
try {
console.log(`Fetching QBit.AI data, page ${i + 1}...`);
const response = await fetch(env.FOLO_DATA_API, {
method: 'POST',
headers: headers,
body: JSON.stringify(body),
});
if (!response.ok) {
console.error(`Failed to fetch QBit.AI data, page ${i + 1}: ${response.statusText}`);
break;
}
const data = await response.json();
if (data && data.data && data.data.length > 0) {
const filteredItems = data.data.filter(entry => isDateWithinLastDays(entry.entries.publishedAt, filterDays));
allQBitItems.push(...filteredItems.map(entry => ({
id: entry.entries.id,
url: entry.entries.url,
title: entry.entries.title,
content_html: entry.entries.content,
date_published: entry.entries.publishedAt,
authors: [{ name: entry.entries.author }],
source: `量子位`,
})));
publishedAfter = data.data[data.data.length - 1].entries.publishedAt;
} else {
console.log(`No more data for QBit.AI, page ${i + 1}.`);
break;
}
} catch (error) {
console.error(`Error fetching QBit.AI data, page ${i + 1}:`, error);
break;
}
// Random wait time between 0 and 5 seconds to avoid rate limiting
await sleep(Math.random() * 5000);
}
return {
version: "https://jsonfeed.org/version/1.1",
title: "QBit.AI Daily Feeds",
home_page_url: "https://www.qbit.ai",
description: "Aggregated QBit.AI Daily feeds",
language: "zh-cn",
items: allQBitItems
};
},
transform: (rawData, sourceType) => {
const unifiedNews = [];
if (rawData && Array.isArray(rawData.items)) {
rawData.items.forEach((item) => {
unifiedNews.push({
id: item.id,
type: sourceType,
url: item.url,
title: item.title,
description: stripHtml(item.content_html || ""),
published_date: item.date_published,
authors: item.authors ? item.authors.map(a => a.name).join(', ') : 'Unknown',
source: item.source || '量子位',
details: {
content_html: item.content_html || ""
}
});
});
}
return unifiedNews;
},
generateHtml: (item) => {
return `
<strong>${escapeHtml(item.title)}</strong><br>
<small>来源: ${escapeHtml(item.source || '未知')} | 发布日期: ${formatDateToChineseWithTime(item.published_date)}</small>
<div class="content-html">${item.details.content_html || '无内容。'}</div>
<a href="${escapeHtml(item.url)}" target="_blank" rel="noopener noreferrer">阅读更多</a>
`;
}
};
export default QBitDataSource;

137
src/dataSources/xinzhiyuan.js Normal file
View File

@@ -0,0 +1,137 @@
import { getRandomUserAgent, sleep, isDateWithinLastDays, stripHtml, formatDateToChineseWithTime, escapeHtml } from '../helpers.js';
const XinZhiYuanDataSource = {
fetch: async (env, foloCookie) => {
const feedId = env.XINZHIYUAN_FEED_ID;
const fetchPages = parseInt(env.XINZHIYUAN_FETCH_PAGES || '3', 10);
const allXinZhiYuanItems = [];
const filterDays = parseInt(env.FOLO_FILTER_DAYS || '3', 10);
if (!feedId) {
console.error('XINZHIYUAN_FEED_ID is not set in environment variables.');
return {
version: "https://jsonfeed.org/version/1.1",
title: "XinZhiYuan.AI Daily Feeds",
home_page_url: "https://www.xinzhiyuan.ai",
description: "Aggregated XinZhiYuan.AI Daily feeds",
language: "zh-cn",
items: []
};
}
let publishedAfter = null;
for (let i = 0; i < fetchPages; i++) {
const userAgent = getRandomUserAgent();
const headers = {
'User-Agent': userAgent,
'Content-Type': 'application/json',
'accept': 'application/json',
'accept-language': 'zh-CN,zh;q=0.9',
'baggage': 'sentry-environment=stable,sentry-release=5251fa921ef6cbb6df0ac4271c41c2b4a0ce7c50,sentry-public_key=e5bccf7428aa4e881ed5cb713fdff181,sentry-trace_id=2da50ca5ad944cb794670097d876ada8,sentry-sampled=true,sentry-sample_rand=0.06211835167903246,sentry-sample_rate=1',
'origin': 'https://app.follow.is',
'priority': 'u=1, i',
'sec-ch-ua': '"Google Chrome";v="135", "Not-A.Brand";v="8", "Chromium";v="135"',
'sec-ch-ua-mobile': '?1',
'sec-ch-ua-platform': '"Android"',
'sec-fetch-dest': 'empty',
'sec-fetch-mode': 'cors',
'sec-fetch-site': 'same-site',
'x-app-name': 'Folo Web',
'x-app-version': '0.4.9',
};
// 直接使用传入的 foloCookie
if (foloCookie) {
headers['Cookie'] = foloCookie;
}
const body = {
feedId: feedId,
view: 1,
withContent: true,
};
if (publishedAfter) {
body.publishedAfter = publishedAfter;
}
try {
console.log(`Fetching XinZhiYuan.AI data, page ${i + 1}...`);
const response = await fetch(env.FOLO_DATA_API, {
method: 'POST',
headers: headers,
body: JSON.stringify(body),
});
if (!response.ok) {
console.error(`Failed to fetch XinZhiYuan.AI data, page ${i + 1}: ${response.statusText}`);
break;
}
const data = await response.json();
if (data && data.data && data.data.length > 0) {
const filteredItems = data.data.filter(entry => isDateWithinLastDays(entry.entries.publishedAt, filterDays));
allXinZhiYuanItems.push(...filteredItems.map(entry => ({
id: entry.entries.id,
url: entry.entries.url,
title: entry.entries.title,
content_html: entry.entries.content,
date_published: entry.entries.publishedAt,
authors: [{ name: entry.entries.author }],
source: `新智元`,
})));
publishedAfter = data.data[data.data.length - 1].entries.publishedAt;
} else {
console.log(`No more data for XinZhiYuan.AI, page ${i + 1}.`);
break;
}
} catch (error) {
console.error(`Error fetching XinZhiYuan.AI data, page ${i + 1}:`, error);
break;
}
// Random wait time between 0 and 5 seconds to avoid rate limiting
await sleep(Math.random() * 5000);
}
return {
version: "https://jsonfeed.org/version/1.1",
title: "XinZhiYuan.AI Daily Feeds",
home_page_url: "https://www.xinzhiyuan.ai",
description: "Aggregated XinZhiYuan.AI Daily feeds",
language: "zh-cn",
items: allXinZhiYuanItems
};
},
transform: (rawData, sourceType) => {
const unifiedNews = [];
if (rawData && Array.isArray(rawData.items)) {
rawData.items.forEach((item) => {
unifiedNews.push({
id: item.id,
type: sourceType,
url: item.url,
title: item.title,
description: stripHtml(item.content_html || ""),
published_date: item.date_published,
authors: item.authors ? item.authors.map(a => a.name).join(', ') : 'Unknown',
source: item.source || '新智元',
details: {
content_html: item.content_html || ""
}
});
});
}
return unifiedNews;
},
generateHtml: (item) => {
return `
<strong>${escapeHtml(item.title)}</strong><br>
<small>来源: ${escapeHtml(item.source || '未知')} | 发布日期: ${formatDateToChineseWithTime(item.published_date)}</small>
<div class="content-html">${item.details.content_html || '无内容。'}</div>
<a href="${escapeHtml(item.url)}" target="_blank" rel="noopener noreferrer">阅读更多</a>
`;
}
};
export default XinZhiYuanDataSource;

6
src/handlers/genAIContent.js
View File

@@ -28,7 +28,7 @@ export async function handleGenAIPodcastScript(request, env) {
outputOfCall1 = formData.get('summarizedContent'); // Get summarized content from form data
if (!outputOfCall1) {
const errorHtml = generateGenAiPageHtml(env, '生成AI播客脚本出错', '<p><strong>Summarized content is missing.</strong> Please go back and generate AI content first.</p>', dateStr, true, null);
const errorHtml = generateGenAiPageHtml(env, '生成AI播客脚本出错', '<p><strong>Summarized content is missing.</strong> Please go back and generate AI content first.</p>', dateStr, true, null, null, null, null, null, null, outputOfCall1, null);
return new Response(errorHtml, { status: 400, headers: { 'Content-Type': 'text/html; charset=utf-8' } });
}
@@ -49,7 +49,7 @@ export async function handleGenAIPodcastScript(request, env) {
console.log("Call 3 (Podcast Formatting) successful. Final output length:", finalAiResponse.length);
} catch (error) {
console.error("Error in Chat API Call 3 (Podcast Formatting):", error);
const errorHtml = generateGenAiPageHtml(env, '生成AI播客脚本出错(播客文案)', `<p><strong>Failed during podcast formatting:</strong> ${escapeHtml(error.message)}</p>${error.stack ? `<pre>${escapeHtml(error.stack)}</pre>` : ''}`, dateStr, true, selectedItemsParams, null, null, fullPromptForCall3_System, fullPromptForCall3_User);
const errorHtml = generateGenAiPageHtml(env, '生成AI播客脚本出错(播客文案)', `<p><strong>Failed during podcast formatting:</strong> ${escapeHtml(error.message)}</p>${error.stack ? `<pre>${escapeHtml(error.stack)}</pre>` : ''}`, dateStr, true, selectedItemsParams, null, null, fullPromptForCall3_System, fullPromptForCall3_User, null, outputOfCall1, null);
return new Response(errorHtml, { status: 500, headers: { 'Content-Type': 'text/html; charset=utf-8' } });
}
let finalAiResponseOut = `## Full: Podcast Formatting ` + `\n\n` + finalAiResponse;
@@ -71,7 +71,7 @@ export async function handleGenAIPodcastScript(request, env) {
console.log("Call 4 (Podcast Formatting) successful. Final output length:", finalAiResponse.length);
} catch (error) {
console.error("Error in Chat API Call 4 (Podcast Formatting):", error);
const errorHtml = generateGenAiPageHtml(env, '生成AI播客脚本出错(播客文案)', `<p><strong>Failed during podcast formatting:</strong> ${escapeHtml(error.message)}</p>${error.stack ? `<pre>${escapeHtml(error.stack)}</pre>` : ''}`, dateStr, true, selectedItemsParams, null, null, fullPromptForCall3_System, fullPromptForCall3_User);
const errorHtml = generateGenAiPageHtml(env, '生成AI播客脚本出错(播客文案)', `<p><strong>Failed during podcast formatting:</strong> ${escapeHtml(error.message)}</p>${error.stack ? `<pre>${escapeHtml(error.stack)}</pre>` : ''}`, dateStr, true, selectedItemsParams, null, null, fullPromptForCall3_System, fullPromptForCall3_User, null, outputOfCall1, null);
return new Response(errorHtml, { status: 500, headers: { 'Content-Type': 'text/html; charset=utf-8' } });
}
finalAiResponseOut += `\n\n` + `## Short: Podcast Formatting ` + `\n\n` + finalAiResponse;

6
wrangler.toml
View File

@@ -27,6 +27,12 @@ XIAOHU_FEED_ID = "151846580097413120"
XIAOHU_FETCH_PAGES = "2"
HGPAPERS_FEED_ID = "41359648680482832"
HGPAPERS_FETCH_PAGES = "2"
JIQIZHIXIN_FEED_ID = "41459996870678583"
JIQIZHIXIN_FETCH_PAGES = "1"
QBIT_FEED_ID = "58864180026527744"
QBIT_FETCH_PAGES = "1"
XINZHIYUAN_FEED_ID = "60901577013168128"
XINZHIYUAN_FETCH_PAGES = "1"
TWITTER_LIST_ID = "153028784690326528"
TWITTER_FETCH_PAGES = "5"
PROJECTS_API_URL = "https://git-trending.justlikemaki.vip/topone/?since=daily"