shen/zenfeed
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat:add RSSHub RSSHubAccessKey

Browse Source
This commit is contained in:
Twelveeee
2025-11-05 14:55:01 +00:00
parent 7cb8069d60
commit bc54cc852e
6 changed files with 68 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
docs/config-zh.md
View File

@@ -59,6 +59,7 @@
| `scrape.past` | `time.Duration` | 抓取 Feed 的回溯时间窗口。例如 `1h` 表示只抓取过去 1 小时的 Feed。 | `24h` | 否 |
| `scrape.interval` | `time.Duration` | 抓取每个源的频率 (全局默认值)。例如 `1h`。 | `1h` | 否 |
| `scrape.rsshub_endpoint` | `string` | RSSHub 的端点。你可以部署自己的 RSSHub 服务器或使用公共实例 (参见 [RSSHub 文档](https://docs.rsshub.app/guide/instances))。例如 `https://rsshub.app`。 | | 是 (如果使用了 `rsshub_route_path`) |
| `scrape.rsshub_access_key` | `string` | RSSHub 的访问密钥。用于访问控制。(详情见 [RSSHub文档访问控制](https://docs.rsshub.app/deploy/config#access-control-configurations)) | | 否 |
| `scrape.sources` | `对象列表` | 用于抓取 Feed 的源列表。详见下方的 **抓取源配置**。 | `[]` | 是 (至少一个) |
### 抓取源配置 (`scrape.sources[]`)

1
docs/config.md
View File

@@ -59,6 +59,7 @@ This section configures parameters related to the Jina AI Reader API, primarily
| `scrape.past` | `time.Duration` | Time window to look back when scraping feeds. E.g., `1h` means only scrape feeds from the past 1 hour. | `24h` | No |
| `scrape.interval` | `time.Duration` | Frequency to scrape each source (global default). E.g., `1h`. | `1h` | No |
| `scrape.rsshub_endpoint` | `string` | Endpoint for RSSHub. You can deploy your own RSSHub server or use a public instance (see [RSSHub Documentation](https://docs.rsshub.app/guide/instances)). E.g., `https://rsshub.app`. | | Yes (if `rsshub_route_path` is used) |
| `scrape.rsshub_access_key` | `string` | The access key for RSSHub. Used for access control. (see [RSSHub config](https://docs.rsshub.app/deploy/config#access-control-configurations))| | No |
| `scrape.sources` | `list of objects` | List of sources to scrape feeds from. See **Scrape Source Configuration** below. | `[]` | Yes (at least one) |
### Scrape Source Configuration (`scrape.sources[]`)

10
pkg/config/config.go
View File

@@ -95,10 +95,11 @@ type LLM struct {
}
type Scrape struct {
Past timeutil.Duration `yaml:"past,omitempty" json:"past,omitempty" desc:"The lookback time window for scraping feeds. e.g. 1h means only scrape feeds in the past 1 hour. Default: 3d"`
Interval timeutil.Duration `yaml:"interval,omitempty" json:"interval,omitempty" desc:"How often to scrape each source, it is a global interval. e.g. 1h. Default: 1h"`
RSSHubEndpoint string `yaml:"rsshub_endpoint,omitempty" json:"rsshub_endpoint,omitempty" desc:"The endpoint of the RSSHub. You can deploy your own RSSHub server or use the public one (https://docs.rsshub.app/guide/instances). e.g. https://rsshub.app. It is required when sources[].rss.rsshub_route_path is set."`
Sources []ScrapeSource `yaml:"sources,omitempty" json:"sources,omitempty" desc:"The sources for scraping feeds."`
Past timeutil.Duration `yaml:"past,omitempty" json:"past,omitempty" desc:"The lookback time window for scraping feeds. e.g. 1h means only scrape feeds in the past 1 hour. Default: 3d"`
Interval timeutil.Duration `yaml:"interval,omitempty" json:"interval,omitempty" desc:"How often to scrape each source, it is a global interval. e.g. 1h. Default: 1h"`
RSSHubEndpoint string `yaml:"rsshub_endpoint,omitempty" json:"rsshub_endpoint,omitempty" desc:"The endpoint of the RSSHub. You can deploy your own RSSHub server or use the public one (https://docs.rsshub.app/guide/instances). e.g. https://rsshub.app. It is required when sources[].rss.rsshub_route_path is set."`
RSSHubAccessKey string `yaml:"rsshub_access_key,omitempty" json:"rsshub_access_key,omitempty" desc:"The access key for RSSHub. Used for access control. (see [RSSHub config](https://docs.rsshub.app/deploy/config#access-control-configurations))"`
Sources []ScrapeSource `yaml:"sources,omitempty" json:"sources,omitempty" desc:"The sources for scraping feeds."`
}
type Storage struct {
@@ -133,6 +134,7 @@ type ScrapeSource struct {
type ScrapeSourceRSS struct {
URL string `yaml:"url,omitempty" json:"url,omitempty" desc:"The URL of the RSS feed. e.g. http://localhost:1200/github/trending/daily/any. You can not set it when rsshub_route_path is set."`
RSSHubRoutePath string `yaml:"rsshub_route_path,omitempty" json:"rsshub_route_path,omitempty" desc:"The RSSHub route path of the RSS feed. e.g. github/trending/daily/any. It will be joined with the rsshub_endpoint as the final URL."`
// RSSHubAccessKey string `yaml:"rsshub_access_key,omitempty" json:"rsshub_access_key,omitempty" desc:"The access key for accessing RSSHub routes. If set, it will be appended as a query parameter to the URL."`
}
type RewriteRule struct {

1
pkg/scrape/manager.go
View File

@@ -80,6 +80,7 @@ func (c *Config) From(app *config.App) {
URL: app.Scrape.Sources[i].RSS.URL,
RSSHubEndpoint: app.Scrape.RSSHubEndpoint,
RSSHubRoutePath: app.Scrape.Sources[i].RSS.RSSHubRoutePath,
RSSHubAccessKey: app.Scrape.RSSHubAccessKey,
}
}
}

10
pkg/scrape/scraper/rss.go
View File

@@ -33,6 +33,7 @@ type ScrapeSourceRSS struct {
URL string
RSSHubEndpoint string
RSSHubRoutePath string
RSSHubAccessKey string
}
func (c *ScrapeSourceRSS) Validate() error {
@@ -46,6 +47,15 @@ func (c *ScrapeSourceRSS) Validate() error {
return errors.New("URL must be a valid HTTP/HTTPS URL")
}
// Append access key as query parameter if provided
if c.RSSHubAccessKey != "" && !strings.Contains(c.URL, "key=") {
if strings.Contains(c.URL, "?") {
c.URL += "&key=" + c.RSSHubAccessKey
} else {
c.URL += "?key=" + c.RSSHubAccessKey
}
}
return nil
}

49
pkg/scrape/scraper/rss_test.go
View File

@@ -122,6 +122,55 @@ func TestNewRSS(t *testing.T) {
},
},
},
{
Scenario: "Valid Configuration - RSSHub with Access Key",
Given: "a valid configuration with RSSHub details and access key",
When: "creating a new RSS reader",
Then: "should succeed, construct the URL with access key, and return a valid reader",
GivenDetail: givenDetail{
config: &ScrapeSourceRSS{
RSSHubEndpoint: "http://rsshub.app/",
RSSHubRoutePath: "/_/test",
RSSHubAccessKey: "testkey",
},
},
WhenDetail: whenDetail{},
ThenExpected: thenExpected{
wantErr: false,
validateFunc: func(t *testing.T, r reader) {
Expect(r).NotTo(BeNil())
rssReader, ok := r.(*rssReader)
Expect(ok).To(BeTrue())
Expect(rssReader.config.URL).To(Equal("http://rsshub.app/_/test?key=testkey"))
Expect(rssReader.config.RSSHubEndpoint).To(Equal("http://rsshub.app/"))
Expect(rssReader.config.RSSHubRoutePath).To(Equal("/_/test"))
Expect(rssReader.config.RSSHubAccessKey).To(Equal("testkey"))
},
},
},
{
Scenario: "Valid Configuration - URL with Access Key",
Given: "a valid configuration with URL and access key",
When: "creating a new RSS reader",
Then: "should succeed, append access key to URL, and return a valid reader",
GivenDetail: givenDetail{
config: &ScrapeSourceRSS{
URL: "http://example.com/feed",
RSSHubAccessKey: "testkey",
},
},
WhenDetail: whenDetail{},
ThenExpected: thenExpected{
wantErr: false,
validateFunc: func(t *testing.T, r reader) {
Expect(r).NotTo(BeNil())
rssReader, ok := r.(*rssReader)
Expect(ok).To(BeTrue())
Expect(rssReader.config.URL).To(Equal("http://example.com/feed?key=testkey"))
Expect(rssReader.config.RSSHubAccessKey).To(Equal("testkey"))
},
},
},
}
// --- Run tests ---