feat:add RSSHub RSSHubAccessKey

pkg/config/config.go

@@ -95,10 +95,11 @@ type LLM struct {
 }
 
 type Scrape struct {
-	Past           timeutil.Duration `yaml:"past,omitempty" json:"past,omitempty" desc:"The lookback time window for scraping feeds. e.g. 1h means only scrape feeds in the past 1 hour. Default: 3d"`
-	Interval       timeutil.Duration `yaml:"interval,omitempty" json:"interval,omitempty" desc:"How often to scrape each source, it is a global interval. e.g. 1h. Default: 1h"`
-	RSSHubEndpoint string            `yaml:"rsshub_endpoint,omitempty" json:"rsshub_endpoint,omitempty" desc:"The endpoint of the RSSHub. You can deploy your own RSSHub server or use the public one (https://docs.rsshub.app/guide/instances). e.g. https://rsshub.app. It is required when sources[].rss.rsshub_route_path is set."`
-	Sources        []ScrapeSource    `yaml:"sources,omitempty" json:"sources,omitempty" desc:"The sources for scraping feeds."`
+	Past            timeutil.Duration `yaml:"past,omitempty" json:"past,omitempty" desc:"The lookback time window for scraping feeds. e.g. 1h means only scrape feeds in the past 1 hour. Default: 3d"`
+	Interval        timeutil.Duration `yaml:"interval,omitempty" json:"interval,omitempty" desc:"How often to scrape each source, it is a global interval. e.g. 1h. Default: 1h"`
+	RSSHubEndpoint  string            `yaml:"rsshub_endpoint,omitempty" json:"rsshub_endpoint,omitempty" desc:"The endpoint of the RSSHub. You can deploy your own RSSHub server or use the public one (https://docs.rsshub.app/guide/instances). e.g. https://rsshub.app. It is required when sources[].rss.rsshub_route_path is set."`
+	RSSHubAccessKey string            `yaml:"rsshub_access_key,omitempty" json:"rsshub_access_key,omitempty" desc:"The access key for RSSHub. Used for access control. (see [RSSHub config](https://docs.rsshub.app/deploy/config#access-control-configurations))"`
+	Sources         []ScrapeSource    `yaml:"sources,omitempty" json:"sources,omitempty" desc:"The sources for scraping feeds."`
 }
 
 type Storage struct {
@@ -133,6 +134,7 @@ type ScrapeSource struct {
 type ScrapeSourceRSS struct {
 	URL             string `yaml:"url,omitempty" json:"url,omitempty" desc:"The URL of the RSS feed. e.g. http://localhost:1200/github/trending/daily/any. You can not set it when rsshub_route_path is set."`
 	RSSHubRoutePath string `yaml:"rsshub_route_path,omitempty" json:"rsshub_route_path,omitempty" desc:"The RSSHub route path of the RSS feed. e.g. github/trending/daily/any. It will be joined with the rsshub_endpoint as the final URL."`
+	// RSSHubAccessKey string `yaml:"rsshub_access_key,omitempty" json:"rsshub_access_key,omitempty" desc:"The access key for accessing RSSHub routes. If set, it will be appended as a query parameter to the URL."`
 }
 
 type RewriteRule struct {

pkg/scrape/manager.go

@@ -80,6 +80,7 @@ func (c *Config) From(app *config.App) {
 				URL:             app.Scrape.Sources[i].RSS.URL,
 				RSSHubEndpoint:  app.Scrape.RSSHubEndpoint,
 				RSSHubRoutePath: app.Scrape.Sources[i].RSS.RSSHubRoutePath,
+				RSSHubAccessKey: app.Scrape.RSSHubAccessKey,
 			}
 		}
 	}

pkg/scrape/scraper/rss.go

@@ -33,6 +33,7 @@ type ScrapeSourceRSS struct {
 	URL             string
 	RSSHubEndpoint  string
 	RSSHubRoutePath string
+	RSSHubAccessKey string
 }
 
 func (c *ScrapeSourceRSS) Validate() error {
@@ -46,6 +47,15 @@ func (c *ScrapeSourceRSS) Validate() error {
 		return errors.New("URL must be a valid HTTP/HTTPS URL")
 	}
 
+	// Append access key as query parameter if provided
+	if c.RSSHubAccessKey != "" && !strings.Contains(c.URL, "key=") {
+		if strings.Contains(c.URL, "?") {
+			c.URL += "&key=" + c.RSSHubAccessKey
+		} else {
+			c.URL += "?key=" + c.RSSHubAccessKey
+		}
+	}
+
 	return nil
 }
 

pkg/scrape/scraper/rss_test.go

@@ -122,6 +122,55 @@ func TestNewRSS(t *testing.T) {
 				},
 			},
 		},
+		{
+			Scenario: "Valid Configuration - RSSHub with Access Key",
+			Given:    "a valid configuration with RSSHub details and access key",
+			When:     "creating a new RSS reader",
+			Then:     "should succeed, construct the URL with access key, and return a valid reader",
+			GivenDetail: givenDetail{
+				config: &ScrapeSourceRSS{
+					RSSHubEndpoint:  "http://rsshub.app/",
+					RSSHubRoutePath: "/_/test",
+					RSSHubAccessKey: "testkey",
+				},
+			},
+			WhenDetail: whenDetail{},
+			ThenExpected: thenExpected{
+				wantErr: false,
+				validateFunc: func(t *testing.T, r reader) {
+					Expect(r).NotTo(BeNil())
+					rssReader, ok := r.(*rssReader)
+					Expect(ok).To(BeTrue())
+					Expect(rssReader.config.URL).To(Equal("http://rsshub.app/_/test?key=testkey"))
+					Expect(rssReader.config.RSSHubEndpoint).To(Equal("http://rsshub.app/"))
+					Expect(rssReader.config.RSSHubRoutePath).To(Equal("/_/test"))
+					Expect(rssReader.config.RSSHubAccessKey).To(Equal("testkey"))
+				},
+			},
+		},
+		{
+			Scenario: "Valid Configuration - URL with Access Key",
+			Given:    "a valid configuration with URL and access key",
+			When:     "creating a new RSS reader",
+			Then:     "should succeed, append access key to URL, and return a valid reader",
+			GivenDetail: givenDetail{
+				config: &ScrapeSourceRSS{
+					URL:             "http://example.com/feed",
+					RSSHubAccessKey: "testkey",
+				},
+			},
+			WhenDetail: whenDetail{},
+			ThenExpected: thenExpected{
+				wantErr: false,
+				validateFunc: func(t *testing.T, r reader) {
+					Expect(r).NotTo(BeNil())
+					rssReader, ok := r.(*rssReader)
+					Expect(ok).To(BeTrue())
+					Expect(rssReader.config.URL).To(Equal("http://example.com/feed?key=testkey"))
+					Expect(rssReader.config.RSSHubAccessKey).To(Equal("testkey"))
+				},
+			},
+		},
 	}
 
 	// --- Run tests ---