This commit is contained in:
glidea
2025-04-19 15:50:26 +08:00
commit 8b33df8a05
109 changed files with 24407 additions and 0 deletions

View File

@@ -0,0 +1,161 @@
// Copyright (C) 2025 wangyusong
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package channel
import (
"context"
"github.com/pkg/errors"
"github.com/glidea/zenfeed/pkg/component"
"github.com/glidea/zenfeed/pkg/notify/route"
"github.com/glidea/zenfeed/pkg/telemetry"
telemetrymodel "github.com/glidea/zenfeed/pkg/telemetry/model"
)
// --- Interface code block ---
type Channel interface {
component.Component
sender
}
type sender interface {
Send(ctx context.Context, receiver Receiver, group *route.FeedGroup) error
}
type Config struct {
Email *Email
}
func (c *Config) Validate() error {
if c.Email != nil {
if err := c.Email.Validate(); err != nil {
return errors.Wrap(err, "validate email")
}
}
return nil
}
type Receiver struct {
Email string
Webhook *WebhookReceiver
}
func (r *Receiver) Validate() error {
if r.Email == "" && r.Webhook == nil {
return errors.New("email or webhook is required")
}
if r.Email != "" && r.Webhook != nil {
return errors.New("email and webhook cannot both be set")
}
if r.Webhook != nil {
if err := r.Webhook.Validate(); err != nil {
return errors.Wrap(err, "validate webhook")
}
}
return nil
}
type Dependencies struct{}
// --- Factory code block ---
type Factory component.Factory[Channel, Config, Dependencies]
func NewFactory(mockOn ...component.MockOption) Factory {
if len(mockOn) > 0 {
return component.FactoryFunc[Channel, Config, Dependencies](
func(instance string, config *Config, dependencies Dependencies) (Channel, error) {
m := &mockChannel{}
component.MockOptions(mockOn).Apply(&m.Mock)
return m, nil
},
)
}
return component.FactoryFunc[Channel, Config, Dependencies](new)
}
func new(instance string, config *Config, dependencies Dependencies) (Channel, error) {
if err := config.Validate(); err != nil {
return nil, errors.Wrap(err, "validate config")
}
var email sender
if config.Email != nil {
var err error
email, err = newEmail(config.Email, dependencies)
if err != nil {
return nil, errors.Wrap(err, "new email")
}
}
return &aggrChannel{
Base: component.New(&component.BaseConfig[Config, Dependencies]{
Name: "NotifyChannel",
Instance: instance,
Config: config,
Dependencies: dependencies,
}),
email: email,
webhook: newWebhook(),
}, nil
}
// --- Implementation code block ---
type aggrChannel struct {
*component.Base[Config, Dependencies]
email, webhook sender
}
func (c *aggrChannel) Send(ctx context.Context, receiver Receiver, group *route.FeedGroup) error {
if receiver.Email != "" && c.email != nil {
return c.send(ctx, receiver, group, c.email, "email")
}
// if receiver.Webhook != nil && c.webhook != nil {
// TODO: temporarily disable webhook to reduce copyright risks.
// return c.send(ctx, receiver, group, c.webhook, "webhook")
// }
return nil
}
func (c *aggrChannel) send(
ctx context.Context,
receiver Receiver,
group *route.FeedGroup,
sender sender,
senderName string,
) (err error) {
ctx = telemetry.StartWith(ctx, append(c.TelemetryLabels(), telemetrymodel.KeyOperation, "channel", senderName)...)
defer func() { telemetry.End(ctx, err) }()
if err := sender.Send(ctx, receiver, group); err != nil {
return errors.Wrap(err, "send")
}
return nil
}
type mockChannel struct {
component.Mock
}
func (m *mockChannel) Send(ctx context.Context, receiver Receiver, group *route.FeedGroup) error {
args := m.Called(ctx, receiver, group)
return args.Error(0)
}

382
pkg/notify/channel/email.go Normal file
View File

@@ -0,0 +1,382 @@
// Copyright (C) 2025 wangyusong
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package channel
import (
"context"
"fmt"
"net"
"strconv"
"strings"
"text/template"
"time"
"github.com/pkg/errors"
"gopkg.in/gomail.v2"
"github.com/glidea/zenfeed/pkg/model"
"github.com/glidea/zenfeed/pkg/notify/route"
"github.com/glidea/zenfeed/pkg/util/buffer"
textconvert "github.com/glidea/zenfeed/pkg/util/text_convert"
timeutil "github.com/glidea/zenfeed/pkg/util/time"
)
type Email struct {
SmtpEndpoint string
host string
port int
From string
Password string
FeedMarkdownTemplate string
feedMakrdownTemplate *template.Template
FeedHTMLSnippetTemplate string
feedHTMLSnippetTemplate *template.Template
}
func (c *Email) Validate() error {
if c.SmtpEndpoint == "" {
return errors.New("email.smtp_endpoint is required")
}
parts := strings.Split(c.SmtpEndpoint, ":")
if len(parts) != 2 {
return errors.New("email.smtp_endpoint must be in the format host:port")
}
c.host = parts[0]
var err error
c.port, err = strconv.Atoi(parts[1])
if err != nil {
return errors.Wrap(err, "invalid email.smtp_endpoint")
}
if c.From == "" {
return errors.New("email.from is required")
}
if c.FeedMarkdownTemplate == "" {
c.FeedMarkdownTemplate = fmt.Sprintf("{{.%s}}", model.LabelContent)
}
t, err := template.New("").Parse(c.FeedMarkdownTemplate)
if err != nil {
return errors.Wrap(err, "parse feed markdown template")
}
c.feedMakrdownTemplate = t
if c.FeedHTMLSnippetTemplate != "" {
t, err := template.New("").Parse(c.FeedHTMLSnippetTemplate)
if err != nil {
return errors.Wrap(err, "parse feed html snippet template")
}
c.feedHTMLSnippetTemplate = t
}
return nil
}
func newEmail(c *Email, dependencies Dependencies) (sender, error) {
host, portStr, err := net.SplitHostPort(c.SmtpEndpoint)
if err != nil {
return nil, errors.Wrap(err, "split host port")
}
port, err := strconv.Atoi(portStr)
if err != nil {
return nil, errors.Wrap(err, "convert port to int")
}
return &email{
config: c,
dependencies: dependencies,
dialer: gomail.NewDialer(host, port, c.From, c.Password),
}, nil
}
type email struct {
config *Email
dependencies Dependencies
dialer *gomail.Dialer
}
func (e *email) Send(ctx context.Context, receiver Receiver, group *route.FeedGroup) error {
email, err := e.buildEmail(receiver, group)
if err != nil {
return errors.Wrap(err, "build email")
}
if err := e.dialer.DialAndSend(email); err != nil {
return errors.Wrap(err, "send email")
}
return nil
}
func (e *email) buildEmail(receiver Receiver, group *route.FeedGroup) (*gomail.Message, error) {
m := gomail.NewMessage()
m.SetHeader("From", e.config.From)
m.SetHeader("To", receiver.Email)
m.SetHeader("Subject", group.Name)
body, err := e.buildBodyHTML(group.Feeds)
if err != nil {
return nil, errors.Wrap(err, "build email body HTML")
}
m.SetBody("text/html", string(body))
return m, nil
}
func (e *email) buildBodyHTML(feeds []*route.Feed) ([]byte, error) {
bodyBuf := buffer.Get()
defer buffer.Put(bodyBuf)
// Write HTML header.
if err := e.writeHTMLHeader(bodyBuf); err != nil {
return nil, errors.Wrap(err, "write HTML header")
}
// Write each feed content.
for i, feed := range feeds {
if err := e.writeFeedContent(bodyBuf, feed); err != nil {
return nil, errors.Wrap(err, "write feed content")
}
// Add separator (except the last feed).
if i < len(feeds)-1 {
if err := e.writeSeparator(bodyBuf); err != nil {
return nil, errors.Wrap(err, "write separator")
}
}
}
// Write disclaimer and HTML footer.
if err := e.writeDisclaimer(bodyBuf); err != nil {
return nil, errors.Wrap(err, "write disclaimer")
}
if err := e.writeHTMLFooter(bodyBuf); err != nil {
return nil, errors.Wrap(err, "write HTML footer")
}
return bodyBuf.Bytes(), nil
}
func (e *email) writeHTMLHeader(buf *buffer.Bytes) error {
_, err := buf.WriteString(`<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Summary</title>
</head>
<body style="margin:0; padding:0; background-color:#f5f7fa; font-family:'Google Sans',Roboto,Arial,sans-serif;">
<div style="max-width:650px; margin:0 auto; padding:30px 20px;">
<div style="background-color:#ffffff; border-radius:12px; box-shadow:0 5px 15px rgba(0,0,0,0.08); padding:30px; margin-bottom:30px;">`)
return err
}
const timeLayout = "01-02 15:04"
func (e *email) writeFeedContent(buf *buffer.Bytes, feed *route.Feed) error {
// Write title and source information.
if err := e.writeFeedHeader(buf, feed); err != nil {
return errors.Wrap(err, "write feed header")
}
// Write content.
if err := e.writeFeedBody(buf, feed); err != nil {
return errors.Wrap(err, "write feed body")
}
// Write related articles.
if len(feed.Related) > 0 {
if err := e.writeRelateds(buf, feed.Related); err != nil {
return errors.Wrap(err, "write relateds")
}
}
if _, err := buf.WriteString(`
</div>`); err != nil {
return errors.Wrap(err, "write feed footer")
}
return nil
}
func (e *email) writeFeedHeader(buf *buffer.Bytes, feed *route.Feed) error {
typ := feed.Labels.Get(model.LabelType)
source := feed.Labels.Get(model.LabelSource)
title := feed.Labels.Get(model.LabelTitle)
link := feed.Labels.Get(model.LabelLink)
pubTimeI, _ := timeutil.Parse(feed.Labels.Get(model.LabelPubTime))
pubTime := pubTimeI.In(time.Local).Format(timeLayout)
scrapeTime := feed.Time.In(time.Local).Format(timeLayout)
if _, err := fmt.Fprintf(buf, `
<div style="margin-bottom:30px;">
<h2 style="font-size:22px; font-weight:500; color:#202124; margin:0 0 10px 0;">
%s
</h2>
<p style="font-size:14px; color:#5f6368; margin:0 0 15px 0;">Source: <a href="%s" style="color:#1a73e8; text-decoration:none;">%s/%s</a></p>
<p style="font-size:14px; color:#5f6368; margin:0 0 15px 0;">Published: %s | Scraped: %s</p>`,
title, link, typ, source, pubTime, scrapeTime); err != nil {
return errors.Wrap(err, "write feed header")
}
return nil
}
func (e *email) writeFeedBody(buf *buffer.Bytes, feed *route.Feed) error {
if _, err := buf.WriteString(`<div style="font-size:15px; color:#444; line-height:1.7;">
<style>
img {
max-width: 100%;
height: auto;
display: block;
margin: 10px 0;
}
pre, code {
white-space: pre-wrap;
word-wrap: break-word;
overflow-wrap: break-word;
max-width: 100%;
overflow-x: auto;
}
table {
max-width: 100%;
overflow-x: auto;
display: block;
}
</style>`); err != nil {
return errors.Wrap(err, "write feed body header")
}
if _, err := e.renderFeedContent(buf, feed); err != nil {
return errors.Wrap(err, "render feed content")
}
if _, err := buf.WriteString(`</div>`); err != nil {
return errors.Wrap(err, "write feed body footer")
}
return nil
}
func (e *email) renderFeedContent(buf *buffer.Bytes, feed *route.Feed) (n int, err error) {
if e.config.feedHTMLSnippetTemplate != nil {
n, err = e.renderHTMLContent(buf, feed)
if err == nil && n > 0 {
return
}
}
// Fallback to markdown.
return e.renderMarkdownContent(buf, feed)
}
func (e *email) renderHTMLContent(buf *buffer.Bytes, feed *route.Feed) (n int, err error) {
oldN := buf.Len()
if err := e.config.feedHTMLSnippetTemplate.Execute(buf, feed.Labels.Map()); err != nil {
return 0, errors.Wrap(err, "execute feed HTML template")
}
return buf.Len() - oldN, nil
}
func (e *email) renderMarkdownContent(buf *buffer.Bytes, feed *route.Feed) (n int, err error) {
oldN := buf.Len()
tempBuf := buffer.Get()
defer buffer.Put(tempBuf)
if err := e.config.feedMakrdownTemplate.Execute(tempBuf, feed.Labels.Map()); err != nil {
return 0, errors.Wrap(err, "execute feed markdown template")
}
contentMarkdown := tempBuf.Bytes()
contentHTML, err := textconvert.MarkdownToHTML(contentMarkdown)
if err != nil {
return 0, errors.Wrap(err, "markdown to HTML")
}
if _, err := buf.Write(contentHTML); err != nil {
return 0, errors.Wrap(err, "write content HTML")
}
return buf.Len() - oldN, nil
}
func (e *email) writeRelateds(buf *buffer.Bytes, related []*route.Feed) error {
if _, err := buf.WriteString(`
<div style="margin-top:20px; padding-top:15px; border-top:1px solid #f1f3f4;">
<p style="font-size:16px; font-weight:500; color:#1a73e8; margin:0 0 10px 0;">Related:</p>`); err != nil {
return errors.Wrapf(err, "write relateds header")
}
for _, f := range related {
relTyp := f.Labels.Get(model.LabelType)
relSource := f.Labels.Get(model.LabelSource)
relTitle := f.Labels.Get(model.LabelTitle)
relLink := f.Labels.Get(model.LabelLink)
if _, err := fmt.Fprintf(buf, `
<div style="margin-bottom:8px; padding-left:15px; position:relative;">
<span style="position:absolute; left:0; top:8px; width:6px; height:6px; background-color:#4285f4; border-radius:50%%;"></span>
<a href="%s" style="color:#1a73e8; text-decoration:none;">%s/%s: %s</a>
</div>`, relLink, relTyp, relSource, relTitle); err != nil {
return errors.Wrapf(err, "write relateds item")
}
}
if _, err := buf.WriteString(`
</div>`); err != nil {
return errors.Wrapf(err, "write relateds footer")
}
return nil
}
func (e *email) writeSeparator(buf *buffer.Bytes) error {
_, err := buf.WriteString(`
<hr style="border:0; height:1px; background:linear-gradient(to right, rgba(0,0,0,0.03), rgba(0,0,0,0.1), rgba(0,0,0,0.03)); margin:25px 0;">`)
return err
}
func (e *email) writeDisclaimer(buf *buffer.Bytes) error {
_, err := buf.WriteString(`
<div style="margin-top:40px; padding:25px; border-top:2px solid #e0e0e0; font-family:'Google Sans',Roboto,Arial,sans-serif; font-size:14px; line-height:1.8; color:#4a4a4a; text-align:center; background-color:#f8f9fa; border-radius:8px;">
<p style="margin:0 0 15px 0;">
<strong style="color:#1a73e8; font-size:15px;">免责声明 / Disclaimer</strong><br>
<span style="display:block; margin-top:8px;">本邮件内容仅用于个人概括性学习和理解,版权归原作者所有。</span>
<span style="display:block; color:#666;">This email content is for personal learning and understanding purposes only. All rights reserved to the original author.</span>
</p>
<p style="margin:0 0 15px 0;">
<strong style="color:#ea4335; font-size:15px;">严禁二次分发或传播!!!<br>NO redistribution or sharing!!!</strong>
</p>
<p style="margin:0; font-size:13px; color:#666;">
如有侵权,请联系 / For copyright issues, please contact:<br>
<a href="mailto:ysking7402@gmail.com" style="color:#1a73e8; text-decoration:none;">ysking7402@gmail.com</a>
</p>
</div>`)
return err
}
func (e *email) writeHTMLFooter(buf *buffer.Bytes) error {
_, err := buf.WriteString(`
</div>
</div>
</body>
</html>`)
return err
}

View File

@@ -0,0 +1,86 @@
// Copyright (C) 2025 wangyusong
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package channel
import (
"bytes"
"context"
"encoding/json"
"net/http"
"github.com/pkg/errors"
"github.com/glidea/zenfeed/pkg/model"
"github.com/glidea/zenfeed/pkg/notify/route"
runtimeutil "github.com/glidea/zenfeed/pkg/util/runtime"
)
type WebhookReceiver struct {
URL string `json:"url"`
}
func (r *WebhookReceiver) Validate() error {
if r.URL == "" {
return errors.New("webhook.url is required")
}
return nil
}
type webhookBody struct {
Group string `json:"group"`
Labels model.Labels `json:"labels"`
Feeds []*route.Feed `json:"feeds"`
}
func newWebhook() sender {
return &webhook{
httpClient: &http.Client{},
}
}
type webhook struct {
httpClient *http.Client
}
func (w *webhook) Send(ctx context.Context, receiver Receiver, group *route.FeedGroup) error {
// Prepare request.
body := &webhookBody{
Group: group.Name,
Labels: group.Labels,
Feeds: group.Feeds,
}
b := runtimeutil.Must1(json.Marshal(body))
req, err := http.NewRequestWithContext(ctx, http.MethodPost, receiver.Webhook.URL, bytes.NewReader(b))
if err != nil {
return errors.Wrap(err, "create request")
}
req.Header.Set("Content-Type", "application/json")
// Send request.
resp, err := w.httpClient.Do(req)
if err != nil {
return errors.Wrap(err, "send request")
}
defer func() { _ = resp.Body.Close() }()
// Handle response.
if resp.StatusCode != http.StatusOK {
return errors.New("send request")
}
return nil
}

463
pkg/notify/notify.go Normal file
View File

@@ -0,0 +1,463 @@
// Copyright (C) 2025 wangyusong
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package notify
import (
"context"
"fmt"
"reflect"
"runtime"
"sync"
"time"
"github.com/pkg/errors"
"github.com/glidea/zenfeed/pkg/component"
"github.com/glidea/zenfeed/pkg/config"
"github.com/glidea/zenfeed/pkg/notify/channel"
"github.com/glidea/zenfeed/pkg/notify/route"
"github.com/glidea/zenfeed/pkg/schedule/rule"
"github.com/glidea/zenfeed/pkg/storage/kv"
"github.com/glidea/zenfeed/pkg/telemetry"
"github.com/glidea/zenfeed/pkg/telemetry/log"
telemetrymodel "github.com/glidea/zenfeed/pkg/telemetry/model"
timeutil "github.com/glidea/zenfeed/pkg/util/time"
)
// --- Interface code block ---
type Notifier interface {
component.Component
config.Watcher
}
type Config struct {
Route route.Config
Receivers Receivers
Channels channel.Config
}
func (c *Config) Validate() error {
if err := (&c.Route).Validate(); err != nil {
return errors.Wrap(err, "invalid route")
}
if err := (&c.Receivers).Validate(); err != nil {
return errors.Wrap(err, "invalid receivers")
}
if err := (&c.Channels).Validate(); err != nil {
return errors.Wrap(err, "invalid channels")
}
return nil
}
func (c *Config) From(app *config.App) *Config {
c.Route = route.Config{
Route: route.Route{
GroupBy: app.Notify.Route.GroupBy,
CompressByRelatedThreshold: app.Notify.Route.CompressByRelatedThreshold,
Receivers: app.Notify.Route.Receivers,
},
}
for i := range app.Notify.Route.SubRoutes {
c.Route.SubRoutes = append(c.Route.SubRoutes, convertSubRoute(&app.Notify.Route.SubRoutes[i]))
}
c.Receivers = make(Receivers, len(app.Notify.Receivers))
for i := range app.Notify.Receivers {
c.Receivers[i] = Receiver{
Name: app.Notify.Receivers[i].Name,
}
if app.Notify.Receivers[i].Email != "" {
c.Receivers[i].Email = app.Notify.Receivers[i].Email
}
// if app.Notify.Receivers[i].Webhook != nil {
// c.Receivers[i].Webhook = &channel.WebhookReceiver{URL: app.Notify.Receivers[i].Webhook.URL}
// }
}
c.Channels = channel.Config{}
if app.Notify.Channels.Email != nil {
c.Channels.Email = &channel.Email{
SmtpEndpoint: app.Notify.Channels.Email.SmtpEndpoint,
From: app.Notify.Channels.Email.From,
Password: app.Notify.Channels.Email.Password,
FeedMarkdownTemplate: app.Notify.Channels.Email.FeedMarkdownTemplate,
FeedHTMLSnippetTemplate: app.Notify.Channels.Email.FeedHTMLSnippetTemplate,
}
}
return c
}
func convertSubRoute(from *config.NotifySubRoute) *route.SubRoute {
to := &route.SubRoute{
Route: route.Route{
GroupBy: from.GroupBy,
CompressByRelatedThreshold: from.CompressByRelatedThreshold,
Receivers: from.Receivers,
},
}
to.Matchers = from.Matchers
to.Receivers = from.Receivers
for i := range from.SubRoutes {
to.SubRoutes = append(to.SubRoutes, convertSubRoute(&from.SubRoutes[i]))
}
return to
}
type Receivers []Receiver
func (rs Receivers) Validate() error {
names := make(map[string]bool)
for i := range rs {
r := &rs[i]
if err := r.Validate(); err != nil {
return errors.Wrap(err, "invalid receiver")
}
if _, ok := names[r.Name]; ok {
return errors.New("receiver name must be unique")
}
names[r.Name] = true
}
return nil
}
func (r Receivers) get(name string) *Receiver {
for _, receiver := range r {
if receiver.Name == name {
return &receiver
}
}
return nil
}
type Receiver struct {
channel.Receiver
Name string
}
func (r *Receiver) Validate() error {
if r.Name == "" {
return errors.New("name is required")
}
if err := (&r.Receiver).Validate(); err != nil {
return errors.Wrap(err, "invalid receiver")
}
return nil
}
type Dependencies struct {
In <-chan *rule.Result
RelatedScore func(a, b [][]float32) (float32, error)
RouterFactory route.Factory
ChannelFactory channel.Factory
KVStorage kv.Storage
}
// --- Factory code block ---
type Factory component.Factory[Notifier, config.App, Dependencies]
func NewFactory(mockOn ...component.MockOption) Factory {
if len(mockOn) > 0 {
return component.FactoryFunc[Notifier, config.App, Dependencies](
func(instance string, app *config.App, dependencies Dependencies) (Notifier, error) {
m := &mockNotifier{}
component.MockOptions(mockOn).Apply(&m.Mock)
return m, nil
},
)
}
return component.FactoryFunc[Notifier, config.App, Dependencies](new)
}
func new(instance string, app *config.App, dependencies Dependencies) (Notifier, error) {
config := &Config{}
config.From(app)
if err := config.Validate(); err != nil {
return nil, errors.Wrap(err, "invalid config")
}
n := &notifier{
Base: component.New(&component.BaseConfig[Config, Dependencies]{
Name: "Notifier",
Instance: instance,
Config: config,
Dependencies: dependencies,
}),
channelSendWork: make(chan sendWork, 100),
}
router, err := n.newRouter(&config.Route)
if err != nil {
return nil, errors.Wrap(err, "create router")
}
n.router = router
channel, err := n.newChannel(&config.Channels)
if err != nil {
return nil, errors.Wrap(err, "create channel")
}
n.channel = channel
return n, nil
}
// --- Implementation code block ---
type notifier struct {
*component.Base[Config, Dependencies]
router route.Router
channel channel.Channel
channelSendWork chan sendWork
mu sync.RWMutex
}
var sendConcurrency = runtime.NumCPU() * 2
func (n *notifier) Run() (err error) {
ctx := telemetry.StartWith(n.Context(), append(n.TelemetryLabels(), telemetrymodel.KeyOperation, "Run")...)
defer func() { telemetry.End(ctx, err) }()
if err := component.RunUntilReady(n.Context(), n.router, 10*time.Second); err != nil {
return errors.Wrap(err, "router not ready")
}
if err := component.RunUntilReady(n.Context(), n.channel, 10*time.Second); err != nil {
return errors.Wrap(err, "channel not ready")
}
for i := range sendConcurrency {
go n.sendWorker(i)
}
n.MarkReady()
for {
select {
case <-ctx.Done():
return nil
case result := <-n.Dependencies().In:
n.handle(ctx, result)
}
}
}
func (n *notifier) Close() error {
if err := n.Base.Close(); err != nil {
return errors.Wrap(err, "close base")
}
if err := n.router.Close(); err != nil {
return errors.Wrap(err, "close router")
}
if err := n.channel.Close(); err != nil {
return errors.Wrap(err, "close channel")
}
close(n.channelSendWork)
return nil
}
func (n *notifier) Reload(app *config.App) error {
newConfig := &Config{}
newConfig.From(app)
if err := newConfig.Validate(); err != nil {
return errors.Wrap(err, "invalid config")
}
if reflect.DeepEqual(n.Config(), newConfig) {
log.Debug(n.Context(), "no changes in notify config")
return nil
}
router, err := n.newRouter(&route.Config{Route: newConfig.Route.Route})
if err != nil {
return errors.Wrap(err, "create router")
}
if component.RunUntilReady(n.Context(), router, 10*time.Second) != nil {
return errors.New("router not ready")
}
channel, err := n.newChannel(&channel.Config{Email: newConfig.Channels.Email})
if err != nil {
return errors.Wrap(err, "create email")
}
if component.RunUntilReady(n.Context(), channel, 10*time.Second) != nil {
return errors.New("channel not ready")
}
if err := n.router.Close(); err != nil {
log.Error(n.Context(), errors.Wrap(err, "close router"))
}
if err := n.channel.Close(); err != nil {
log.Error(n.Context(), errors.Wrap(err, "close channel"))
}
n.mu.Lock()
defer n.mu.Unlock()
n.SetConfig(newConfig)
n.router = router
n.channel = channel
return nil
}
func (n *notifier) newRouter(config *route.Config) (route.Router, error) {
return n.Dependencies().RouterFactory.New(
n.Instance(),
config,
route.Dependencies{RelatedScore: n.Dependencies().RelatedScore},
)
}
func (n *notifier) newChannel(config *channel.Config) (channel.Channel, error) {
return n.Dependencies().ChannelFactory.New(
n.Instance(),
config,
channel.Dependencies{},
)
}
func (n *notifier) handle(ctx context.Context, result *rule.Result) {
n.mu.RLock()
router := n.router
n.mu.RUnlock()
groups, err := router.Route(result)
if err != nil {
// We don't retry in notifier, retry should be upstream.
log.Error(ctx, errors.Wrap(err, "route"))
return
}
for _, group := range groups {
for i := range group.Receivers {
n.trySummitSendWork(ctx, group, group.Receivers[i])
}
}
}
func (n *notifier) trySummitSendWork(ctx context.Context, group *route.Group, receiverName string) {
config := n.Config()
receiver := config.Receivers.get(receiverName)
if receiver == nil {
log.Error(ctx, errors.New("receiver not found"), "receiver", receiverName)
return
}
if n.isSent(ctx, &group.FeedGroup, *receiver) {
log.Debug(ctx, "already sent")
return
}
n.channelSendWork <- sendWork{
group: &group.FeedGroup,
receiver: *receiver,
}
}
func (n *notifier) sendWorker(i int) {
for {
select {
case <-n.Context().Done():
return
case work := <-n.channelSendWork:
workCtx := telemetry.StartWith(n.Context(),
append(n.TelemetryLabels(),
telemetrymodel.KeyOperation, "Run",
"worker", i,
"group", work.group.Name,
"time", timeutil.Format(work.group.Time),
"receiver", work.receiver.Name,
)...,
)
defer func() { telemetry.End(workCtx, nil) }()
workCtx, cancel := context.WithTimeout(workCtx, 30*time.Second)
defer cancel()
if err := n.duplicateSend(workCtx, work); err != nil {
log.Error(workCtx, err, "duplicate send")
continue
}
log.Info(workCtx, "send success")
}
}
}
func (n *notifier) duplicateSend(ctx context.Context, work sendWork) error {
if n.isSent(ctx, work.group, work.receiver) { // Double check.
return nil
}
if err := n.send(ctx, work); err != nil {
return errors.Wrap(err, "send")
}
if err := n.markSent(ctx, work.group, work.receiver); err != nil {
log.Error(ctx, errors.Wrap(err, "set nlog, may duplicate sending in next time"))
}
return nil
}
func (n *notifier) send(ctx context.Context, work sendWork) error {
n.mu.RLock()
channel := n.channel
n.mu.RUnlock()
return channel.Send(ctx, work.receiver.Receiver, work.group)
}
var nlogKey = func(group *route.FeedGroup, receiver Receiver) string {
return fmt.Sprintf("notifier.group.%s.receiver.%s", group.Name, receiver.Name)
}
func (n *notifier) isSent(ctx context.Context, group *route.FeedGroup, receiver Receiver) bool {
_, err := n.Dependencies().KVStorage.Get(ctx, nlogKey(group, receiver))
switch {
case err == nil:
return true // Already sent.
case errors.Is(err, kv.ErrNotFound):
return false
default:
log.Warn(ctx, errors.Wrap(err, "get nlog, continue sending"))
return false
}
}
func (n *notifier) markSent(ctx context.Context, group *route.FeedGroup, receiver Receiver) error {
return n.Dependencies().KVStorage.Set(ctx, nlogKey(group, receiver), timeutil.Format(time.Now()), timeutil.Day)
}
type sendWork struct {
group *route.FeedGroup
receiver Receiver
}
type mockNotifier struct {
component.Mock
}
func (m *mockNotifier) Reload(app *config.App) error {
return m.Called(app).Error(0)
}

358
pkg/notify/route/route.go Normal file
View File

@@ -0,0 +1,358 @@
// Copyright (C) 2025 wangyusong
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package route
import (
"fmt"
"sort"
"strings"
"time"
"github.com/pkg/errors"
"k8s.io/utils/ptr"
"github.com/glidea/zenfeed/pkg/component"
"github.com/glidea/zenfeed/pkg/model"
"github.com/glidea/zenfeed/pkg/schedule/rule"
"github.com/glidea/zenfeed/pkg/storage/feed/block"
timeutil "github.com/glidea/zenfeed/pkg/util/time"
)
// --- Interface code block ---
type Router interface {
component.Component
Route(result *rule.Result) (groups []*Group, err error)
}
type Config struct {
Route
}
type Route struct {
GroupBy []string
CompressByRelatedThreshold *float32
Receivers []string
SubRoutes SubRoutes
}
type SubRoutes []*SubRoute
func (s SubRoutes) Match(feed *block.FeedVO) *SubRoute {
for _, sub := range s {
if matched := sub.Match(feed); matched != nil {
return matched
}
}
return nil
}
type SubRoute struct {
Route
Matchers []string
matchers []matcher
}
func (r *SubRoute) Match(feed *block.FeedVO) *SubRoute {
for _, subRoute := range r.SubRoutes {
if matched := subRoute.Match(feed); matched != nil {
return matched
}
}
for _, m := range r.matchers {
fv := feed.Labels.Get(m.key)
switch m.equal {
case true:
if fv != m.value {
return nil
}
default:
if fv == m.value {
return nil
}
}
}
return r
}
type matcher struct {
key string
value string
equal bool
}
var (
matcherEqual = "="
matcherNotEqual = "!="
parseMatcher = func(filter string) (matcher, error) {
eq := false
parts := strings.Split(filter, matcherNotEqual)
if len(parts) != 2 {
parts = strings.Split(filter, matcherEqual)
eq = true
}
if len(parts) != 2 {
return matcher{}, errors.New("invalid matcher")
}
return matcher{key: parts[0], value: parts[1], equal: eq}, nil
}
)
func (r *SubRoute) Validate() error {
if len(r.GroupBy) == 0 {
r.GroupBy = []string{model.LabelSource}
}
if r.CompressByRelatedThreshold == nil {
r.CompressByRelatedThreshold = ptr.To(float32(0.85))
}
if len(r.Matchers) == 0 {
return errors.New("matchers is required")
}
r.matchers = make([]matcher, len(r.Matchers))
for i, matcher := range r.Matchers {
m, err := parseMatcher(matcher)
if err != nil {
return errors.Wrap(err, "invalid matcher")
}
r.matchers[i] = m
}
for _, subRoute := range r.SubRoutes {
if err := subRoute.Validate(); err != nil {
return errors.Wrap(err, "invalid sub_route")
}
}
return nil
}
func (c *Config) Validate() error {
if len(c.GroupBy) == 0 {
c.GroupBy = []string{model.LabelSource}
}
if c.CompressByRelatedThreshold == nil {
c.CompressByRelatedThreshold = ptr.To(float32(0.85))
}
for _, subRoute := range c.SubRoutes {
if err := subRoute.Validate(); err != nil {
return errors.Wrap(err, "invalid sub_route")
}
}
return nil
}
type Dependencies struct {
RelatedScore func(a, b [][]float32) (float32, error) // MUST same with vector index.
}
type Group struct {
FeedGroup
Receivers []string
}
type FeedGroup struct {
Name string
Time time.Time
Labels model.Labels
Feeds []*Feed
}
func (g *FeedGroup) ID() string {
return fmt.Sprintf("%s-%s", g.Name, timeutil.Format(g.Time))
}
type Feed struct {
*model.Feed
Related []*Feed `json:"related"`
Vectors [][]float32 `json:"-"`
}
// --- Factory code block ---
type Factory component.Factory[Router, Config, Dependencies]
func NewFactory(mockOn ...component.MockOption) Factory {
if len(mockOn) > 0 {
return component.FactoryFunc[Router, Config, Dependencies](
func(instance string, config *Config, dependencies Dependencies) (Router, error) {
m := &mockRouter{}
component.MockOptions(mockOn).Apply(&m.Mock)
return m, nil
},
)
}
return component.FactoryFunc[Router, Config, Dependencies](new)
}
func new(instance string, config *Config, dependencies Dependencies) (Router, error) {
return &router{
Base: component.New(&component.BaseConfig[Config, Dependencies]{
Name: "NotifyRouter",
Instance: instance,
Config: config,
Dependencies: dependencies,
}),
}, nil
}
// --- Implementation code block ---
type router struct {
*component.Base[Config, Dependencies]
}
func (r *router) Route(result *rule.Result) (groups []*Group, err error) {
// Find route for each feed.
feedsByRoute := r.routeFeeds(result.Feeds)
// Process each route and its feeds.
for route, feeds := range feedsByRoute {
// Group feeds by labels.
groupedFeeds := r.groupFeedsByLabels(route, feeds)
// Compress related feeds.
relatedGroups, err := r.compressRelatedFeeds(route, groupedFeeds)
if err != nil {
return nil, errors.Wrap(err, "compress related feeds")
}
// Build final groups.
for ls, feeds := range relatedGroups {
groups = append(groups, &Group{
FeedGroup: FeedGroup{
Name: fmt.Sprintf("%s %s", result.Rule, ls.String()),
Time: result.Time,
Labels: *ls,
Feeds: feeds,
},
Receivers: route.Receivers,
})
}
}
sort.Slice(groups, func(i, j int) bool {
return groups[i].Name < groups[j].Name
})
return groups, nil
}
func (r *router) routeFeeds(feeds []*block.FeedVO) map[*Route][]*block.FeedVO {
config := r.Config()
feedsByRoute := make(map[*Route][]*block.FeedVO)
for _, feed := range feeds {
var targetRoute *Route
if matched := config.SubRoutes.Match(feed); matched != nil {
targetRoute = &matched.Route
} else {
// Fallback to default route.
targetRoute = &config.Route
}
feedsByRoute[targetRoute] = append(feedsByRoute[targetRoute], feed)
}
return feedsByRoute
}
func (r *router) groupFeedsByLabels(route *Route, feeds []*block.FeedVO) map[*model.Labels][]*block.FeedVO {
groupedFeeds := make(map[*model.Labels][]*block.FeedVO)
labelGroups := make(map[string]*model.Labels)
for _, feed := range feeds {
var group model.Labels
for _, key := range route.GroupBy {
value := feed.Labels.Get(key)
group.Put(key, value, true)
}
groupKey := group.String()
labelGroup, exists := labelGroups[groupKey]
if !exists {
labelGroups[groupKey] = &group
labelGroup = &group
}
groupedFeeds[labelGroup] = append(groupedFeeds[labelGroup], feed)
}
return groupedFeeds
}
func (r *router) compressRelatedFeeds(
route *Route, // config
groupedFeeds map[*model.Labels][]*block.FeedVO, // group id -> feeds
) (map[*model.Labels][]*Feed, error) { // group id -> feeds with related feeds
result := make(map[*model.Labels][]*Feed)
for ls, feeds := range groupedFeeds { // per group
fs, err := r.compressRelatedFeedsForGroup(route, feeds)
if err != nil {
return nil, errors.Wrap(err, "compress related feeds")
}
result[ls] = fs
}
return result, nil
}
func (r *router) compressRelatedFeedsForGroup(
route *Route, // config
feeds []*block.FeedVO, // feeds
) ([]*Feed, error) {
feedsWithRelated := make([]*Feed, 0, len(feeds)/2)
for _, feed := range feeds {
foundRelated := false
for i := range feedsWithRelated {
// Try join with previous feeds.
score, err := r.Dependencies().RelatedScore(feedsWithRelated[i].Vectors, feed.Vectors)
if err != nil {
return nil, errors.Wrap(err, "related score")
}
if score >= *route.CompressByRelatedThreshold {
foundRelated = true
feedsWithRelated[i].Related = append(feedsWithRelated[i].Related, &Feed{
Feed: feed.Feed,
})
break
}
}
// If not found related, create a group by itself.
if !foundRelated {
feedsWithRelated = append(feedsWithRelated, &Feed{
Feed: feed.Feed,
Vectors: feed.Vectors,
})
}
}
return feedsWithRelated, nil
}
type mockRouter struct {
component.Mock
}
func (m *mockRouter) Route(result *rule.Result) (groups []*Group, err error) {
m.Called(result)
return groups, err
}

View File

@@ -0,0 +1,770 @@
package route
import (
"fmt"
"testing"
"time"
. "github.com/onsi/gomega"
"github.com/pkg/errors"
"github.com/stretchr/testify/mock"
"k8s.io/utils/ptr"
"github.com/glidea/zenfeed/pkg/component"
"github.com/glidea/zenfeed/pkg/model"
"github.com/glidea/zenfeed/pkg/schedule/rule"
"github.com/glidea/zenfeed/pkg/storage/feed/block"
"github.com/glidea/zenfeed/pkg/test"
timeutil "github.com/glidea/zenfeed/pkg/util/time"
)
func TestRoute(t *testing.T) {
RegisterTestingT(t)
type givenDetail struct {
config *Config
relatedScore func(m *mock.Mock) // Mock setup for RelatedScore.
}
type whenDetail struct {
ruleResult *rule.Result
}
type thenExpected struct {
groups []*Group
isErr bool
errMsg string
}
now := time.Now()
testFeeds := []*block.FeedVO{
{
Feed: &model.Feed{
ID: 1,
Labels: model.Labels{
{Key: model.LabelSource, Value: "TechCrunch"},
{Key: "category", Value: "AI"},
{Key: model.LabelTitle, Value: "Tech News 1"},
{Key: model.LabelLink, Value: "http://example.com/tech1"},
},
Time: now,
},
Vectors: [][]float32{{0.1, 0.2}},
},
{
Feed: &model.Feed{
ID: 2,
Labels: model.Labels{
{Key: model.LabelSource, Value: "TechCrunch"},
{Key: "category", Value: "AI"},
{Key: model.LabelTitle, Value: "Tech News 2"},
{Key: model.LabelLink, Value: "http://example.com/tech2"},
},
Time: now,
},
Vectors: [][]float32{{0.11, 0.21}},
},
{
Feed: &model.Feed{
ID: 3,
Labels: model.Labels{
{Key: model.LabelSource, Value: "Bloomberg"},
{Key: "category", Value: "Markets"},
{Key: model.LabelTitle, Value: "Finance News 1"},
{Key: model.LabelLink, Value: "http://example.com/finance1"},
},
Time: now,
},
Vectors: [][]float32{{0.8, 0.9}},
},
{
Feed: &model.Feed{
ID: 4,
Labels: model.Labels{
{Key: model.LabelSource, Value: "TechCrunch"},
{Key: "category", Value: "Hardware"},
{Key: model.LabelTitle, Value: "Specific Tech News"},
{Key: model.LabelLink, Value: "http://example.com/tech_specific"},
},
Time: now,
},
Vectors: [][]float32{{0.5, 0.5}},
},
{
Feed: &model.Feed{
ID: 5,
Labels: model.Labels{
{Key: model.LabelSource, Value: "OtherSource"},
{Key: "category", Value: "Sports"},
{Key: model.LabelTitle, Value: "Non-Matching Category News"},
{Key: model.LabelLink, Value: "http://example.com/other"},
},
Time: now,
},
Vectors: [][]float32{{0.9, 0.1}},
},
}
for _, tf := range testFeeds {
tf.Labels.EnsureSorted()
}
tests := []test.Case[givenDetail, whenDetail, thenExpected]{
{
Scenario: "Basic routing and grouping by source",
Given: "a default router config grouping by source and high related threshold",
When: "routing feeds from different sources",
Then: "should group feeds by source into separate groups without compression",
GivenDetail: givenDetail{
config: &Config{
Route: Route{
GroupBy: []string{model.LabelSource},
CompressByRelatedThreshold: ptr.To(float32(0.99)),
Receivers: []string{"default-receiver"},
},
},
},
WhenDetail: whenDetail{
ruleResult: &rule.Result{
Rule: "TestRule",
Time: now,
Feeds: []*block.FeedVO{testFeeds[0], testFeeds[2], testFeeds[4]},
},
},
ThenExpected: thenExpected{
groups: []*Group{
{
FeedGroup: FeedGroup{
Name: fmt.Sprintf("TestRule %s", model.Labels{{Key: model.LabelSource, Value: "Bloomberg"}}.String()),
Time: now,
Labels: model.Labels{{Key: model.LabelSource, Value: "Bloomberg"}},
Feeds: []*Feed{
{Feed: testFeeds[2].Feed, Vectors: testFeeds[2].Vectors},
},
},
Receivers: []string{"default-receiver"},
},
{
FeedGroup: FeedGroup{
Name: fmt.Sprintf("TestRule %s", model.Labels{{Key: model.LabelSource, Value: "OtherSource"}}.String()),
Time: now,
Labels: model.Labels{{Key: model.LabelSource, Value: "OtherSource"}},
Feeds: []*Feed{
{Feed: testFeeds[4].Feed, Vectors: testFeeds[4].Vectors},
},
},
Receivers: []string{"default-receiver"},
},
{
FeedGroup: FeedGroup{
Name: fmt.Sprintf("TestRule %s", model.Labels{{Key: model.LabelSource, Value: "TechCrunch"}}.String()),
Time: now,
Labels: model.Labels{{Key: model.LabelSource, Value: "TechCrunch"}},
Feeds: []*Feed{
{Feed: testFeeds[0].Feed, Vectors: testFeeds[0].Vectors},
},
},
Receivers: []string{"default-receiver"},
},
},
isErr: false,
},
},
{
Scenario: "Routing with sub-route matching",
Given: "a router config with a sub-route for AI category",
When: "routing feeds including AI category",
Then: "should apply the sub-route's receivers and settings to matching feeds",
GivenDetail: givenDetail{
config: &Config{
Route: Route{
GroupBy: []string{model.LabelSource},
CompressByRelatedThreshold: ptr.To(float32(0.99)),
Receivers: []string{"default-receiver"},
SubRoutes: SubRoutes{
{
Route: Route{
GroupBy: []string{model.LabelSource, "category"},
CompressByRelatedThreshold: ptr.To(float32(0.99)),
Receivers: []string{"ai-receiver"},
},
Matchers: []string{"category=AI"},
},
},
},
},
relatedScore: func(m *mock.Mock) {
m.On("RelatedScore", mock.Anything, mock.Anything).Return(float32(0.1), nil)
},
},
WhenDetail: whenDetail{
ruleResult: &rule.Result{
Rule: "SubRouteRule",
Time: now,
Feeds: []*block.FeedVO{testFeeds[0], testFeeds[1], testFeeds[4]},
},
},
ThenExpected: thenExpected{
groups: []*Group{
{
FeedGroup: FeedGroup{
Name: fmt.Sprintf("SubRouteRule %s", model.Labels{
{Key: model.LabelSource, Value: "TechCrunch"},
{Key: "category", Value: "AI"},
}.String()),
Time: now,
Labels: model.Labels{
{Key: "category", Value: "AI"},
{Key: model.LabelSource, Value: "TechCrunch"},
},
Feeds: []*Feed{
{Feed: testFeeds[0].Feed, Vectors: testFeeds[0].Vectors},
{Feed: testFeeds[1].Feed, Vectors: testFeeds[1].Vectors},
},
},
Receivers: []string{"ai-receiver"},
},
{
FeedGroup: FeedGroup{
Name: fmt.Sprintf("SubRouteRule %s", model.Labels{{Key: model.LabelSource, Value: "OtherSource"}}.String()),
Time: now,
Labels: model.Labels{{Key: model.LabelSource, Value: "OtherSource"}},
Feeds: []*Feed{
{Feed: testFeeds[4].Feed, Vectors: testFeeds[4].Vectors},
},
},
Receivers: []string{"default-receiver"},
},
},
isErr: false,
},
},
{
Scenario: "Compressing related feeds",
Given: "a router config with a low related threshold",
When: "routing feeds with similar vectors",
Then: "should compress related feeds into a single group entry",
GivenDetail: givenDetail{
config: &Config{
Route: Route{
GroupBy: []string{model.LabelSource, "category"},
CompressByRelatedThreshold: ptr.To(float32(0.8)),
Receivers: []string{"compress-receiver"},
},
},
relatedScore: func(m *mock.Mock) {
m.On("RelatedScore", testFeeds[0].Vectors, testFeeds[1].Vectors).Return(float32(0.9), nil)
m.On("RelatedScore", mock.Anything, mock.Anything).Maybe().Return(float32(0.1), nil)
},
},
WhenDetail: whenDetail{
ruleResult: &rule.Result{
Rule: "CompressRule",
Time: now,
Feeds: []*block.FeedVO{testFeeds[0], testFeeds[1], testFeeds[3]},
},
},
ThenExpected: thenExpected{
groups: []*Group{
{
FeedGroup: FeedGroup{
Name: fmt.Sprintf("CompressRule %s", model.Labels{
{Key: model.LabelSource, Value: "TechCrunch"},
{Key: "category", Value: "AI"},
}.String()),
Time: now,
Labels: model.Labels{
{Key: "category", Value: "AI"},
{Key: model.LabelSource, Value: "TechCrunch"},
},
Feeds: []*Feed{
{
Feed: testFeeds[0].Feed,
Vectors: testFeeds[0].Vectors,
Related: []*Feed{
{Feed: testFeeds[1].Feed},
},
},
},
},
Receivers: []string{"compress-receiver"},
},
{
FeedGroup: FeedGroup{
Name: fmt.Sprintf("CompressRule %s", model.Labels{
{Key: model.LabelSource, Value: "TechCrunch"},
{Key: "category", Value: "Hardware"},
}.String()),
Time: now,
Labels: model.Labels{
{Key: "category", Value: "Hardware"},
{Key: model.LabelSource, Value: "TechCrunch"},
},
Feeds: []*Feed{
{Feed: testFeeds[3].Feed, Vectors: testFeeds[3].Vectors},
},
},
Receivers: []string{"compress-receiver"},
},
},
isErr: false,
},
},
{
Scenario: "Error during related score calculation",
Given: "a router config and RelatedScore dependency returns an error",
When: "routing feeds requiring related score check",
Then: "should return an error originating from RelatedScore",
GivenDetail: givenDetail{
config: &Config{
Route: Route{
GroupBy: []string{model.LabelSource},
CompressByRelatedThreshold: ptr.To(float32(0.8)),
Receivers: []string{"error-receiver"},
},
},
relatedScore: func(m *mock.Mock) {
m.On("RelatedScore", testFeeds[0].Vectors, testFeeds[1].Vectors).Return(float32(0), errors.New("related score calculation failed"))
m.On("RelatedScore", mock.Anything, mock.Anything).Maybe().Return(float32(0.1), nil)
},
},
WhenDetail: whenDetail{
ruleResult: &rule.Result{
Rule: "ErrorRule",
Time: now,
Feeds: []*block.FeedVO{testFeeds[0], testFeeds[1]},
},
},
ThenExpected: thenExpected{
groups: nil,
isErr: true,
errMsg: "compress related feeds: compress related feeds: related score: related score calculation failed",
},
},
{
Scenario: "No feeds to route",
Given: "a standard router config",
When: "routing an empty list of feeds",
Then: "should return an empty list of groups without error",
GivenDetail: givenDetail{
config: &Config{
Route: Route{
GroupBy: []string{model.LabelSource},
CompressByRelatedThreshold: ptr.To(float32(0.85)),
Receivers: []string{"default-receiver"},
},
},
relatedScore: func(m *mock.Mock) {
},
},
WhenDetail: whenDetail{
ruleResult: &rule.Result{
Rule: "EmptyRule",
Time: now,
Feeds: []*block.FeedVO{},
},
},
ThenExpected: thenExpected{
groups: []*Group{},
isErr: false,
},
},
}
for _, tt := range tests {
t.Run(tt.Scenario, func(t *testing.T) {
for _, group := range tt.ThenExpected.groups {
group.Labels.EnsureSorted()
}
err := tt.GivenDetail.config.Validate()
Expect(err).NotTo(HaveOccurred(), "Config validation failed during test setup")
mockDep := mockDependencies{}
if tt.GivenDetail.relatedScore != nil {
tt.GivenDetail.relatedScore(&mockDep.Mock)
}
routerInstance := &router{
Base: component.New(&component.BaseConfig[Config, Dependencies]{
Name: "TestRouter",
Instance: "test",
Config: tt.GivenDetail.config,
Dependencies: Dependencies{
RelatedScore: mockDep.RelatedScore,
},
}),
}
groups, err := routerInstance.Route(tt.WhenDetail.ruleResult)
if tt.ThenExpected.isErr {
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring(tt.ThenExpected.errMsg))
Expect(groups).To(BeNil())
} else {
Expect(err).NotTo(HaveOccurred())
compareGroups(groups, tt.ThenExpected.groups)
}
mockDep.AssertExpectations(t)
})
}
}
type mockDependencies struct {
mock.Mock
}
func (m *mockDependencies) RelatedScore(a, b [][]float32) (float32, error) {
args := m.Called(a, b)
return args.Get(0).(float32), args.Error(1)
}
func compareGroups(actual, expected []*Group) {
Expect(actual).To(HaveLen(len(expected)), "Number of groups mismatch")
for i := range expected {
actualGroup := actual[i]
expectedGroup := expected[i]
Expect(actualGroup.Name).To(Equal(expectedGroup.Name), fmt.Sprintf("Group %d Name mismatch", i))
Expect(timeutil.Format(actualGroup.Time)).To(Equal(timeutil.Format(expectedGroup.Time)), fmt.Sprintf("Group %d Time mismatch", i))
Expect(actualGroup.Labels).To(Equal(expectedGroup.Labels), fmt.Sprintf("Group %d Labels mismatch", i))
Expect(actualGroup.Receivers).To(Equal(expectedGroup.Receivers), fmt.Sprintf("Group %d Receivers mismatch", i))
compareFeedsWithRelated(actualGroup.Feeds, expectedGroup.Feeds, i)
}
}
func compareFeedsWithRelated(actual, expected []*Feed, groupIndex int) {
Expect(actual).To(HaveLen(len(expected)), fmt.Sprintf("Group %d: Number of primary feeds mismatch", groupIndex))
for i := range expected {
actualFeed := actual[i]
expectedFeed := expected[i]
Expect(actualFeed.Feed).To(Equal(expectedFeed.Feed), fmt.Sprintf("Group %d, Feed %d: Primary feed mismatch", groupIndex, i))
Expect(actualFeed.Related).To(HaveLen(len(expectedFeed.Related)), fmt.Sprintf("Group %d, Feed %d: Number of related feeds mismatch", groupIndex, i))
for j := range expectedFeed.Related {
Expect(actualFeed.Related[j].Feed).To(Equal(expectedFeed.Related[j].Feed), fmt.Sprintf("Group %d, Feed %d, Related %d: Related feed mismatch", groupIndex, i, j))
}
}
}
func TestConfig_Validate(t *testing.T) {
RegisterTestingT(t)
tests := []struct {
name string
config *Config
wantErr bool
errMsg string
}{
{
name: "Valid default config",
config: &Config{
Route: Route{
Receivers: []string{"rec1"},
},
},
wantErr: false,
},
{
name: "Valid config with explicit defaults",
config: &Config{
Route: Route{
GroupBy: []string{model.LabelSource},
CompressByRelatedThreshold: ptr.To(float32(0.85)),
Receivers: []string{"rec1"},
},
},
wantErr: false,
},
{
name: "Valid config with sub-route",
config: &Config{
Route: Route{
Receivers: []string{"rec1"},
SubRoutes: SubRoutes{
{
Route: Route{
Receivers: []string{"rec2"},
},
Matchers: []string{"label=value"},
},
},
},
},
wantErr: false,
},
{
name: "Invalid sub-route missing matchers",
config: &Config{
Route: Route{
Receivers: []string{"rec1"},
SubRoutes: SubRoutes{
{
Route: Route{
Receivers: []string{"rec2"},
},
},
},
},
},
wantErr: true,
errMsg: "invalid sub_route: matchers is required",
},
{
name: "Invalid sub-route matcher format",
config: &Config{
Route: Route{
Receivers: []string{"rec1"},
SubRoutes: SubRoutes{
{
Route: Route{
Receivers: []string{"rec2"},
},
Matchers: []string{"invalid-matcher"},
},
},
},
},
wantErr: true,
errMsg: "invalid sub_route: invalid matcher: invalid matcher",
},
{
name: "Valid nested sub-route",
config: &Config{
Route: Route{
Receivers: []string{"rec1"},
SubRoutes: SubRoutes{
{
Route: Route{
Receivers: []string{"rec2"},
SubRoutes: SubRoutes{
{
Route: Route{
Receivers: []string{"rec3"},
},
Matchers: []string{"nested=true"},
},
},
},
Matchers: []string{"label=value"},
},
},
},
},
wantErr: false,
},
{
name: "Invalid nested sub-route",
config: &Config{
Route: Route{
Receivers: []string{"rec1"},
SubRoutes: SubRoutes{
{
Route: Route{
Receivers: []string{"rec2"},
SubRoutes: SubRoutes{
{
Route: Route{
Receivers: []string{"rec3"},
},
},
},
},
Matchers: []string{"label=value"},
},
},
},
},
wantErr: true,
errMsg: "invalid sub_route: invalid sub_route: matchers is required",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := tt.config.Validate()
if tt.wantErr {
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring(tt.errMsg))
} else {
Expect(err).NotTo(HaveOccurred())
Expect(tt.config.GroupBy).NotTo(BeEmpty())
Expect(tt.config.CompressByRelatedThreshold).NotTo(BeNil())
for _, sr := range tt.config.SubRoutes {
Expect(sr.GroupBy).NotTo(BeEmpty())
Expect(sr.CompressByRelatedThreshold).NotTo(BeNil())
for _, nestedSr := range sr.SubRoutes {
Expect(nestedSr.GroupBy).NotTo(BeEmpty())
Expect(nestedSr.CompressByRelatedThreshold).NotTo(BeNil())
}
}
}
})
}
}
func TestSubRoutes_Match(t *testing.T) {
RegisterTestingT(t)
now := time.Now()
feedAI := &block.FeedVO{
Feed: &model.Feed{
ID: 10,
Labels: model.Labels{
{Key: "category", Value: "AI"},
{Key: model.LabelSource, Value: "TechCrunch"},
{Key: model.LabelTitle, Value: "AI Feed"},
{Key: model.LabelLink, Value: "http://example.com/ai"},
},
Time: now,
},
}
feedHardware := &block.FeedVO{
Feed: &model.Feed{
ID: 11,
Labels: model.Labels{
{Key: "category", Value: "Hardware"},
{Key: model.LabelSource, Value: "TechCrunch"},
{Key: model.LabelTitle, Value: "Hardware Feed"},
{Key: model.LabelLink, Value: "http://example.com/hw"},
},
Time: now,
},
}
feedSports := &block.FeedVO{
Feed: &model.Feed{
ID: 12,
Labels: model.Labels{
{Key: "category", Value: "Sports"},
{Key: model.LabelSource, Value: "OtherSource"},
{Key: model.LabelTitle, Value: "Sports Feed"},
{Key: model.LabelLink, Value: "http://example.com/sports"},
},
Time: now,
},
}
feedNestedLow := &block.FeedVO{
Feed: &model.Feed{
ID: 13,
Labels: model.Labels{
{Key: "category", Value: "Nested"},
{Key: "priority", Value: "low"},
{Key: model.LabelTitle, Value: "Nested Low Prio"},
{Key: model.LabelLink, Value: "http://example.com/nested_low"},
},
Time: now,
},
}
feedNestedHigh := &block.FeedVO{
Feed: &model.Feed{
ID: 14,
Labels: model.Labels{
{Key: "category", Value: "Nested"},
{Key: "priority", Value: "high"},
{Key: model.LabelTitle, Value: "Nested High Prio"},
{Key: model.LabelLink, Value: "http://example.com/nested_high"},
},
Time: now,
},
}
feedAI.Labels.EnsureSorted()
feedHardware.Labels.EnsureSorted()
feedSports.Labels.EnsureSorted()
feedNestedLow.Labels.EnsureSorted()
feedNestedHigh.Labels.EnsureSorted()
subRouteAI := &SubRoute{
Route: Route{Receivers: []string{"ai"}},
Matchers: []string{"category=AI"},
}
err := subRouteAI.Validate()
Expect(err).NotTo(HaveOccurred())
subRouteHardware := &SubRoute{
Route: Route{Receivers: []string{"hardware"}},
Matchers: []string{"category=Hardware"},
}
err = subRouteHardware.Validate()
Expect(err).NotTo(HaveOccurred())
subRouteTechCrunchNotAI := &SubRoute{
Route: Route{Receivers: []string{"tc-not-ai"}},
Matchers: []string{model.LabelSource + "=TechCrunch", "category!=AI"},
}
err = subRouteTechCrunchNotAI.Validate()
Expect(err).NotTo(HaveOccurred())
subRouteNested := &SubRoute{
Route: Route{
Receivers: []string{"nested-route"},
SubRoutes: SubRoutes{
{
Route: Route{Receivers: []string{"deep-nested"}},
Matchers: []string{"priority=high"},
},
},
},
Matchers: []string{"category=Nested"},
}
err = subRouteNested.Validate()
Expect(err).NotTo(HaveOccurred())
nestedDeepSubRoute := subRouteNested.SubRoutes[0]
err = nestedDeepSubRoute.Validate()
Expect(err).NotTo(HaveOccurred())
routes := SubRoutes{subRouteAI, subRouteHardware, subRouteTechCrunchNotAI, subRouteNested}
tests := []struct {
name string
feed *block.FeedVO
expectedRoute *SubRoute
}{
{
name: "Match AI category",
feed: feedAI,
expectedRoute: subRouteAI,
},
{
name: "Match Hardware category",
feed: feedHardware,
expectedRoute: subRouteHardware,
},
{
name: "Match TechCrunch but not AI",
feed: feedHardware,
expectedRoute: subRouteHardware,
},
{
name: "No matching category",
feed: feedSports,
expectedRoute: nil,
},
{
name: "Match nested route (top level)",
feed: feedNestedLow,
expectedRoute: subRouteNested,
},
{
name: "Match nested route (deep level)",
feed: feedNestedHigh,
expectedRoute: nestedDeepSubRoute,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
matchedRoute := routes.Match(tt.feed)
if tt.expectedRoute == nil {
Expect(matchedRoute).To(BeNil())
} else {
Expect(matchedRoute).NotTo(BeNil())
Expect(matchedRoute.Receivers).To(Equal(tt.expectedRoute.Receivers))
Expect(matchedRoute.Matchers).To(Equal(tt.expectedRoute.Matchers))
}
})
}
}