Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2de0cf77fc |
6
.github/workflows/ci.yml
vendored
6
.github/workflows/ci.yml
vendored
@@ -5,8 +5,6 @@ on:
|
|||||||
branches: [ main, dev ]
|
branches: [ main, dev ]
|
||||||
pull_request:
|
pull_request:
|
||||||
branches: [ main ]
|
branches: [ main ]
|
||||||
release:
|
|
||||||
types: [ published ]
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
test:
|
test:
|
||||||
@@ -27,7 +25,7 @@ jobs:
|
|||||||
build-and-push:
|
build-and-push:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
needs: test
|
needs: test
|
||||||
if: github.event_name == 'release' || (github.event_name == 'push' && github.ref_name == 'dev')
|
if: github.event_name == 'push'
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v4
|
||||||
- name: Set up Docker Buildx
|
- name: Set up Docker Buildx
|
||||||
@@ -38,7 +36,7 @@ jobs:
|
|||||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||||
- name: Build and push Docker image (main)
|
- name: Build and push Docker image (main)
|
||||||
if: github.event_name == 'release'
|
if: github.ref_name == 'main'
|
||||||
run: make push
|
run: make push
|
||||||
- name: Build and push Docker image (dev)
|
- name: Build and push Docker image (dev)
|
||||||
if: github.ref_name == 'dev'
|
if: github.ref_name == 'dev'
|
||||||
|
|||||||
@@ -46,7 +46,7 @@ llms:
|
|||||||
|
|
||||||
首先,您需要在 Cloudflare R2 中创建一个存储桶(Bucket)。然后获取以下信息:
|
首先,您需要在 Cloudflare R2 中创建一个存储桶(Bucket)。然后获取以下信息:
|
||||||
|
|
||||||
- `endpoint`: 您的 R2 API 端点。通常格式为 `<account_id>.r2.cloudflarestorage.com`。您可以在 R2 存储桶的主页找到它。
|
- `endpoint`: 您的 R2 API 端点。通常格式为 `https://<account_id>.r2.cloudflarestorage.com`。您可以在 R2 存储桶的主页找到它。
|
||||||
- `access_key_id` 和 `secret_access_key`: R2 API 令牌。您可以在 "R2" -> "管理 R2 API 令牌" 页面创建。
|
- `access_key_id` 和 `secret_access_key`: R2 API 令牌。您可以在 "R2" -> "管理 R2 API 令牌" 页面创建。
|
||||||
- `bucket`: 您创建的存储桶的名称。
|
- `bucket`: 您创建的存储桶的名称。
|
||||||
- `bucket_url`: 存储桶的公开访问 URL。要获取此 URL,您需要将存储桶连接到一个自定义域,或者使用 R2 提供的 `r2.dev` 公开访问地址。
|
- `bucket_url`: 存储桶的公开访问 URL。要获取此 URL,您需要将存储桶连接到一个自定义域,或者使用 R2 提供的 `r2.dev` 公开访问地址。
|
||||||
@@ -56,7 +56,7 @@ llms:
|
|||||||
```yaml
|
```yaml
|
||||||
storage:
|
storage:
|
||||||
object:
|
object:
|
||||||
endpoint: "<your_account_id>.r2.cloudflarestorage.com"
|
endpoint: "https://<your_account_id>.r2.cloudflarestorage.com"
|
||||||
access_key_id: "..."
|
access_key_id: "..."
|
||||||
secret_access_key: "..."
|
secret_access_key: "..."
|
||||||
bucket: "zenfeed-podcasts"
|
bucket: "zenfeed-podcasts"
|
||||||
@@ -91,10 +91,10 @@ storage:
|
|||||||
to_podcast:
|
to_podcast:
|
||||||
llm: "openai-chat"
|
llm: "openai-chat"
|
||||||
tts_llm: "gemini-tts"
|
tts_llm: "gemini-tts"
|
||||||
transcript_additional_prompt: "使用中文回复"
|
transcript_additional_prompt: "请让对话更生动有趣一些。使用中文回复"
|
||||||
speakers:
|
speakers:
|
||||||
- name: "主持人小雅"
|
- name: "主持人小雅"
|
||||||
role: "一位经验丰富、声音甜美、风格活泼的科技播客主持人。擅长联系实际生活场景。"
|
role: "一位经验丰富、声音甜美、风格活泼的科技播客主持人。"
|
||||||
voice: "zh-CN-Standard-A" # 女声
|
voice: "zh-CN-Standard-A" # 女声
|
||||||
- name: "技术评论员老王"
|
- name: "技术评论员老王"
|
||||||
role: "一位对技术有深入见解、观点犀利的评论员,说话直接,偶尔有些愤世嫉俗。"
|
role: "一位对技术有深入见解、观点犀利的评论员,说话直接,偶尔有些愤世嫉俗。"
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"io"
|
"io"
|
||||||
"net/url"
|
"net/url"
|
||||||
"strings"
|
"reflect"
|
||||||
|
|
||||||
"github.com/minio/minio-go/v7"
|
"github.com/minio/minio-go/v7"
|
||||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||||
@@ -46,50 +46,26 @@ type Config struct {
|
|||||||
Endpoint string
|
Endpoint string
|
||||||
AccessKeyID string
|
AccessKeyID string
|
||||||
SecretAccessKey string
|
SecretAccessKey string
|
||||||
client *minio.Client
|
Bucket string
|
||||||
|
BucketURL string
|
||||||
Bucket string
|
|
||||||
BucketURL string
|
|
||||||
bucketURL *url.URL
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Config) Validate() error {
|
func (c *Config) Validate() error {
|
||||||
if c.Empty() {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if c.Endpoint == "" {
|
if c.Endpoint == "" {
|
||||||
return errors.New("endpoint is required")
|
return errors.New("endpoint is required")
|
||||||
}
|
}
|
||||||
c.Endpoint = strings.TrimPrefix(c.Endpoint, "https://") // S3 endpoint should not have https:// prefix.
|
|
||||||
c.Endpoint = strings.TrimPrefix(c.Endpoint, "http://")
|
|
||||||
|
|
||||||
if c.AccessKeyID == "" {
|
if c.AccessKeyID == "" {
|
||||||
return errors.New("access key id is required")
|
return errors.New("access key id is required")
|
||||||
}
|
}
|
||||||
if c.SecretAccessKey == "" {
|
if c.SecretAccessKey == "" {
|
||||||
return errors.New("secret access key is required")
|
return errors.New("secret access key is required")
|
||||||
}
|
}
|
||||||
client, err := minio.New(c.Endpoint, &minio.Options{
|
|
||||||
Creds: credentials.NewStaticV4(c.AccessKeyID, c.SecretAccessKey, ""),
|
|
||||||
Secure: true,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return errors.Wrap(err, "new minio client")
|
|
||||||
}
|
|
||||||
c.client = client
|
|
||||||
|
|
||||||
if c.Bucket == "" {
|
if c.Bucket == "" {
|
||||||
return errors.New("bucket is required")
|
return errors.New("bucket is required")
|
||||||
}
|
}
|
||||||
if c.BucketURL == "" {
|
if c.BucketURL == "" {
|
||||||
return errors.New("bucket url is required")
|
return errors.New("bucket url is required")
|
||||||
}
|
}
|
||||||
u, err := url.Parse(c.BucketURL)
|
|
||||||
if err != nil {
|
|
||||||
return errors.Wrap(err, "parse public url")
|
|
||||||
}
|
|
||||||
c.bucketURL = u
|
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -106,10 +82,6 @@ func (c *Config) From(app *config.App) *Config {
|
|||||||
return c
|
return c
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Config) Empty() bool {
|
|
||||||
return c.Endpoint == "" && c.AccessKeyID == "" && c.SecretAccessKey == "" && c.Bucket == "" && c.BucketURL == ""
|
|
||||||
}
|
|
||||||
|
|
||||||
type Dependencies struct{}
|
type Dependencies struct{}
|
||||||
|
|
||||||
// --- Factory code block ---
|
// --- Factory code block ---
|
||||||
@@ -137,6 +109,19 @@ func new(instance string, app *config.App, dependencies Dependencies) (Storage,
|
|||||||
return nil, errors.Wrap(err, "validate config")
|
return nil, errors.Wrap(err, "validate config")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
client, err := minio.New(config.Endpoint, &minio.Options{
|
||||||
|
Creds: credentials.NewStaticV4(config.AccessKeyID, config.SecretAccessKey, ""),
|
||||||
|
Secure: true,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.Wrap(err, "new minio client")
|
||||||
|
}
|
||||||
|
|
||||||
|
u, err := url.Parse(config.BucketURL)
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.Wrap(err, "parse public url")
|
||||||
|
}
|
||||||
|
|
||||||
return &s3{
|
return &s3{
|
||||||
Base: component.New(&component.BaseConfig[Config, Dependencies]{
|
Base: component.New(&component.BaseConfig[Config, Dependencies]{
|
||||||
Name: "ObjectStorage",
|
Name: "ObjectStorage",
|
||||||
@@ -144,40 +129,39 @@ func new(instance string, app *config.App, dependencies Dependencies) (Storage,
|
|||||||
Config: config,
|
Config: config,
|
||||||
Dependencies: dependencies,
|
Dependencies: dependencies,
|
||||||
}),
|
}),
|
||||||
|
client: client,
|
||||||
|
bucketURL: u,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// --- Implementation code block ---
|
// --- Implementation code block ---
|
||||||
type s3 struct {
|
type s3 struct {
|
||||||
*component.Base[Config, Dependencies]
|
*component.Base[Config, Dependencies]
|
||||||
|
|
||||||
|
client *minio.Client
|
||||||
|
bucketURL *url.URL
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *s3) Put(ctx context.Context, key string, body io.Reader, contentType string) (publicURL string, err error) {
|
func (s *s3) Put(ctx context.Context, key string, body io.Reader, contentType string) (publicURL string, err error) {
|
||||||
ctx = telemetry.StartWith(ctx, append(s.TelemetryLabels(), telemetrymodel.KeyOperation, "Put")...)
|
ctx = telemetry.StartWith(ctx, append(s.TelemetryLabels(), telemetrymodel.KeyOperation, "Put")...)
|
||||||
defer func() { telemetry.End(ctx, err) }()
|
defer func() { telemetry.End(ctx, err) }()
|
||||||
config := s.Config()
|
bucket := s.Config().Bucket
|
||||||
if config.Empty() {
|
|
||||||
return "", errors.New("not configured")
|
|
||||||
}
|
|
||||||
|
|
||||||
if _, err := config.client.PutObject(ctx, config.Bucket, key, body, -1, minio.PutObjectOptions{
|
if _, err := s.client.PutObject(ctx, bucket, key, body, -1, minio.PutObjectOptions{
|
||||||
ContentType: contentType,
|
ContentType: contentType,
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
return "", errors.Wrap(err, "put object")
|
return "", errors.Wrap(err, "put object")
|
||||||
}
|
}
|
||||||
|
|
||||||
return config.bucketURL.JoinPath(key).String(), nil
|
return s.bucketURL.JoinPath(key).String(), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *s3) Get(ctx context.Context, key string) (publicURL string, err error) {
|
func (s *s3) Get(ctx context.Context, key string) (publicURL string, err error) {
|
||||||
ctx = telemetry.StartWith(ctx, append(s.TelemetryLabels(), telemetrymodel.KeyOperation, "Get")...)
|
ctx = telemetry.StartWith(ctx, append(s.TelemetryLabels(), telemetrymodel.KeyOperation, "Get")...)
|
||||||
defer func() { telemetry.End(ctx, err) }()
|
defer func() { telemetry.End(ctx, err) }()
|
||||||
config := s.Config()
|
bucket := s.Config().Bucket
|
||||||
if config.Empty() {
|
|
||||||
return "", errors.New("not configured")
|
|
||||||
}
|
|
||||||
|
|
||||||
if _, err := config.client.StatObject(ctx, config.Bucket, key, minio.StatObjectOptions{}); err != nil {
|
if _, err := s.client.StatObject(ctx, bucket, key, minio.StatObjectOptions{}); err != nil {
|
||||||
errResponse := minio.ToErrorResponse(err)
|
errResponse := minio.ToErrorResponse(err)
|
||||||
if errResponse.Code == minio.NoSuchKey {
|
if errResponse.Code == minio.NoSuchKey {
|
||||||
return "", ErrNotFound
|
return "", ErrNotFound
|
||||||
@@ -186,7 +170,7 @@ func (s *s3) Get(ctx context.Context, key string) (publicURL string, err error)
|
|||||||
return "", errors.Wrap(err, "stat object")
|
return "", errors.Wrap(err, "stat object")
|
||||||
}
|
}
|
||||||
|
|
||||||
return config.bucketURL.JoinPath(key).String(), nil
|
return s.bucketURL.JoinPath(key).String(), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *s3) Reload(app *config.App) (err error) {
|
func (s *s3) Reload(app *config.App) (err error) {
|
||||||
@@ -199,7 +183,29 @@ func (s *s3) Reload(app *config.App) (err error) {
|
|||||||
return errors.Wrap(err, "validate config")
|
return errors.Wrap(err, "validate config")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if reflect.DeepEqual(s.Config(), newConfig) {
|
||||||
|
log.Debug(ctx, "object storage config not changed")
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
client, err := minio.New(newConfig.Endpoint, &minio.Options{
|
||||||
|
Creds: credentials.NewStaticV4(newConfig.AccessKeyID, newConfig.SecretAccessKey, ""),
|
||||||
|
Secure: true,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return errors.Wrap(err, "new minio client")
|
||||||
|
}
|
||||||
|
|
||||||
|
u, err := url.Parse(newConfig.BucketURL)
|
||||||
|
if err != nil {
|
||||||
|
return errors.Wrap(err, "parse public url")
|
||||||
|
}
|
||||||
|
|
||||||
|
s.client = client
|
||||||
|
s.bucketURL = u
|
||||||
s.SetConfig(newConfig)
|
s.SetConfig(newConfig)
|
||||||
|
|
||||||
log.Info(ctx, "object storage reloaded")
|
log.Info(ctx, "object storage reloaded")
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
Reference in New Issue
Block a user