init
This commit is contained in:
glidea
285
pkg/storage/feed/block/index/primary/primary.go
Normal file
285
pkg/storage/feed/block/index/primary/primary.go
Normal file
@@ -0,0 +1,285 @@
|
||||
package primary
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"io"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/glidea/zenfeed/pkg/component"
|
||||
"github.com/glidea/zenfeed/pkg/storage/feed/block/index"
|
||||
telemetry "github.com/glidea/zenfeed/pkg/telemetry"
|
||||
telemetrymodel "github.com/glidea/zenfeed/pkg/telemetry/model"
|
||||
)
|
||||
|
||||
// --- Interface code block ---
|
||||
type Index interface {
|
||||
component.Component
|
||||
index.Codec
|
||||
|
||||
// Search returns item location by ID.
|
||||
Search(ctx context.Context, id uint64) (ref FeedRef, ok bool)
|
||||
// Add adds item location to the index.
|
||||
Add(ctx context.Context, id uint64, item FeedRef)
|
||||
// IDs returns all item IDs.
|
||||
IDs(ctx context.Context) (ids map[uint64]bool)
|
||||
// Count returns the number of feeds in the index.
|
||||
Count(ctx context.Context) (count uint32)
|
||||
}
|
||||
|
||||
type Config struct{}
|
||||
|
||||
type Dependencies struct{}
|
||||
|
||||
var (
|
||||
headerMagicNumber = []byte{0x77, 0x79, 0x73, 0x20, 0x69, 0x73, 0x20,
|
||||
0x61, 0x77, 0x65, 0x73, 0x6f, 0x6d, 0x65, 0x00, 0x00}
|
||||
headerVersion = uint8(1)
|
||||
)
|
||||
|
||||
type FeedRef struct {
|
||||
Chunk uint32
|
||||
Offset uint64
|
||||
Time time.Time
|
||||
}
|
||||
|
||||
// --- Factory code block ---
|
||||
type Factory component.Factory[Index, Config, Dependencies]
|
||||
|
||||
func NewFactory(mockOn ...component.MockOption) Factory {
|
||||
if len(mockOn) > 0 {
|
||||
return component.FactoryFunc[Index, Config, Dependencies](
|
||||
func(instance string, config *Config, dependencies Dependencies) (Index, error) {
|
||||
m := &mockIndex{}
|
||||
component.MockOptions(mockOn).Apply(&m.Mock)
|
||||
|
||||
return m, nil
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
return component.FactoryFunc[Index, Config, Dependencies](new)
|
||||
}
|
||||
|
||||
func new(instance string, config *Config, dependencies Dependencies) (Index, error) {
|
||||
return &idx{
|
||||
Base: component.New(&component.BaseConfig[Config, Dependencies]{
|
||||
Name: "FeedPrimaryIndex",
|
||||
Instance: instance,
|
||||
Config: config,
|
||||
Dependencies: dependencies,
|
||||
}),
|
||||
m: make(map[uint64]FeedRef, 64),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// --- Implementation code block ---
|
||||
type idx struct {
|
||||
*component.Base[Config, Dependencies]
|
||||
|
||||
m map[uint64]FeedRef
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
func (idx *idx) Search(ctx context.Context, id uint64) (ref FeedRef, ok bool) {
|
||||
ctx = telemetry.StartWith(ctx, append(idx.TelemetryLabels(), telemetrymodel.KeyOperation, "Search")...)
|
||||
defer func() { telemetry.End(ctx, nil) }()
|
||||
|
||||
idx.mu.RLock()
|
||||
defer idx.mu.RUnlock()
|
||||
ref, ok = idx.m[id]
|
||||
|
||||
return ref, ok
|
||||
}
|
||||
|
||||
func (idx *idx) Add(ctx context.Context, id uint64, item FeedRef) {
|
||||
ctx = telemetry.StartWith(ctx, append(idx.TelemetryLabels(), telemetrymodel.KeyOperation, "Add")...)
|
||||
defer func() { telemetry.End(ctx, nil) }()
|
||||
|
||||
idx.mu.Lock()
|
||||
defer idx.mu.Unlock()
|
||||
item.Time = item.Time.In(time.UTC)
|
||||
idx.m[id] = item
|
||||
}
|
||||
|
||||
func (idx *idx) IDs(ctx context.Context) (ids map[uint64]bool) {
|
||||
ctx = telemetry.StartWith(ctx, append(idx.TelemetryLabels(), telemetrymodel.KeyOperation, "IDs")...)
|
||||
defer func() { telemetry.End(ctx, nil) }()
|
||||
|
||||
idx.mu.RLock()
|
||||
defer idx.mu.RUnlock()
|
||||
result := make(map[uint64]bool, len(idx.m))
|
||||
for id := range idx.m {
|
||||
result[id] = true
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
func (idx *idx) Count(ctx context.Context) (count uint32) {
|
||||
ctx = telemetry.StartWith(ctx, append(idx.TelemetryLabels(), telemetrymodel.KeyOperation, "Count")...)
|
||||
defer func() { telemetry.End(ctx, nil) }()
|
||||
|
||||
idx.mu.RLock()
|
||||
defer idx.mu.RUnlock()
|
||||
|
||||
return uint32(len(idx.m))
|
||||
}
|
||||
|
||||
func (idx *idx) EncodeTo(ctx context.Context, w io.Writer) (err error) {
|
||||
ctx = telemetry.StartWith(ctx, append(idx.TelemetryLabels(), telemetrymodel.KeyOperation, "EncodeTo")...)
|
||||
defer func() { telemetry.End(ctx, err) }()
|
||||
idx.mu.RLock()
|
||||
defer idx.mu.RUnlock()
|
||||
|
||||
// Write header.
|
||||
if _, err := w.Write(headerMagicNumber); err != nil {
|
||||
return errors.Wrap(err, "write header magic number")
|
||||
}
|
||||
if _, err := w.Write([]byte{headerVersion}); err != nil {
|
||||
return errors.Wrap(err, "write header version")
|
||||
}
|
||||
|
||||
// Write map count.
|
||||
count := uint64(len(idx.m))
|
||||
if err := binary.Write(w, binary.LittleEndian, count); err != nil {
|
||||
return errors.Wrap(err, "write map count")
|
||||
}
|
||||
|
||||
// Write all key-value pairs.
|
||||
for id, ref := range idx.m {
|
||||
// Write Key.
|
||||
if err := binary.Write(w, binary.LittleEndian, id); err != nil {
|
||||
return errors.Wrap(err, "write id")
|
||||
}
|
||||
|
||||
// Write Value.
|
||||
if err := binary.Write(w, binary.LittleEndian, ref.Chunk); err != nil {
|
||||
return errors.Wrap(err, "write chunk")
|
||||
}
|
||||
if err := binary.Write(w, binary.LittleEndian, ref.Offset); err != nil {
|
||||
return errors.Wrap(err, "write offset")
|
||||
}
|
||||
if err := binary.Write(w, binary.LittleEndian, ref.Time.UnixNano()); err != nil {
|
||||
return errors.Wrap(err, "write time")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (idx *idx) DecodeFrom(ctx context.Context, r io.Reader) (err error) {
|
||||
ctx = telemetry.StartWith(ctx, append(idx.TelemetryLabels(), telemetrymodel.KeyOperation, "DecodeFrom")...)
|
||||
defer func() { telemetry.End(ctx, err) }()
|
||||
idx.mu.Lock()
|
||||
defer idx.mu.Unlock()
|
||||
|
||||
// Read header.
|
||||
if err := idx.readHeader(r); err != nil {
|
||||
return errors.Wrap(err, "read header")
|
||||
}
|
||||
|
||||
// Read map count.
|
||||
var count uint64
|
||||
if err := binary.Read(r, binary.LittleEndian, &count); err != nil {
|
||||
return errors.Wrap(err, "read map count")
|
||||
}
|
||||
idx.m = make(map[uint64]FeedRef, count)
|
||||
|
||||
// Read all key-value pairs.
|
||||
for range count {
|
||||
id, ref, err := idx.readEntry(r)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "read entry")
|
||||
}
|
||||
idx.m[id] = ref
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// readHeader reads and validates the index file header.
|
||||
func (idx *idx) readHeader(r io.Reader) error {
|
||||
magicNumber := make([]byte, len(headerMagicNumber))
|
||||
if _, err := io.ReadFull(r, magicNumber); err != nil {
|
||||
return errors.Wrap(err, "read magic number")
|
||||
}
|
||||
if !bytes.Equal(magicNumber, headerMagicNumber) {
|
||||
return errors.New("invalid magic number")
|
||||
}
|
||||
|
||||
versionByte := make([]byte, 1)
|
||||
if _, err := io.ReadFull(r, versionByte); err != nil {
|
||||
return errors.Wrap(err, "read version")
|
||||
}
|
||||
if versionByte[0] != headerVersion {
|
||||
return errors.New("invalid version")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// readEntry reads a single key-value pair (feed ID and FeedRef) from the reader.
|
||||
func (idx *idx) readEntry(r io.Reader) (id uint64, ref FeedRef, err error) {
|
||||
// Read Key (ID).
|
||||
if err := binary.Read(r, binary.LittleEndian, &id); err != nil {
|
||||
return 0, FeedRef{}, errors.Wrap(err, "read id")
|
||||
}
|
||||
|
||||
// Read Value (FeedRef).
|
||||
if err := binary.Read(r, binary.LittleEndian, &ref.Chunk); err != nil {
|
||||
return 0, FeedRef{}, errors.Wrap(err, "read chunk")
|
||||
}
|
||||
if err := binary.Read(r, binary.LittleEndian, &ref.Offset); err != nil {
|
||||
return 0, FeedRef{}, errors.Wrap(err, "read offset")
|
||||
}
|
||||
var timestamp int64
|
||||
if err := binary.Read(r, binary.LittleEndian, ×tamp); err != nil {
|
||||
return 0, FeedRef{}, errors.Wrap(err, "read time")
|
||||
}
|
||||
ref.Time = time.Unix(0, timestamp).In(time.UTC)
|
||||
|
||||
return id, ref, nil
|
||||
}
|
||||
|
||||
type mockIndex struct {
|
||||
component.Mock
|
||||
}
|
||||
|
||||
func (m *mockIndex) Search(ctx context.Context, id uint64) (ref FeedRef, ok bool) {
|
||||
args := m.Called(ctx, id)
|
||||
|
||||
return args.Get(0).(FeedRef), args.Bool(1)
|
||||
}
|
||||
|
||||
func (m *mockIndex) Add(ctx context.Context, id uint64, item FeedRef) {
|
||||
m.Called(ctx, id, item)
|
||||
}
|
||||
|
||||
func (m *mockIndex) IDs(ctx context.Context) (ids map[uint64]bool) {
|
||||
args := m.Called(ctx)
|
||||
|
||||
return args.Get(0).(map[uint64]bool)
|
||||
}
|
||||
|
||||
func (m *mockIndex) Count(ctx context.Context) (count uint32) {
|
||||
args := m.Called(ctx)
|
||||
|
||||
return args.Get(0).(uint32)
|
||||
}
|
||||
|
||||
func (m *mockIndex) EncodeTo(ctx context.Context, w io.Writer) (err error) {
|
||||
args := m.Called(ctx, w)
|
||||
|
||||
return args.Error(0)
|
||||
}
|
||||
|
||||
func (m *mockIndex) DecodeFrom(ctx context.Context, r io.Reader) (err error) {
|
||||
args := m.Called(ctx, r)
|
||||
|
||||
return args.Error(0)
|
||||
}
|
||||
222
pkg/storage/feed/block/index/primary/primary_test.go
Normal file
222
pkg/storage/feed/block/index/primary/primary_test.go
Normal file
@@ -0,0 +1,222 @@
|
||||
package primary
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/glidea/zenfeed/pkg/test"
|
||||
)
|
||||
|
||||
func TestAdd(t *testing.T) {
|
||||
RegisterTestingT(t)
|
||||
|
||||
type givenDetail struct {
|
||||
existingItems map[uint64]FeedRef
|
||||
}
|
||||
type whenDetail struct {
|
||||
id uint64
|
||||
item FeedRef
|
||||
}
|
||||
type thenExpected struct {
|
||||
items map[uint64]FeedRef
|
||||
}
|
||||
|
||||
tests := []test.Case[givenDetail, whenDetail, thenExpected]{
|
||||
{
|
||||
Scenario: "Add Single Feed",
|
||||
Given: "An index with existing item",
|
||||
When: "Adding a single item",
|
||||
Then: "Should store the item correctly",
|
||||
GivenDetail: givenDetail{
|
||||
existingItems: map[uint64]FeedRef{
|
||||
0: {Chunk: 0, Offset: 0},
|
||||
},
|
||||
},
|
||||
WhenDetail: whenDetail{
|
||||
id: 1,
|
||||
item: FeedRef{Chunk: 1, Offset: 100},
|
||||
},
|
||||
ThenExpected: thenExpected{
|
||||
items: map[uint64]FeedRef{
|
||||
0: {Chunk: 0, Offset: 0},
|
||||
1: {Chunk: 1, Offset: 100},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Scenario: "Update Existing Feed",
|
||||
Given: "An index with existing item",
|
||||
When: "Adding item with same ID",
|
||||
Then: "Should update the item reference",
|
||||
GivenDetail: givenDetail{
|
||||
existingItems: map[uint64]FeedRef{
|
||||
1: {Chunk: 1, Offset: 100},
|
||||
},
|
||||
},
|
||||
WhenDetail: whenDetail{
|
||||
id: 1,
|
||||
item: FeedRef{Chunk: 2, Offset: 200},
|
||||
},
|
||||
ThenExpected: thenExpected{
|
||||
items: map[uint64]FeedRef{
|
||||
1: {Chunk: 2, Offset: 200},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.Scenario, func(t *testing.T) {
|
||||
// Given.
|
||||
idx0, err := NewFactory().New("test", &Config{}, Dependencies{})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
for id, item := range tt.GivenDetail.existingItems {
|
||||
idx0.Add(context.Background(), id, item)
|
||||
}
|
||||
|
||||
// When.
|
||||
idx0.Add(context.Background(), tt.WhenDetail.id, tt.WhenDetail.item)
|
||||
|
||||
// Then.
|
||||
primIdx := idx0.(*idx)
|
||||
for id, expected := range tt.ThenExpected.items {
|
||||
Expect(primIdx.m).To(HaveKey(id))
|
||||
Expect(primIdx.m[id]).To(Equal(expected))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSearch(t *testing.T) {
|
||||
RegisterTestingT(t)
|
||||
|
||||
type givenDetail struct {
|
||||
feeds map[uint64]FeedRef
|
||||
}
|
||||
type whenDetail struct {
|
||||
searchID uint64
|
||||
}
|
||||
type thenExpected struct {
|
||||
feedRef FeedRef
|
||||
found bool
|
||||
}
|
||||
|
||||
tests := []test.Case[givenDetail, whenDetail, thenExpected]{
|
||||
{
|
||||
Scenario: "Search Existing Feed",
|
||||
Given: "An index with feeds",
|
||||
When: "Searching for existing ID",
|
||||
Then: "Should return correct FeedRef",
|
||||
GivenDetail: givenDetail{
|
||||
feeds: map[uint64]FeedRef{
|
||||
1: {Chunk: 1, Offset: 100},
|
||||
2: {Chunk: 2, Offset: 200},
|
||||
},
|
||||
},
|
||||
WhenDetail: whenDetail{
|
||||
searchID: 1,
|
||||
},
|
||||
ThenExpected: thenExpected{
|
||||
feedRef: FeedRef{Chunk: 1, Offset: 100},
|
||||
found: true,
|
||||
},
|
||||
},
|
||||
{
|
||||
Scenario: "Search Non-Existing Feed",
|
||||
Given: "An index with feeds",
|
||||
When: "Searching for non-existing ID",
|
||||
Then: "Should return empty FeedRef",
|
||||
GivenDetail: givenDetail{
|
||||
feeds: map[uint64]FeedRef{
|
||||
1: {Chunk: 1, Offset: 100},
|
||||
},
|
||||
},
|
||||
WhenDetail: whenDetail{
|
||||
searchID: 2,
|
||||
},
|
||||
ThenExpected: thenExpected{
|
||||
feedRef: FeedRef{},
|
||||
found: false,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.Scenario, func(t *testing.T) {
|
||||
// Given.
|
||||
idx, err := NewFactory().New("test", &Config{}, Dependencies{})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
for id, item := range tt.GivenDetail.feeds {
|
||||
idx.Add(context.Background(), id, item)
|
||||
}
|
||||
|
||||
// When.
|
||||
result, ok := idx.Search(context.Background(), tt.WhenDetail.searchID)
|
||||
|
||||
// Then.
|
||||
Expect(result).To(Equal(tt.ThenExpected.feedRef))
|
||||
Expect(ok).To(Equal(tt.ThenExpected.found))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncodeDecode(t *testing.T) {
|
||||
RegisterTestingT(t)
|
||||
|
||||
type givenDetail struct {
|
||||
feeds map[uint64]FeedRef
|
||||
}
|
||||
type whenDetail struct{}
|
||||
type thenExpected struct {
|
||||
success bool
|
||||
}
|
||||
|
||||
tests := []test.Case[givenDetail, whenDetail, thenExpected]{
|
||||
{
|
||||
Scenario: "Encode and Decode Index with Data",
|
||||
Given: "An index with feeds",
|
||||
When: "Encoding and decoding",
|
||||
Then: "Should restore all data correctly",
|
||||
GivenDetail: givenDetail{
|
||||
feeds: map[uint64]FeedRef{
|
||||
1: {Chunk: 1, Offset: 100, Time: time.Now()},
|
||||
2: {Chunk: 2, Offset: 200, Time: time.Now()},
|
||||
},
|
||||
},
|
||||
WhenDetail: whenDetail{},
|
||||
ThenExpected: thenExpected{
|
||||
success: true,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.Scenario, func(t *testing.T) {
|
||||
// Given.
|
||||
original, err := NewFactory().New("test", &Config{}, Dependencies{})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
for id, item := range tt.GivenDetail.feeds {
|
||||
original.Add(context.Background(), id, item)
|
||||
}
|
||||
|
||||
// When.
|
||||
var buf bytes.Buffer
|
||||
err = original.EncodeTo(context.Background(), &buf)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
decoded, err := NewFactory().New("test", &Config{}, Dependencies{})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
err = decoded.DecodeFrom(context.Background(), &buf)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
// Then.
|
||||
origIdx := original.(*idx)
|
||||
decodedIdx := decoded.(*idx)
|
||||
Expect(decodedIdx.m).To(Equal(origIdx.m))
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user