Send kafka notification messages in batches when queue_dir is enabled (#18164)

Fixes #18124

internal/store/store.go

@@ -39,7 +39,7 @@ var ErrNotConnected = errors.New("not connected to target server/service")
 // Target - store target interface
 type Target interface {
 	Name() string
-	SendFromStore(key string) error
+	SendFromStore(key Key) error
 }
 
 // Store - Used to persist items.
@@ -49,16 +49,23 @@ type Store[I any] interface {
 	Len() int
 	List() ([]string, error)
 	Del(key string) error
+	DelList(key []string) error
 	Open() error
 	Extension() string
 }
 
+// Key denotes the key present in the store.
+type Key struct {
+	Name   string
+	IsLast bool
+}
+
 // replayItems - Reads the items from the store and replays.
-func replayItems[I any](store Store[I], doneCh <-chan struct{}, log logger, id string) <-chan string {
-	itemKeyCh := make(chan string)
+func replayItems[I any](store Store[I], doneCh <-chan struct{}, log logger, id string) <-chan Key {
+	keyCh := make(chan Key)
 
 	go func() {
-		defer close(itemKeyCh)
+		defer close(keyCh)
 
 		retryTicker := time.NewTicker(retryInterval)
 		defer retryTicker.Stop()
@@ -68,9 +75,10 @@ func replayItems[I any](store Store[I], doneCh <-chan struct{}, log logger, id s
 			if err != nil {
 				log(context.Background(), fmt.Errorf("store.List() failed with: %w", err), id)
 			} else {
-				for _, name := range names {
+				keyCount := len(names)
+				for i, name := range names {
 					select {
-					case itemKeyCh <- strings.TrimSuffix(name, store.Extension()):
+					case keyCh <- Key{strings.TrimSuffix(name, store.Extension()), keyCount == i+1}:
 					// Get next key.
 					case <-doneCh:
 						return
@@ -86,17 +94,17 @@ func replayItems[I any](store Store[I], doneCh <-chan struct{}, log logger, id s
 		}
 	}()
 
-	return itemKeyCh
+	return keyCh
 }
 
 // sendItems - Reads items from the store and re-plays.
-func sendItems(target Target, itemKeyCh <-chan string, doneCh <-chan struct{}, logger logger) {
+func sendItems(target Target, keyCh <-chan Key, doneCh <-chan struct{}, logger logger) {
 	retryTicker := time.NewTicker(retryInterval)
 	defer retryTicker.Stop()
 
-	send := func(itemKey string) bool {
+	send := func(key Key) bool {
 		for {
-			err := target.SendFromStore(itemKey)
+			err := target.SendFromStore(key)
 			if err == nil {
 				break
 			}
@@ -120,13 +128,13 @@ func sendItems(target Target, itemKeyCh <-chan string, doneCh <-chan struct{}, l
 
 	for {
 		select {
-		case itemKey, ok := <-itemKeyCh:
+		case key, ok := <-keyCh:
 			if !ok {
 				// closed channel.
 				return
 			}
 
-			if !send(itemKey) {
+			if !send(key) {
 				return
 			}
 		case <-doneCh:
@@ -139,8 +147,8 @@ func sendItems(target Target, itemKeyCh <-chan string, doneCh <-chan struct{}, l
 func StreamItems[I any](store Store[I], target Target, doneCh <-chan struct{}, logger logger) {
 	go func() {
 		// Replays the items from the store.
-		itemKeyCh := replayItems(store, doneCh, logger, target.Name())
+		keyCh := replayItems(store, doneCh, logger, target.Name())
 		// Send items from the store.
-		sendItems(target, itemKeyCh, doneCh, logger)
+		sendItems(target, keyCh, doneCh, logger)
 	}()
 }