use a more traditional linked list

Author: jcalabro

internal/server/redis/list.go

@@ -3,7 +3,6 @@ package redis
 import (
 	"context"
 	"fmt"
-	"math"
 	"strconv"
 
 	"github.com/apple/foundationdb/bindings/go/src/fdb"
@@ -98,27 +97,43 @@ func (s *session) handlePush(ctx context.Context, args []resp.Value, left bool)
 		if listMeta == nil {
 			// create a new list
 			listMeta = &types.ListMeta{
-				Created:   now,
-				ItemStart: math.MaxUint64 / 2,
-				ItemEnd:   math.MaxUint64 / 2,
+				Created: now,
 			}
 		}
 
-		// assign slots for these new list items and create object blobs for each
+		// create each item meta object and store the blob itself
 		for _, member := range members {
-			listMeta.NumItems += 1
+			objIDInt, err := s.allocateNewUID(ctx, tx)
+			if err != nil {
+				return nil, fmt.Errorf("failed to allocate new uid: %w", err)
+			}
+			objID := strconv.FormatUint(objIDInt, 10)
 
-			var objKeyInt uint64
+			objMeta := &types.ListObjectMeta{
+				Created: now,
+				Updated: now,
+				Id:      objID,
+			}
+
+			listMeta.NumItems += 1
 			if left {
-				listMeta.ItemStart -= 1
-				objKeyInt = listMeta.ItemStart
+				objMeta.Next = listMeta.ItemHead
+				listMeta.ItemHead = objID
 			} else {
-				listMeta.ItemEnd += 1
-				objKeyInt = listMeta.ItemEnd
+				objMeta.Previous = listMeta.ItemTail
+				listMeta.ItemTail = objID
+			}
+
+			objMetaKey, err := s.listObjMetaKey(key, objID)
+			if err != nil {
+				return nil, fmt.Errorf("failed to get list object meta key: %w", err)
+			}
+
+			if err := s.setProtoItem(objMetaKey, objMeta); err != nil {
+				return nil, fmt.Errorf("failed to write list object meta: %w", err)
 			}
 
-			objKey := strconv.FormatUint(objKeyInt, 10)
-			if err := s.writeObject(ctx, tx, objKey, []byte(member)); err != nil {
+			if err := s.writeObject(ctx, tx, objID, []byte(member)); err != nil {
 				return nil, fmt.Errorf("failed to create object: %w", err)
 			}
 		}

internal/server/redis/redis.go

@@ -254,6 +254,22 @@ func (s *session) listKey(id string) (fdb.Key, error) {
 	return s.user.listDir.Pack(tuple.Tuple{id}), nil
 }
 
+// Returns the FDB key of an object in the per-user, per-list item directory
+func (s *session) listObjMetaKey(listID, objID string) (fdb.Key, error) {
+	s.userMu.RLock()
+	defer s.userMu.RUnlock()
+
+	if s.user == nil {
+		return nil, fmt.Errorf("authentication is required")
+	}
+
+	dir, err := s.user.listDir.CreateOrOpen(s.fdb, []string{listID}, nil)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open per-list directory: %w", err)
+	}
+
+	return dir.Pack(tuple.Tuple{objID}), nil
+}
 func (s *session) write(msg string) {
 	_, err := s.conn.Write([]byte(msg))
 	if err != nil {

internal/server/redis/redis_test.go

@@ -3,14 +3,12 @@ package redis
 import (
 	"bufio"
 	"bytes"
-	"math"
 	"math/rand/v2"
 	"strings"
 	"testing"
 
 	"github.com/apple/foundationdb/bindings/go/src/fdb"
 	"github.com/bluesky-social/kvdb/internal/testutil"
-	"github.com/bluesky-social/kvdb/internal/types"
 	"github.com/bluesky-social/kvdb/pkg/serde/resp"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/crypto/bcrypt"
@@ -962,21 +960,4 @@ func TestLists(t *testing.T) {
 		Args: []resp.Value{resp.SimpleStringValue(list)},
 	})
 	require.Equal(resp.FormatInt(5), res)
-
-	{
-		// ensure our item meta is correct
-		metaAny, err := sess.fdb.ReadTransact(func(tx fdb.ReadTransaction) (any, error) {
-			_, meta, err := sess.getListMeta(ctx, tx, list)
-			return meta, err
-		})
-		require.NoError(err)
-		require.NotNil(metaAny)
-
-		meta, err := cast[*types.ListMeta](metaAny)
-		require.NoError(err)
-
-		mid := uint64(math.MaxUint64 / 2)
-		require.Equal(mid-3, meta.ItemStart)
-		require.Equal(mid+2, meta.ItemEnd)
-	}
 }

internal/types/redis.pb.go

@@ -38,6 +38,14 @@ message ListMeta {
   google.protobuf.Timestamp updated = 2;
   google.protobuf.Timestamp last_access = 3;
   uint64 num_items = 4;
-  uint64 item_start = 5;
-  uint64 item_end = 6;
+  string item_head = 5;
+  string item_tail= 6;
+}
+
+message ListObjectMeta {
+  google.protobuf.Timestamp created = 1;
+  google.protobuf.Timestamp updated = 2;
+  string id = 3;
+  string next = 4;
+  string previous = 5;
 }