-
Notifications
You must be signed in to change notification settings - Fork 107
fix: implement cursor-based pagination for ListRepositories endpoint #6342
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
5b5fad3
6c4d591
81e80be
a9ab09b
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -138,3 +138,34 @@ func WithFailedListRepositories(err error) func(RepoServiceMock) { | |
| Return(nil, err).AnyTimes() | ||
| } | ||
| } | ||
|
|
||
| func WithSuccessfulListRepositoriesPaginated( | ||
| repositories []*models.EntityWithProperties, | ||
| nextCursor string, | ||
| ) func(RepoServiceMock) { | ||
| return func(mock RepoServiceMock) { | ||
| mock.EXPECT(). | ||
| ListRepositoriesPaginated( | ||
| gomock.Any(), | ||
| gomock.Any(), | ||
| gomock.Any(), | ||
| gomock.Any(), | ||
| gomock.Any(), | ||
| ). | ||
| Return(repositories, nextCursor, nil).AnyTimes() | ||
| } | ||
|
Comment on lines
+154
to
+156
|
||
| } | ||
|
|
||
| func WithFailedListRepositoriesPaginated(err error) func(RepoServiceMock) { | ||
| return func(mock RepoServiceMock) { | ||
| mock.EXPECT(). | ||
| ListRepositoriesPaginated( | ||
| gomock.Any(), | ||
| gomock.Any(), | ||
| gomock.Any(), | ||
| gomock.Any(), | ||
| gomock.Any(), | ||
| ). | ||
| Return(nil, "", err).AnyTimes() | ||
| } | ||
| } | ||
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -7,6 +7,7 @@ package repositories | |
| import ( | ||
| "context" | ||
| "database/sql" | ||
| "encoding/base64" | ||
| "errors" | ||
| "fmt" | ||
|
|
||
|
|
@@ -70,6 +71,16 @@ type RepositoryService interface { | |
| providerID uuid.UUID, | ||
| ) ([]*models.EntityWithProperties, error) | ||
|
|
||
| // ListRepositoriesPaginated retrieves repositories with cursor-based pagination. | ||
| // If cursor is empty, returns the first page. limit defaults to 100. | ||
| ListRepositoriesPaginated( | ||
| ctx context.Context, | ||
| projectID uuid.UUID, | ||
| providerID uuid.UUID, | ||
| cursor string, | ||
| limit int64, | ||
| ) ([]*models.EntityWithProperties, string, error) | ||
|
|
||
| // GetRepositoryById retrieves a repository by its ID and project. | ||
| GetRepositoryById(ctx context.Context, repositoryID uuid.UUID, projectID uuid.UUID) (*pb.Repository, error) | ||
| // GetRepositoryByName retrieves a repository by its name, owner, project and provider (if specified). | ||
|
|
@@ -210,6 +221,112 @@ func (r *repositoryService) ListRepositories( | |
| return ents, nil | ||
| } | ||
|
|
||
| // DefaultListRepositoriesPageLimit is the default (and maximum) page size for | ||
| // ListRepositoriesPaginated when the caller does not request a specific value | ||
| // or requests one above the cap. | ||
| const DefaultListRepositoriesPageLimit = 100 | ||
|
|
||
| func (r *repositoryService) ListRepositoriesPaginated( | ||
| ctx context.Context, | ||
| projectID uuid.UUID, | ||
| providerID uuid.UUID, | ||
| cursorStr string, | ||
| limit int64, | ||
| ) (ents []*models.EntityWithProperties, nextCursor string, outErr error) { | ||
| if limit <= 0 || limit > DefaultListRepositoriesPageLimit { | ||
| limit = DefaultListRepositoriesPageLimit | ||
| } | ||
|
Comment on lines
+236
to
+238
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. If you remove the upper check on |
||
|
|
||
| cursor, err := decodeCursor(cursorStr) | ||
| if err != nil { | ||
| return nil, "", err | ||
| } | ||
|
|
||
| tx, err := r.store.BeginTransaction() | ||
| if err != nil { | ||
| return nil, "", fmt.Errorf("error starting transaction: %w", err) | ||
| } | ||
|
|
||
| defer func() { | ||
| if outErr != nil { | ||
| if rbErr := tx.Rollback(); rbErr != nil && !errors.Is(rbErr, sql.ErrTxDone) { | ||
| zerolog.Ctx(ctx).Error().Err(rbErr).Msg("error rolling back transaction") | ||
| } | ||
| } | ||
| }() | ||
|
|
||
| qtx := r.store.GetQuerierWithTransaction(tx) | ||
|
|
||
| repoEnts, err := qtx.ListEntitiesByTypePaginated(ctx, db.ListEntitiesByTypePaginatedParams{ | ||
| EntityType: db.EntitiesRepository, | ||
| ProviderID: providerID, | ||
| Projects: []uuid.UUID{projectID}, | ||
| Cursor: cursor, | ||
| PageLimit: limit, | ||
| }) | ||
| if err != nil { | ||
| return nil, "", fmt.Errorf("error fetching repositories: %w", err) | ||
| } | ||
|
|
||
| hasMore := int64(len(repoEnts)) > limit | ||
| if hasMore { | ||
| repoEnts = repoEnts[:limit] | ||
| } | ||
|
|
||
| ents = make([]*models.EntityWithProperties, 0, len(repoEnts)) | ||
| var lastID uuid.UUID | ||
| for _, ent := range repoEnts { | ||
| ewp, err := r.hydrateEntityProperties(ctx, qtx, ent.ID) | ||
| if err != nil { | ||
| return nil, "", err | ||
| } | ||
| ents = append(ents, ewp) | ||
| lastID = ent.ID | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Rather than setting |
||
| } | ||
|
|
||
| if err := tx.Commit(); err != nil { | ||
| return nil, "", fmt.Errorf("error committing transaction: %w", err) | ||
| } | ||
|
|
||
| if hasMore && lastID != uuid.Nil { | ||
| nextCursor = base64.StdEncoding.EncodeToString(lastID[:]) | ||
| } | ||
|
|
||
| return ents, nextCursor, nil | ||
| } | ||
|
|
||
| func (r *repositoryService) hydrateEntityProperties( | ||
| ctx context.Context, | ||
| qtx db.ExtendQuerier, | ||
| entityID uuid.UUID, | ||
| ) (*models.EntityWithProperties, error) { | ||
| ewp, err := r.propSvc.EntityWithPropertiesByID(ctx, entityID, | ||
| service.CallBuilder().WithStoreOrTransaction(qtx)) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("error fetching properties for repository: %w", err) | ||
| } | ||
| if err := r.propSvc.RetrieveAllPropertiesForEntity(ctx, ewp, r.providerManager, | ||
| service.ReadBuilder().WithStoreOrTransaction(qtx).TolerateStaleData()); err != nil { | ||
| return nil, fmt.Errorf("error fetching properties for repository: %w", err) | ||
| } | ||
| return ewp, nil | ||
| } | ||
|
|
||
| func decodeCursor(cursorStr string) (uuid.UUID, error) { | ||
| if cursorStr == "" { | ||
| return uuid.Nil, nil | ||
| } | ||
|
Comment on lines
+316
to
+318
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think this needs a comment -- "empty string means start at the beginning" or suchlike. |
||
| cursorBytes, err := base64.StdEncoding.DecodeString(cursorStr) | ||
| if err != nil { | ||
| return uuid.Nil, fmt.Errorf("invalid cursor format: %w", err) | ||
| } | ||
| cursor, err := uuid.FromBytes(cursorBytes) | ||
| if err != nil { | ||
| return uuid.Nil, fmt.Errorf("invalid cursor format: %w", err) | ||
| } | ||
| return cursor, nil | ||
| } | ||
|
|
||
| func (r *repositoryService) GetRepositoryById( | ||
| ctx context.Context, | ||
| repositoryID uuid.UUID, | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This says it is a base64-encoded UUID, but there's no parsing being done here. In fact, it doesn't seem like we need the temporary
cursorStrat all -- it's only used once and we could just put it inline there.