diff --git a/database/mock/store.go b/database/mock/store.go index 4ee5419eb5..a630e15758 100644 --- a/database/mock/store.go +++ b/database/mock/store.go @@ -1913,6 +1913,21 @@ func (mr *MockStoreMockRecorder) ListEntitiesAfterID(ctx, arg any) *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListEntitiesAfterID", reflect.TypeOf((*MockStore)(nil).ListEntitiesAfterID), ctx, arg) } +// ListEntitiesByTypePaginated mocks base method. +func (m *MockStore) ListEntitiesByTypePaginated(ctx context.Context, arg db.ListEntitiesByTypePaginatedParams) ([]db.EntityInstance, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListEntitiesByTypePaginated", ctx, arg) + ret0, _ := ret[0].([]db.EntityInstance) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListEntitiesByTypePaginated indicates an expected call of ListEntitiesByTypePaginated. +func (mr *MockStoreMockRecorder) ListEntitiesByTypePaginated(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListEntitiesByTypePaginated", reflect.TypeOf((*MockStore)(nil).ListEntitiesByTypePaginated), ctx, arg) +} + // ListEvaluationHistory mocks base method. func (m *MockStore) ListEvaluationHistory(ctx context.Context, arg db.ListEvaluationHistoryParams) ([]db.ListEvaluationHistoryRow, error) { m.ctrl.T.Helper() diff --git a/database/query/entities.sql b/database/query/entities.sql index 771a008251..047c7e7bcc 100644 --- a/database/query/entities.sql +++ b/database/query/entities.sql @@ -80,6 +80,19 @@ WHERE entity_instances.entity_type = $1 ORDER BY entity_instances.id LIMIT sqlc.arg('limit')::bigint; +-- ListEntitiesByTypePaginated retrieves entities of a given type for a project/provider +-- with cursor-based pagination. Pass a null (all-zeros) UUID as cursor for the first page. +-- The query over-fetches one row to determine if there are more results. + +-- name: ListEntitiesByTypePaginated :many +SELECT * FROM entity_instances +WHERE entity_type = sqlc.arg(entity_type) + AND provider_id = sqlc.arg(provider_id) + AND project_id = ANY(sqlc.arg(projects)::uuid[]) + AND id > sqlc.arg(cursor)::uuid +ORDER BY id +LIMIT (sqlc.arg(page_limit)::bigint + 1); + -- EntityExistsAfterID checks if any entity of a given type exists after a cursor ID. -- name: EntityExistsAfterID :one diff --git a/internal/controlplane/handlers_repositories.go b/internal/controlplane/handlers_repositories.go index 5fd823e857..ba38179218 100644 --- a/internal/controlplane/handlers_repositories.go +++ b/internal/controlplane/handlers_repositories.go @@ -135,9 +135,11 @@ func (s *Server) repoCreateInfoFromUpstreamEntityRef( // ListRepositories returns a list of repositories for a given project // This function will typically be called by the client to get a list of -// repositories that are registered present in the minder database +// repositories that are registered present in the minder database. +// Pagination is cursor-based: pass an empty cursor for the first page, +// then use the returned cursor for subsequent pages. func (s *Server) ListRepositories(ctx context.Context, - _ *pb.ListRepositoriesRequest) (*pb.ListRepositoriesResponse, error) { + in *pb.ListRepositoriesRequest) (*pb.ListRepositoriesResponse, error) { entityCtx := engcontext.EntityFromContext(ctx) projectID := entityCtx.Project.ID providerName := entityCtx.Provider.Name @@ -154,8 +156,18 @@ func (s *Server) ListRepositories(ctx context.Context, return nil, status.Errorf(codes.Internal, "cannot get provider: %v", err) } - // Fetch repositories using the service - repoEntities, err := s.repos.ListRepositories(ctx, projectID, provider.ID) + // Parse cursor from request (base64-encoded UUID) + cursorStr := in.GetCursor() + + // Use request limit if provided, otherwise default + limit := in.GetLimit() + if limit <= 0 { + limit = 100 + } + + // Fetch repositories using the paginated service method + repoEntities, nextCursor, err := s.repos.ListRepositoriesPaginated( + ctx, projectID, provider.ID, cursorStr, limit) if err != nil { return nil, err } @@ -191,11 +203,8 @@ func (s *Server) ListRepositories(ctx context.Context, results = append(results, pbRepo) } - // TODO: Implement cursor-based pagination using entity IDs - // For now, return all results without pagination - resp.Results = results - resp.Cursor = "" + resp.Cursor = nextCursor return &resp, nil } diff --git a/internal/controlplane/handlers_repositories_test.go b/internal/controlplane/handlers_repositories_test.go index ce7f31e8ae..4e2cea9cc3 100644 --- a/internal/controlplane/handlers_repositories_test.go +++ b/internal/controlplane/handlers_repositories_test.go @@ -175,9 +175,12 @@ func TestServer_ListRepositories(t *testing.T) { { Name: "List repositories succeeds with multiple results", RepoServiceSetup: rf.NewRepoService( - rf.WithSuccessfulListRepositories( - simpleDbRepository(repoName, remoteRepoId), - simpleDbRepository(repoName2, remoteRepoId2), + rf.WithSuccessfulListRepositoriesPaginated( + []*models.EntityWithProperties{ + simpleDbRepository(repoName, remoteRepoId), + simpleDbRepository(repoName2, remoteRepoId2), + }, + "", ), ), ProviderSetup: func(ctrl *gomock.Controller, mgr *mockmanager.MockProviderManager) { @@ -246,14 +249,14 @@ func TestServer_ListRepositories(t *testing.T) { { Name: "List repositories succeeds with empty results", RepoServiceSetup: rf.NewRepoService( - rf.WithSuccessfulListRepositories(), + rf.WithSuccessfulListRepositoriesPaginated(nil, ""), ), ExpectedResults: nil, }, { Name: "List repositories fails when repo service returns error", RepoServiceSetup: rf.NewRepoService( - rf.WithFailedListRepositories(errDefault), + rf.WithFailedListRepositoriesPaginated(errDefault), ), ExpectedError: errDefault.Error(), }, diff --git a/internal/db/entities.sql.go b/internal/db/entities.sql.go index fe75ce30cf..19c688358a 100644 --- a/internal/db/entities.sql.go +++ b/internal/db/entities.sql.go @@ -590,6 +590,65 @@ func (q *Queries) ListEntitiesAfterID(ctx context.Context, arg ListEntitiesAfter return items, nil } +const listEntitiesByTypePaginated = `-- name: ListEntitiesByTypePaginated :many + +SELECT id, entity_type, name, project_id, provider_id, created_at, originated_from FROM entity_instances +WHERE entity_type = $1 + AND provider_id = $2 + AND project_id = ANY($3::uuid[]) + AND id > $4::uuid +ORDER BY id +LIMIT ($5::bigint + 1) +` + +type ListEntitiesByTypePaginatedParams struct { + EntityType Entities `json:"entity_type"` + ProviderID uuid.UUID `json:"provider_id"` + Projects []uuid.UUID `json:"projects"` + Cursor uuid.UUID `json:"cursor"` + PageLimit int64 `json:"page_limit"` +} + +// ListEntitiesByTypePaginated retrieves entities of a given type for a project/provider +// with cursor-based pagination. Pass a null (all-zeros) UUID as cursor for the first page. +// The query over-fetches one row to determine if there are more results. +func (q *Queries) ListEntitiesByTypePaginated(ctx context.Context, arg ListEntitiesByTypePaginatedParams) ([]EntityInstance, error) { + rows, err := q.db.QueryContext(ctx, listEntitiesByTypePaginated, + arg.EntityType, + arg.ProviderID, + pq.Array(arg.Projects), + arg.Cursor, + arg.PageLimit, + ) + if err != nil { + return nil, err + } + defer rows.Close() + items := []EntityInstance{} + for rows.Next() { + var i EntityInstance + if err := rows.Scan( + &i.ID, + &i.EntityType, + &i.Name, + &i.ProjectID, + &i.ProviderID, + &i.CreatedAt, + &i.OriginatedFrom, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const upsertProperty = `-- name: UpsertProperty :one INSERT INTO properties ( entity_id, diff --git a/internal/db/querier.go b/internal/db/querier.go index 5c6286796f..acdd46d2f9 100644 --- a/internal/db/querier.go +++ b/internal/db/querier.go @@ -207,6 +207,10 @@ type Querier interface { // ListEntitiesAfterID retrieves entities of a given type after a cursor ID, for pagination. // This is used for cursor-based iteration over all entities (e.g., in the reminder service). ListEntitiesAfterID(ctx context.Context, arg ListEntitiesAfterIDParams) ([]EntityInstance, error) + // ListEntitiesByTypePaginated retrieves entities of a given type for a project/provider + // with cursor-based pagination. Pass a null (all-zeros) UUID as cursor for the first page. + // The query over-fetches one row to determine if there are more results. + ListEntitiesByTypePaginated(ctx context.Context, arg ListEntitiesByTypePaginatedParams) ([]EntityInstance, error) ListEvaluationHistory(ctx context.Context, arg ListEvaluationHistoryParams) ([]ListEvaluationHistoryRow, error) ListEvaluationHistoryStaleRecords(ctx context.Context, arg ListEvaluationHistoryStaleRecordsParams) ([]ListEvaluationHistoryStaleRecordsRow, error) ListFlushCache(ctx context.Context) ([]FlushCache, error) diff --git a/internal/repositories/mock/fixtures/service.go b/internal/repositories/mock/fixtures/service.go index ad34557208..891b67d5e8 100644 --- a/internal/repositories/mock/fixtures/service.go +++ b/internal/repositories/mock/fixtures/service.go @@ -138,3 +138,34 @@ func WithFailedListRepositories(err error) func(RepoServiceMock) { Return(nil, err).AnyTimes() } } + +func WithSuccessfulListRepositoriesPaginated( + repositories []*models.EntityWithProperties, + nextCursor string, +) func(RepoServiceMock) { + return func(mock RepoServiceMock) { + mock.EXPECT(). + ListRepositoriesPaginated( + gomock.Any(), + gomock.Any(), + gomock.Any(), + gomock.Any(), + gomock.Any(), + ). + Return(repositories, nextCursor, nil).AnyTimes() + } +} + +func WithFailedListRepositoriesPaginated(err error) func(RepoServiceMock) { + return func(mock RepoServiceMock) { + mock.EXPECT(). + ListRepositoriesPaginated( + gomock.Any(), + gomock.Any(), + gomock.Any(), + gomock.Any(), + gomock.Any(), + ). + Return(nil, "", err).AnyTimes() + } +} diff --git a/internal/repositories/mock/service.go b/internal/repositories/mock/service.go index a06c3faa5d..f4f064fa9a 100644 --- a/internal/repositories/mock/service.go +++ b/internal/repositories/mock/service.go @@ -132,3 +132,19 @@ func (mr *MockRepositoryServiceMockRecorder) ListRepositories(ctx, projectID, pr mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListRepositories", reflect.TypeOf((*MockRepositoryService)(nil).ListRepositories), ctx, projectID, providerID) } + +// ListRepositoriesPaginated mocks base method. +func (m *MockRepositoryService) ListRepositoriesPaginated(ctx context.Context, projectID, providerID uuid.UUID, cursor string, limit int64) ([]*models.EntityWithProperties, string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListRepositoriesPaginated", ctx, projectID, providerID, cursor, limit) + ret0, _ := ret[0].([]*models.EntityWithProperties) + ret1, _ := ret[1].(string) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// ListRepositoriesPaginated indicates an expected call of ListRepositoriesPaginated. +func (mr *MockRepositoryServiceMockRecorder) ListRepositoriesPaginated(ctx, projectID, providerID, cursor, limit any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListRepositoriesPaginated", reflect.TypeOf((*MockRepositoryService)(nil).ListRepositoriesPaginated), ctx, projectID, providerID, cursor, limit) +} diff --git a/internal/repositories/service.go b/internal/repositories/service.go index 1dea8b818a..8b61039444 100644 --- a/internal/repositories/service.go +++ b/internal/repositories/service.go @@ -7,6 +7,7 @@ package repositories import ( "context" "database/sql" + "encoding/base64" "errors" "fmt" @@ -70,6 +71,16 @@ type RepositoryService interface { providerID uuid.UUID, ) ([]*models.EntityWithProperties, error) + // ListRepositoriesPaginated retrieves repositories with cursor-based pagination. + // If cursor is empty, returns the first page. limit defaults to 100. + ListRepositoriesPaginated( + ctx context.Context, + projectID uuid.UUID, + providerID uuid.UUID, + cursor string, + limit int64, + ) ([]*models.EntityWithProperties, string, error) + // GetRepositoryById retrieves a repository by its ID and project. GetRepositoryById(ctx context.Context, repositoryID uuid.UUID, projectID uuid.UUID) (*pb.Repository, error) // GetRepositoryByName retrieves a repository by its name, owner, project and provider (if specified). @@ -210,6 +221,112 @@ func (r *repositoryService) ListRepositories( return ents, nil } +// DefaultListRepositoriesPageLimit is the default (and maximum) page size for +// ListRepositoriesPaginated when the caller does not request a specific value +// or requests one above the cap. +const DefaultListRepositoriesPageLimit = 100 + +func (r *repositoryService) ListRepositoriesPaginated( + ctx context.Context, + projectID uuid.UUID, + providerID uuid.UUID, + cursorStr string, + limit int64, +) (ents []*models.EntityWithProperties, nextCursor string, outErr error) { + if limit <= 0 || limit > DefaultListRepositoriesPageLimit { + limit = DefaultListRepositoriesPageLimit + } + + cursor, err := decodeCursor(cursorStr) + if err != nil { + return nil, "", err + } + + tx, err := r.store.BeginTransaction() + if err != nil { + return nil, "", fmt.Errorf("error starting transaction: %w", err) + } + + defer func() { + if outErr != nil { + if rbErr := tx.Rollback(); rbErr != nil && !errors.Is(rbErr, sql.ErrTxDone) { + zerolog.Ctx(ctx).Error().Err(rbErr).Msg("error rolling back transaction") + } + } + }() + + qtx := r.store.GetQuerierWithTransaction(tx) + + repoEnts, err := qtx.ListEntitiesByTypePaginated(ctx, db.ListEntitiesByTypePaginatedParams{ + EntityType: db.EntitiesRepository, + ProviderID: providerID, + Projects: []uuid.UUID{projectID}, + Cursor: cursor, + PageLimit: limit, + }) + if err != nil { + return nil, "", fmt.Errorf("error fetching repositories: %w", err) + } + + hasMore := int64(len(repoEnts)) > limit + if hasMore { + repoEnts = repoEnts[:limit] + } + + ents = make([]*models.EntityWithProperties, 0, len(repoEnts)) + var lastID uuid.UUID + for _, ent := range repoEnts { + ewp, err := r.hydrateEntityProperties(ctx, qtx, ent.ID) + if err != nil { + return nil, "", err + } + ents = append(ents, ewp) + lastID = ent.ID + } + + if err := tx.Commit(); err != nil { + return nil, "", fmt.Errorf("error committing transaction: %w", err) + } + + if hasMore && lastID != uuid.Nil { + nextCursor = base64.StdEncoding.EncodeToString(lastID[:]) + } + + return ents, nextCursor, nil +} + +func (r *repositoryService) hydrateEntityProperties( + ctx context.Context, + qtx db.ExtendQuerier, + entityID uuid.UUID, +) (*models.EntityWithProperties, error) { + ewp, err := r.propSvc.EntityWithPropertiesByID(ctx, entityID, + service.CallBuilder().WithStoreOrTransaction(qtx)) + if err != nil { + return nil, fmt.Errorf("error fetching properties for repository: %w", err) + } + if err := r.propSvc.RetrieveAllPropertiesForEntity(ctx, ewp, r.providerManager, + service.ReadBuilder().WithStoreOrTransaction(qtx).TolerateStaleData()); err != nil { + return nil, fmt.Errorf("error fetching properties for repository: %w", err) + } + return ewp, nil +} + +func decodeCursor(cursorStr string) (uuid.UUID, error) { + if cursorStr == "" { + return uuid.Nil, nil + } + cursorBytes, err := base64.StdEncoding.DecodeString(cursorStr) + if err != nil { + return uuid.Nil, fmt.Errorf("invalid cursor format: %w", err) + } + cursor, err := uuid.FromBytes(cursorBytes) + if err != nil { + return uuid.Nil, fmt.Errorf("invalid cursor format: %w", err) + } + return cursor, nil +} + func (r *repositoryService) GetRepositoryById( ctx context.Context, repositoryID uuid.UUID,