diff --git a/src/adapters/postgres/cohortMembers-adapter.ts b/src/adapters/postgres/cohortMembers-adapter.ts index db3bdc51..878d8752 100644 --- a/src/adapters/postgres/cohortMembers-adapter.ts +++ b/src/adapters/postgres/cohortMembers-adapter.ts @@ -34,7 +34,13 @@ import { FormSubmissionSearchDto } from 'src/forms/dto/form-submission-search.dt import { FormsService } from 'src/forms/forms.service'; import { isElasticsearchEnabled } from 'src/common/utils/elasticsearch.util'; import { UserElasticsearchService } from 'src/elasticsearch/user-elasticsearch.service'; +import { ElasticsearchDataFetcherService } from 'src/elasticsearch/elasticsearch-data-fetcher.service'; import axios from 'axios'; +import { + ElasticsearchSyncService, + SyncSection, +} from '../../elasticsearch/elasticsearch-sync.service'; + @Injectable() export class PostgresCohortMembersService { constructor( @@ -56,7 +62,9 @@ export class PostgresCohortMembersService { private readonly userService: PostgresUserService, private readonly formsService: FormsService, private readonly formSubmissionService: FormSubmissionService, - private readonly userElasticsearchService: UserElasticsearchService + private readonly userElasticsearchService: UserElasticsearchService, + private readonly elasticsearchDataFetcherService: ElasticsearchDataFetcherService, + private readonly elasticsearchSyncService: ElasticsearchSyncService ) {} //Get cohort member @@ -720,7 +728,20 @@ export class PostgresCohortMembersService { // Update Elasticsearch with cohort member status if (isElasticsearchEnabled()) { try { - // First get the existing user document from Elasticsearch + // Use comprehensive sync to get complete user document including courses and assessment data + const userDocument = + await this.elasticsearchDataFetcherService.comprehensiveUserSync( + cohortMembers.userId + ); + + if (!userDocument) { + LoggerUtil.warn( + `User document not found for ${cohortMembers.userId}, skipping Elasticsearch update` + ); + return; + } + + // Get the existing user document from Elasticsearch const userDoc = await this.userElasticsearchService.getUser( cohortMembers.userId ); @@ -733,6 +754,7 @@ export class PostgresCohortMembersService { ? [...source.applications] : []; + // Find existing application for this cohort const appIndex = applications.findIndex( (app) => app.cohortId === cohortMembers.cohortId ); @@ -754,14 +776,15 @@ export class PostgresCohortMembersService { }); } - // Now update the user document in Elasticsearch with the merged applications array + // Now update the user document in Elasticsearch with comprehensive data const baseDoc = typeof userDoc?._source === 'object' ? userDoc._source : {}; await this.userElasticsearchService.updateUser( cohortMembers.userId, - { doc: { ...baseDoc, applications } }, + { doc: userDocument }, // Use comprehensive user document async (userId: string) => { - return await this.formSubmissionService.buildUserDocumentForElasticsearch( + // Use comprehensive sync to build the full user document for Elasticsearch + return await this.elasticsearchDataFetcherService.comprehensiveUserSync( userId ); } @@ -775,6 +798,15 @@ export class PostgresCohortMembersService { ); } } + + // Sync to Elasticsearch using centralized service + if (isElasticsearchEnabled()) { + await this.elasticsearchSyncService.syncUserToElasticsearch( + cohortMembers.userId, + { section: SyncSection.APPLICATIONS } + ); + } + return APIResponse.success( res, apiId, @@ -1013,7 +1045,7 @@ export class PostgresCohortMembersService { completionPercentageRanges: { min: number; max: number }[], formId: string ): { query: string; parameters: any[]; limit: number; offset: number } { - // Build completion percentage filter conditions with proper casting + // Build completion percentage filter conditions with proper numeric casting const completionConditions = completionPercentageRanges .map( (range) => @@ -1176,9 +1208,9 @@ export class PostgresCohortMembersService { : undefined; if (!existingApplication) { - // If application is missing, build and upsert the full user document (with progress pages) + // If application is missing, use comprehensive sync to build and upsert the full user document const fullUserDoc = - await this.formSubmissionService.buildUserDocumentForElasticsearch( + await this.elasticsearchDataFetcherService.comprehensiveUserSync( cohortMembershipToUpdate.userId ); if (fullUserDoc) { @@ -1186,7 +1218,7 @@ export class PostgresCohortMembersService { cohortMembershipToUpdate.userId, { doc: fullUserDoc }, async (userId: string) => { - return await this.formSubmissionService.buildUserDocumentForElasticsearch( + return await this.elasticsearchDataFetcherService.comprehensiveUserSync( userId ); } @@ -1217,6 +1249,14 @@ export class PostgresCohortMembersService { } } + // Sync to Elasticsearch using centralized service + if (isElasticsearchEnabled()) { + await this.elasticsearchSyncService.syncUserToElasticsearch( + cohortMembershipToUpdate.userId, + { section: SyncSection.APPLICATIONS } + ); + } + // Send notification if applicable for this status only let notifyStatuses: string[] = []; const { status, statusReason } = cohortMembersUpdateDto; diff --git a/src/adapters/postgres/user-adapter.ts b/src/adapters/postgres/user-adapter.ts index 2eeb7225..c04d4c30 100644 --- a/src/adapters/postgres/user-adapter.ts +++ b/src/adapters/postgres/user-adapter.ts @@ -48,8 +48,10 @@ import config from '../../common/config'; import { CalendarField } from 'src/fields/fieldValidators/fieldTypeClasses'; import { UserCreateSsoDto } from 'src/user/dto/user-create-sso.dto'; import { UserElasticsearchService } from '../../elasticsearch/user-elasticsearch.service'; +import { ElasticsearchDataFetcherService } from '../../elasticsearch/elasticsearch-data-fetcher.service'; import { IUser } from '../../elasticsearch/interfaces/user.interface'; import { isElasticsearchEnabled } from 'src/common/utils/elasticsearch.util'; +import { ElasticsearchSyncService, SyncSection } from '../../elasticsearch/elasticsearch-sync.service'; interface UpdateField { userId: string; // Required @@ -94,7 +96,9 @@ export class PostgresUserService implements IServicelocator { private postgresAcademicYearService: PostgresAcademicYearService, private readonly cohortAcademicYearService: CohortAcademicYearService, private readonly authUtils: AuthUtils, - private readonly userElasticsearchService: UserElasticsearchService + private readonly userElasticsearchService: UserElasticsearchService, + private readonly elasticsearchDataFetcherService: ElasticsearchDataFetcherService, + private readonly elasticsearchSyncService: ElasticsearchSyncService, ) { this.jwt_secret = this.configService.get('RBAC_JWT_SECRET'); this.jwt_password_reset_expires_In = this.configService.get( @@ -1042,8 +1046,27 @@ export class PostgresUserService implements IServicelocator { const updatedUser = await this.updateBasicUserDetails(userId, userDto); - // Sync to Elasticsearch - await this.syncUserToElasticsearch(updatedUser); + // Sync to Elasticsearch using centralized service + if (isElasticsearchEnabled()) { + // Check if user exists in Elasticsearch + const existingUser = await this.userElasticsearchService.getUser(userId); + + if (!existingUser) { + // User doesn't exist in Elasticsearch, fetch all data + LoggerUtil.log(`User ${userId} not found in Elasticsearch, fetching all data from database`, apiId); + await this.elasticsearchSyncService.syncUserToElasticsearch( + updatedUser.userId, + { section: SyncSection.ALL } + ); + } else { + // User exists, only update profile section + LoggerUtil.log(`User ${userId} exists in Elasticsearch, updating profile section only`, apiId); + await this.elasticsearchSyncService.syncUserToElasticsearch( + updatedUser.userId, + { section: SyncSection.PROFILE } + ); + } + } return await APIResponse.success( response, @@ -1573,7 +1596,7 @@ export class PostgresUserService implements IServicelocator { customFields: elasticCustomFields, }, applications: [], - courses: [], + // Removed root-level courses field as requested createdAt: result.createdAt ? result.createdAt.toISOString() : new Date().toISOString(), @@ -2783,90 +2806,16 @@ export class PostgresUserService implements IServicelocator { } /** - * Sync user profile to Elasticsearch. - * This will upsert (update or create) the user document in Elasticsearch. - * If the document is missing, it will fetch the user from the database and create it. + * Sync user profile to Elasticsearch using centralized service. + * This method is now deprecated in favor of the centralized service. + * @deprecated Use elasticsearchSyncService.syncUserToElasticsearch instead */ private async syncUserToElasticsearch(user: User) { try { - const customFields = await this.getFilteredCustomFields(user.userId); - - let formattedDob: string | null = null; - if (user.dob instanceof Date) { - formattedDob = user.dob.toISOString(); - } else if (typeof user.dob === 'string') { - formattedDob = user.dob; - } - // Prepare the profile data - const profile = { - userId: user.userId, - username: user.username, - firstName: user.firstName, - lastName: user.lastName, - middleName: user.middleName || '', - email: user.email || '', - mobile: user.mobile?.toString() || '', - mobile_country_code: user.mobile_country_code || '', - dob: formattedDob, - country: user.country, - gender: user.gender, - address: user.address || '', - district: user.district || '', - state: user.state || '', - pincode: user.pincode || '', - status: user.status, - customFields, // Now filtered to exclude form schema fields - }; - - // Upsert (update or create) the user profile in Elasticsearch if (isElasticsearchEnabled()) { - await this.userElasticsearchService.updateUserProfile( - user.userId, - profile, - async (userId: string) => { - // Fetch the latest user from the database for upsert - const dbUser = await this.usersRepository.findOne({ - where: { userId }, - }); - if (!dbUser) return null; - const customFields = await this.getFilteredCustomFields(userId); - let formattedDob: string | null = null; - if (dbUser.dob instanceof Date) { - formattedDob = dbUser.dob.toISOString(); - } else if (typeof dbUser.dob === 'string') { - formattedDob = dbUser.dob; - } - return { - userId: dbUser.userId, - profile: { - userId: dbUser.userId, - username: dbUser.username, - firstName: dbUser.firstName, - lastName: dbUser.lastName, - middleName: dbUser.middleName || '', - email: dbUser.email || '', - mobile: dbUser.mobile?.toString() || '', - mobile_country_code: dbUser.mobile_country_code || '', - dob: formattedDob, - gender: dbUser.gender, - country: dbUser.country || '', - address: dbUser.address || '', - district: dbUser.district || '', - state: dbUser.state || '', - pincode: dbUser.pincode || '', - status: dbUser.status, - customFields, - }, - applications: [], - courses: [], - createdAt: dbUser.createdAt - ? dbUser.createdAt.toISOString() - : new Date().toISOString(), - updatedAt: dbUser.updatedAt - ? dbUser.updatedAt.toISOString() - : new Date().toISOString(), - }; - } + await this.elasticsearchSyncService.syncUserToElasticsearch( + user.userId, + { section: SyncSection.ALL } ); } } catch (error) { diff --git a/src/app.controller.ts b/src/app.controller.ts index 3b70c990..7a9cd4cc 100644 --- a/src/app.controller.ts +++ b/src/app.controller.ts @@ -25,6 +25,15 @@ export class AppController { return this.appService.getHello(); } + @Get("health") + getHealth(): object { + return { + status: "healthy", + timestamp: new Date().toISOString(), + service: "user-microservice", + }; + } + @Get("files/:fileName") seeUploadedFile(@Param("fileName") fileName: string, @Res() res) { return res.sendFile(fileName, { root: "./uploads" }); diff --git a/src/app.module.ts b/src/app.module.ts index 40f176aa..424050c3 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -21,6 +21,7 @@ import { RbacModule } from './rbac/rbac.module'; import { AssignTenantModule } from './userTenantMapping/user-tenant-mapping.module'; import { FormsModule } from './forms/forms.module'; import { HttpService } from '@utils/http-service'; +import { LMSService } from './common/services/lms.service'; import { TenantModule } from './tenant/tenant.module'; import { AcademicyearsModule } from './academicyears/academicyears.module'; import { CohortAcademicYearModule } from './cohortAcademicYear/cohortAcademicYear.module'; @@ -84,6 +85,7 @@ import { BulkImportModule } from './bulk-import/bulk-import.module'; providers: [ AppService, HttpService, + LMSService, { provide: 'STORAGE_CONFIG', useValue: storageConfig, diff --git a/src/bulk-import/services/bulk-import.service.ts b/src/bulk-import/services/bulk-import.service.ts index fe06e376..48047bef 100644 --- a/src/bulk-import/services/bulk-import.service.ts +++ b/src/bulk-import/services/bulk-import.service.ts @@ -811,6 +811,7 @@ export class BulkImportService { formId: formSubmission.formId, itemId: userId, status: formSubmission.status || 'active', + completionPercentage: 100, // Set completion percentage to 100 for bulk import createdBy: adminId, updatedBy: adminId, }); diff --git a/src/common/services/lms.service.ts b/src/common/services/lms.service.ts new file mode 100644 index 00000000..0c9bdbea --- /dev/null +++ b/src/common/services/lms.service.ts @@ -0,0 +1,223 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { HttpService } from '../utils/http-service'; +import { ConfigService } from '@nestjs/config'; + +export interface LMSCourse { + courseId: string; + courseTitle: string; + progress: number; + units: { + type: 'nested'; + values: LMSUnit[]; + }; +} + +export interface LMSUnit { + unitId: string; + unitTitle: string; + progress: number; + contents: { + type: 'nested'; + values: LMSContent[]; + }; +} + +export interface LMSContent { + contentId: string; + type: string; + title: string; + status: string; + tracking: { + percentComplete?: number; + lastPosition?: number; + currentPosition?: number; + timeSpent?: number; + visitedPages?: number[]; + totalPages?: number; + lastPage?: number; + currentPage?: number; + questionsAttempted?: number; + totalQuestions?: number; + score?: number; + answers?: { + type: 'nested'; + values: { + questionId: string; + type: string; + submittedAnswer: string | string[]; + }[]; + }; + }; +} + +@Injectable() +export class LMSService { + private readonly logger = new Logger(LMSService.name); + private readonly lmsBaseUrl: string; + private readonly assessmentBaseUrl: string; + + constructor( + private readonly httpService: HttpService, + private readonly configService: ConfigService, + ) { + // Get service URLs from config + this.lmsBaseUrl = this.configService.get('LMS_SERVICE_URL', 'http://localhost:4002'); + this.assessmentBaseUrl = this.configService.get('ASSESSMENT_SERVICE_URL', 'http://localhost:3002'); + } + + /** + * Fetch initial course structure from LMS service + * Only used for initializing course structure, not for real-time updates + * + * @param userId - User ID to fetch courses for + * @param cohortId - Cohort ID to filter courses by + * @returns Promise - Array of basic course structure + */ + async getInitialCourseStructure(userId: string, cohortId?: string): Promise { + try { + + this.logger.debug(`Fetching initial course structure for user ${userId} from LMS service`); + + // For now, return empty structure since LMS API integration is not complete + // TODO: Replace with actual LMS API endpoint for course enrollment + // The actual endpoint might be something like: + // GET /api/v1/courses/structure?userId={userId}&cohortId={cohortId} + + this.logger.warn(`LMS API integration not yet implemented, returning empty course structure`); + return []; + + } catch (error) { + this.logger.error(`Failed to fetch initial course structure for user ${userId}:`, error); + return []; + } + } + + /** + * Transform LMS course data to Elasticsearch format + * + * @param courses - Raw course data from LMS service + * @returns LMSCourse[] - Transformed courses + */ + private transformCoursesToElasticsearchFormat(courses: any[]): LMSCourse[] { + return courses.map(course => ({ + courseId: course.id || course.courseId, + courseTitle: course.title || course.name || 'Untitled Course', + progress: course.progress || 0, + units: { + type: 'nested', + values: this.transformUnits(course.units || []), + }, + })); + } + + /** + * Transform units data to Elasticsearch format + * + * @param units - Raw units data from LMS service + * @returns LMSUnit[] - Transformed units + */ + private transformUnits(units: any[]): LMSUnit[] { + return units.map(unit => ({ + unitId: unit.id || unit.unitId, + unitTitle: unit.title || unit.name || 'Untitled Unit', + progress: unit.progress || 0, + contents: { + type: 'nested', + values: this.transformContents(unit.contents || []), + }, + })); + } + + /** + * Transform contents data to Elasticsearch format + * + * @param contents - Raw contents data from LMS service + * @returns LMSContent[] - Transformed contents + */ + private transformContents(contents: any[]): LMSContent[] { + return contents.map(content => ({ + contentId: content.id || content.contentId, + type: content.type || 'unknown', + title: content.title || content.name || 'Untitled Content', + status: content.status || 'not_started', + tracking: this.transformTracking(content.tracking || {}), + })); + } + + /** + * Transform tracking data to Elasticsearch format + * + * @param tracking - Raw tracking data from LMS service + * @returns Tracking object - Transformed tracking data + */ + private transformTracking(tracking: any): LMSContent['tracking'] { + return { + percentComplete: tracking.percentComplete || 0, + lastPosition: tracking.lastPosition || 0, + currentPosition: tracking.currentPosition || 0, + timeSpent: tracking.timeSpent || 0, + visitedPages: tracking.visitedPages || [], + totalPages: tracking.totalPages || 0, + lastPage: tracking.lastPage || 0, + currentPage: tracking.currentPage || 0, + questionsAttempted: tracking.questionsAttempted || 0, + totalQuestions: tracking.totalQuestions || 0, + score: tracking.score || 0, + answers: tracking.answers ? { + type: 'nested', + values: tracking.answers.map((answer: any) => ({ + questionId: answer.questionId || '', + type: answer.type || 'text', + submittedAnswer: answer.submittedAnswer || '', + })), + } : { + type: 'nested', + values: [], + }, + }; + } + + /** + * Health check for LMS service connectivity + * + * @returns Promise - true if LMS service is reachable + */ + async healthCheckLMS(): Promise { + try { + const response = await this.httpService.get(`${this.lmsBaseUrl}/health`); + return response.status === 200; + } catch (error) { + this.logger.warn('LMS service health check failed:', error); + return false; + } + } + + /** + * Health check for Assessment service connectivity + * + * @returns Promise - true if Assessment service is reachable + */ + async healthCheckAssessment(): Promise { + try { + const response = await this.httpService.get(`${this.assessmentBaseUrl}/health`); + return response.status === 200; + } catch (error) { + this.logger.warn('Assessment service health check failed:', error); + return false; + } + } + + /** + * Health check for both services + * + * @returns Promise<{lms: boolean, assessment: boolean}> - Health status of both services + */ + async healthCheck(): Promise<{lms: boolean, assessment: boolean}> { + const [lms, assessment] = await Promise.all([ + this.healthCheckLMS(), + this.healthCheckAssessment() + ]); + + return { lms, assessment }; + } +} \ No newline at end of file diff --git a/src/elasticsearch/controllers/course-webhook.controller.ts b/src/elasticsearch/controllers/course-webhook.controller.ts new file mode 100644 index 00000000..15517396 --- /dev/null +++ b/src/elasticsearch/controllers/course-webhook.controller.ts @@ -0,0 +1,305 @@ +import { + Controller, + Post, + Body, + HttpCode, + HttpStatus, + Logger, + Patch, + Param +} from '@nestjs/common'; +import { + ApiTags, + ApiOperation, + ApiResponse, + ApiBody +} from '@nestjs/swagger'; +import { CourseElasticsearchService, CourseProgressUpdate, QuizAnswerUpdate } from '../services/course-elasticsearch.service'; + +class CourseProgressDto { + userId: string; + cohortId: string; + courseId: string; + courseTitle: string; + progress: number; + unitId?: string; + unitTitle?: string; + unitProgress?: number; + contentId?: string; + contentType?: string; + contentTitle?: string; + contentStatus?: string; + tracking?: { + percentComplete?: number; + lastPosition?: number; + currentPosition?: number; + timeSpent?: number; + visitedPages?: number[]; + totalPages?: number; + lastPage?: number; + currentPage?: number; + }; +} + +class QuizAnswerDto { + userId: string; + cohortId: string; + courseId: string; + unitId: string; + contentId: string; + attemptId: string; + answers: Array<{ + questionId: string; + type: string; + submittedAnswer: string | string[]; + }>; + score?: number; + questionsAttempted: number; + totalQuestions: number; + percentComplete: number; + timeSpent: number; +} + +class CourseInitializationDto { + userId: string; + cohortId: string; + courseData: any[]; +} + +@ApiTags('Course Elasticsearch Webhooks') +@Controller('elasticsearch/courses') +export class CourseWebhookController { + private readonly logger = new Logger(CourseWebhookController.name); + + constructor( + private readonly courseElasticsearchService: CourseElasticsearchService, + ) {} + + /** + * Webhook endpoint for lesson tracking updates from LMS SERVICE + * Called by shiksha-lms-service when user progress is updated + * Corresponds to: PATCH /attempts/progress/:attemptId in shiksha-lms-service + */ + @Patch('lms/progress') + @HttpCode(HttpStatus.OK) + @ApiOperation({ + summary: 'Update course progress from LMS service', + description: 'Webhook endpoint called by shiksha-lms-service when lesson tracking is updated' + }) + @ApiBody({ type: CourseProgressDto }) + @ApiResponse({ + status: 200, + description: 'Course progress updated successfully in Elasticsearch' + }) + @ApiResponse({ + status: 400, + description: 'Invalid request data' + }) + @ApiResponse({ + status: 500, + description: 'Internal server error' + }) + async updateLMSCourseProgress(@Body() progressData: CourseProgressDto): Promise<{ + success: boolean; + message: string; + data?: any; + }> { + try { + this.logger.log(`[LMS SERVICE] Received course progress update for user ${progressData.userId}, course ${progressData.courseId}`); + + // Validate required fields + if (!progressData.userId || !progressData.cohortId || !progressData.courseId) { + throw new Error('Missing required fields: userId, cohortId, courseId'); + } + + // Update course progress in Elasticsearch + await this.courseElasticsearchService.updateCourseProgress(progressData); + + return { + success: true, + message: 'LMS course progress updated successfully', + data: { + userId: progressData.userId, + courseId: progressData.courseId, + progress: progressData.progress, + source: 'lms-service' + } + }; + + } catch (error) { + this.logger.error(`[LMS SERVICE] Failed to update course progress:`, error); + throw error; + } + } + + /** + * Webhook endpoint for quiz answer submissions from ASSESSMENT SERVICE + * Called by shiksha-assessment-service when quiz answers are submitted + * Corresponds to: POST /:attemptId/answers in shiksha-assessment-service + */ + @Post('assessment/quiz-answers') + @HttpCode(HttpStatus.OK) + @ApiOperation({ + summary: 'Update quiz answers from Assessment service', + description: 'Webhook endpoint called by shiksha-assessment-service when quiz answers are submitted' + }) + @ApiBody({ type: QuizAnswerDto }) + @ApiResponse({ + status: 200, + description: 'Quiz answers updated successfully in Elasticsearch' + }) + @ApiResponse({ + status: 400, + description: 'Invalid request data' + }) + @ApiResponse({ + status: 500, + description: 'Internal server error' + }) + async updateAssessmentQuizAnswers(@Body() quizData: QuizAnswerDto): Promise<{ + success: boolean; + message: string; + data?: any; + }> { + try { + this.logger.log(`[ASSESSMENT SERVICE] Received quiz answer update for user ${quizData.userId}, assessment ${quizData.contentId}`); + + // Validate required fields + if (!quizData.userId || !quizData.cohortId || !quizData.courseId || !quizData.contentId) { + throw new Error('Missing required fields: userId, cohortId, courseId, contentId'); + } + + // Update quiz answers in Elasticsearch + await this.courseElasticsearchService.updateQuizAnswers(quizData); + + return { + success: true, + message: 'Assessment quiz answers updated successfully', + data: { + userId: quizData.userId, + contentId: quizData.contentId, + score: quizData.score, + percentComplete: quizData.percentComplete, + source: 'assessment-service' + } + }; + + } catch (error) { + this.logger.error(`[ASSESSMENT SERVICE] Failed to update quiz answers:`, error); + throw error; + } + } + + /** + * Initialize course structure for a user + * Called when a user is enrolled in courses + */ + @Post('initialize') + @HttpCode(HttpStatus.OK) + @ApiOperation({ + summary: 'Initialize course structure in Elasticsearch', + description: 'Initialize the course structure when a user is enrolled in courses' + }) + @ApiBody({ type: CourseInitializationDto }) + @ApiResponse({ + status: 200, + description: 'Course structure initialized successfully' + }) + @ApiResponse({ + status: 400, + description: 'Invalid request data' + }) + @ApiResponse({ + status: 500, + description: 'Internal server error' + }) + async initializeCourseStructure(@Body() initData: CourseInitializationDto): Promise<{ + success: boolean; + message: string; + data?: any; + }> { + try { + this.logger.log(`Initializing course structure for user ${initData.userId}, cohort ${initData.cohortId}`); + + // Validate required fields + if (!initData.userId || !initData.cohortId || !initData.courseData) { + throw new Error('Missing required fields: userId, cohortId, courseData'); + } + + // Initialize course structure in Elasticsearch + await this.courseElasticsearchService.initializeCourseStructure( + initData.userId, + initData.cohortId, + initData.courseData + ); + + return { + success: true, + message: 'Course structure initialized successfully', + data: { + userId: initData.userId, + cohortId: initData.cohortId, + coursesCount: initData.courseData.length + } + }; + + } catch (error) { + this.logger.error(`Failed to initialize course structure:`, error); + throw error; + } + } + + /** + * Bulk update course progress for multiple users + * Useful for batch operations + */ + @Post('bulk-progress') + @HttpCode(HttpStatus.OK) + @ApiOperation({ + summary: 'Bulk update course progress', + description: 'Update course progress for multiple users in batch' + }) + @ApiResponse({ + status: 200, + description: 'Bulk progress update completed' + }) + async bulkUpdateProgress(@Body() bulkData: { + updates: CourseProgressDto[]; + }): Promise<{ + success: boolean; + message: string; + data?: any; + }> { + try { + this.logger.log(`Received bulk progress update for ${bulkData.updates.length} items`); + + const results = { + successful: 0, + failed: 0, + errors: [] as string[] + }; + + // Process each update + for (const update of bulkData.updates) { + try { + await this.courseElasticsearchService.updateCourseProgress(update); + results.successful++; + } catch (error) { + results.failed++; + results.errors.push(`User ${update.userId}: ${error.message}`); + this.logger.error(`Failed to update progress for user ${update.userId}:`, error); + } + } + + return { + success: true, + message: `Bulk update completed: ${results.successful} successful, ${results.failed} failed`, + data: results + }; + + } catch (error) { + this.logger.error(`Failed to process bulk progress update:`, error); + throw error; + } + } +} \ No newline at end of file diff --git a/src/elasticsearch/controllers/elasticsearch.controller.ts b/src/elasticsearch/controllers/elasticsearch.controller.ts new file mode 100644 index 00000000..975f8a99 --- /dev/null +++ b/src/elasticsearch/controllers/elasticsearch.controller.ts @@ -0,0 +1,682 @@ +import { Controller, Post, Body, Get, Param, Put, Delete, HttpCode, HttpStatus } from '@nestjs/common'; +import { UserElasticsearchService } from '../user-elasticsearch.service'; +import { ElasticsearchDataFetcherService } from '../elasticsearch-data-fetcher.service'; +import { Logger } from '@nestjs/common'; +import { ApiOperation, ApiResponse } from '@nestjs/swagger'; + +@Controller('elasticsearch/users') +export class ElasticsearchController { + private readonly logger = new Logger(ElasticsearchController.name); + + constructor( + private readonly userElasticsearchService: UserElasticsearchService, + private readonly dataFetcherService: ElasticsearchDataFetcherService, + ) {} + + @Post('search') + async searchUsers(@Body() body: any) { + return this.userElasticsearchService.searchUsers(body); + } + + @Get(':userId') + async getUser(@Param('userId') userId: string) { + return this.userElasticsearchService.getUser(userId); + } + + @Get(':userId/profile-applications') + @HttpCode(HttpStatus.OK) + @ApiOperation({ + summary: 'Fetch user profile and applications data', + description: 'Fetches complete user profile and applications data from database without syncing to Elasticsearch. Used by external services like LMS.' + }) + @ApiResponse({ + status: 200, + description: 'User profile and applications fetched successfully' + }) + @ApiResponse({ + status: 404, + description: 'User not found' + }) + async fetchUserProfileAndApplications(@Param('userId') userId: string) { + try { + // Use comprehensive sync to fetch all data but don't save to Elasticsearch + const userData = await this.dataFetcherService.comprehensiveUserSync(userId); + + if (!userData) { + return { + status: 'error', + message: `User ${userId} not found in database`, + }; + } + + this.logger.log(`Successfully fetched profile and applications for userId: ${userId}`); + + return { + status: 'success', + message: `User ${userId} profile and applications fetched successfully`, + data: userData, + }; + } catch (error) { + this.logger.error(`Failed to fetch profile and applications for userId: ${userId}:`, error); + return { + status: 'error', + message: `Failed to fetch user data: ${error.message}`, + }; + } + } + + @Post(':userId/sync') + @HttpCode(HttpStatus.OK) + @ApiOperation({ + summary: 'Sync user data from database to Elasticsearch', + description: 'Fetches complete user data from database and syncs to Elasticsearch' + }) + @ApiResponse({ + status: 200, + description: 'User synced successfully' + }) + @ApiResponse({ + status: 404, + description: 'User not found' + }) + async syncUserFromDatabase(@Param('userId') userId: string, @Body() webhookData?: any) { + try { + // Use comprehensive sync to ensure all data is fetched and synced together + const userData = await this.dataFetcherService.comprehensiveUserSync(userId); + + if (!userData) { + return { + status: 'error', + message: `User ${userId} not found in database`, + }; + } + + // Clean up duplicate lessonTrackId entries before processing webhook data + this.cleanupDuplicateLessonTrackIds(userData); + + // If webhook data contains assessment data, enhance the user data + if (webhookData && webhookData.assessmentData) { + this.logger.log(`Enhancing user data with assessment data for userId: ${userId}`); + + // Extract testId as contentId for assessment data + const contentId = webhookData.assessmentData.testId; + const testId = webhookData.assessmentData.testId; + + // Find or create application for this test + let application = userData.applications?.find((app: any) => + app.courses?.values?.some((course: any) => + course.units?.values?.some((unit: any) => + unit.contents?.values?.some((content: any) => + content.contentId === contentId || content.contentId === testId + ) + ) + ) + ); + + if (!application) { + // Create new application if it doesn't exist + application = { + cohortId: testId, // Using testId as cohortId for assessment + formId: '', + submissionId: '', + cohortmemberstatus: 'enrolled', + formstatus: 'active', + completionPercentage: 0, + progress: { + pages: {}, + overall: { + total: 0, + completed: 0 + } + }, + lastSavedAt: null, + submittedAt: null, + cohortDetails: { + cohortId: testId, + name: `Assessment ${testId}`, + type: 'ASSESSMENT', + status: 'active', + }, + courses: { + type: 'nested', + values: [] + } + }; + + if (!userData.applications) { + userData.applications = []; + } + userData.applications.push(application); + } + + // Find or create course structure for assessment + let course = application.courses.values.find((c: any) => c.courseId === testId); + if (!course) { + course = { + courseId: testId, + courseTitle: `Assessment ${testId}`, + progress: 0, + units: { + type: 'nested', + values: [] + } + }; + application.courses.values.push(course); + } + + // Find or create unit for assessment + let unit = course.units.values.find((u: any) => u.unitId === testId); + if (!unit) { + unit = { + unitId: testId, + unitTitle: `Assessment Unit ${testId}`, + progress: 0, + contents: { + type: 'nested', + values: [] + } + }; + course.units.values.push(unit); + } + + // Find or create content for assessment + let content = unit.contents.values.find((c: any) => c.contentId === contentId || c.contentId === testId); + if (!content) { + content = { + contentId: contentId || testId, + type: 'test', + title: `Assessment ${testId}`, + status: 'incomplete', + tracking: { + timeSpent: 0, + currentPosition: 0, + lastPosition: 0, + percentComplete: 0, + questionsAttempted: 0, + totalQuestions: 0, + score: 0, + answers: { + type: 'nested', + values: [] + } + } + }; + unit.contents.values.push(content); + } + + // Update content with assessment data + content.tracking = { + ...content.tracking, + questionsAttempted: webhookData.assessmentData.questionsAttempted || 0, + totalQuestions: webhookData.assessmentData.totalQuestions || 0, + score: webhookData.assessmentData.score || 0, + percentComplete: webhookData.assessmentData.percentComplete || 0, + timeSpent: webhookData.assessmentData.timeSpent || 0, + answers: { + type: 'nested', + values: webhookData.assessmentData.answers || [] + } + }; + + // Update content status based on completion + if (webhookData.assessmentData.percentComplete >= 100) { + content.status = 'completed'; + } else if (webhookData.assessmentData.percentComplete > 0) { + content.status = 'in_progress'; + } + + this.logger.log(`Updated assessment content ${contentId} with ${webhookData.assessmentData.answers?.length || 0} answers`); + } + + // If webhook data contains course hierarchy, enhance the user data + if (webhookData && webhookData.courseHierarchy) { + this.logger.log(`Enhancing user data with course hierarchy for userId: ${userId}`); + + // Extract cohortId from course params if available + const cohortId = webhookData.courseHierarchy.params?.cohortId || + webhookData.courseId || + webhookData.courseHierarchy.courseId; + + // Find or create application for this cohort + let application = userData.applications?.find((app: any) => app.cohortId === cohortId); + + // If no application found with this cohortId, check if any existing application has this course + if (!application) { + application = userData.applications?.find((app: any) => + app.courses?.values?.some((course: any) => course.courseId === webhookData.courseHierarchy.courseId) + ); + + if (application) { + this.logger.log(`Found existing application with course ${webhookData.courseHierarchy.courseId}, will merge course data`); + } + } + + if (!application) { + // Create new application if it doesn't exist + application = { + cohortId: cohortId, + formId: '', + submissionId: '', + cohortmemberstatus: 'enrolled', + formstatus: 'active', + completionPercentage: 0, + progress: { + pages: {}, + overall: { + total: 0, + completed: 0 + } + }, + lastSavedAt: null, + submittedAt: null, + cohortDetails: { + cohortId: cohortId, + name: webhookData.courseHierarchy.title || webhookData.courseHierarchy.name || 'Unknown Cohort', + type: 'COHORT', + status: 'active', + }, + courses: { + type: 'nested', + values: [] + } + }; + + if (!userData.applications) { + userData.applications = []; + } + userData.applications.push(application); + } + + // Initialize course structure with hierarchy data + if (!application.courses) { + application.courses = { + type: 'nested', + values: [] + }; + } + + // Build course data from hierarchy + const courseData = { + courseId: webhookData.courseHierarchy.courseId, + courseTitle: webhookData.courseHierarchy.title || webhookData.courseHierarchy.name, + progress: 0, + units: { + type: 'nested' as const, + values: webhookData.courseHierarchy.modules?.map((module: any) => ({ + unitId: module.moduleId, + unitTitle: module.title, + progress: 0, + contents: { + type: 'nested' as const, + values: module.lessons?.map((lesson: any) => ({ + contentId: lesson.lessonId, + lessonId: lesson.lessonId, // Add lessonId for proper mapping + type: lesson.format || 'video', + title: lesson.title, + status: 'incomplete', + tracking: { + timeSpent: 0, + currentPosition: 0, + lastPosition: 0, + percentComplete: 0 + }, + // Add lesson tracking information if available + ...(webhookData.lessonTrackingInfo && { + expectedLessonTrackId: `${lesson.lessonId}-${webhookData.userId}-1`, + lessonTrackingInfo: { + courseId: webhookData.courseId, + userId: webhookData.userId, + tenantId: webhookData.lessonTrackingInfo.tenantId, + organisationId: webhookData.lessonTrackingInfo.organisationId + } + }) + })) || [] + } + })) || [] + } + }; + + // Find or update course in application (preserve existing structure) + let existingCourse = application.courses.values.find((c: any) => c.courseId === courseData.courseId); + + if (existingCourse) { + // Update existing course structure by merging with new hierarchy + this.logger.log(`Updating existing course structure for courseId: ${courseData.courseId}`); + + // Update course title if different + if (courseData.courseTitle && existingCourse.courseTitle !== courseData.courseTitle) { + existingCourse.courseTitle = courseData.courseTitle; + } + + // Merge units from hierarchy with existing units + for (const newUnit of courseData.units.values) { + let existingUnit = existingCourse.units.values.find((u: any) => u.unitId === newUnit.unitId); + + if (existingUnit) { + // Update existing unit + existingUnit.unitTitle = newUnit.unitTitle; + + // Merge contents from hierarchy with existing contents + for (const newContent of newUnit.contents.values) { + let existingContent = existingUnit.contents.values.find((c: any) => c.contentId === newContent.contentId); + + if (existingContent) { + // Update existing content with new data but preserve tracking + existingContent.title = newContent.title; + existingContent.type = newContent.type; + // Don't overwrite existing tracking data + } else { + // Add new content to existing unit + existingUnit.contents.values.push(newContent); + } + } + } else { + // Add new unit to existing course + existingCourse.units.values.push(newUnit); + } + } + } else { + // Add new course to application + application.courses.values.push(courseData); + } + + // If lesson attempt data is provided, update the specific lesson in the existing structure + if (webhookData.lessonAttemptData && webhookData.lessonAttemptData.lessonId) { + this.logger.log(`Updating lesson attempt data for lessonId: ${webhookData.lessonAttemptData.lessonId}`); + + // Find the course and update the specific lesson + const targetCourse = application.courses.values.find((c: any) => c.courseId === courseData.courseId); + if (targetCourse) { + let lessonUpdated = false; + + for (const unit of targetCourse.units.values) { + for (const content of unit.contents.values) { + // Only update if contentId matches lessonId AND this content doesn't already have a different lessonTrackId + if (content.contentId === webhookData.lessonAttemptData.lessonId && + (!content.lessonTrackId || content.lessonTrackId === webhookData.lessonAttemptData.attemptId)) { + + // Update lesson tracking data + content.tracking = { + ...content.tracking, + timeSpent: webhookData.lessonAttemptData.timeSpent || 0, + currentPosition: webhookData.lessonAttemptData.currentPosition || 0, + lastPosition: webhookData.lessonAttemptData.currentPosition || 0, + percentComplete: webhookData.lessonAttemptData.completionPercentage || 0 + }; + + // Update lesson status based on completion + if (webhookData.lessonAttemptData.completionPercentage >= 100) { + content.status = 'complete'; + } else if (webhookData.lessonAttemptData.completionPercentage > 0) { + content.status = 'incomplete'; + } + + // Add lessonTrackId for tracking - only if not already set + if (!content.lessonTrackId) { + content.lessonTrackId = webhookData.lessonAttemptData.attemptId; + } + content.lessonId = webhookData.lessonAttemptData.lessonId; + + this.logger.log(`Updated lesson ${webhookData.lessonAttemptData.lessonId} with tracking data and lessonTrackId: ${webhookData.lessonAttemptData.attemptId}`); + lessonUpdated = true; + break; // Exit inner loop once we find and update the specific lesson + } + } + if (lessonUpdated) break; // Exit outer loop once lesson is updated + } + + // If no existing content was found with this lessonId, create a new one + if (!lessonUpdated) { + this.logger.log(`No existing content found for lessonId: ${webhookData.lessonAttemptData.lessonId}, creating new content`); + + // Find the first unit to add the content to (or create a default unit) + let targetUnit = targetCourse.units.values[0]; + if (!targetUnit) { + targetUnit = { + unitId: 'default-unit', + unitTitle: 'Default Unit', + progress: 0, + contents: { + type: 'nested', + values: [] + } + }; + targetCourse.units.values.push(targetUnit); + } + + // Check if we're trying to create a unit with the same ID as a content item + const contentId = webhookData.lessonAttemptData.lessonId; + const existingUnitWithContentId = targetCourse.units.values.find((unit: any) => unit.unitId === contentId); + + if (existingUnitWithContentId) { + this.logger.warn(`Found existing unit with same ID as content: ${contentId}, using that unit instead of creating new content`); + targetUnit = existingUnitWithContentId; + } + + // Create new content with lesson attempt data + const newContent = { + contentId: webhookData.lessonAttemptData.lessonId, + lessonId: webhookData.lessonAttemptData.lessonId, + lessonTrackId: webhookData.lessonAttemptData.attemptId, + type: 'video', // Default type + title: `Lesson ${webhookData.lessonAttemptData.lessonId}`, + status: webhookData.lessonAttemptData.completionPercentage >= 100 ? 'complete' : 'incomplete', + tracking: { + timeSpent: webhookData.lessonAttemptData.timeSpent || 0, + currentPosition: webhookData.lessonAttemptData.currentPosition || 0, + lastPosition: webhookData.lessonAttemptData.currentPosition || 0, + percentComplete: webhookData.lessonAttemptData.completionPercentage || 0 + } + }; + + targetUnit.contents.values.push(newContent); + this.logger.log(`Created new content for lessonId: ${webhookData.lessonAttemptData.lessonId} with lessonTrackId: ${webhookData.lessonAttemptData.attemptId}`); + } + } + } + } + + // Create or update user in Elasticsearch + await this.userElasticsearchService.createUser(userData); + + return { + status: 'success', + message: `User ${userId} synced successfully from database`, + data: { + userId: userData.userId, + profileFields: Object.keys(userData.profile || {}).length, + applicationsCount: userData.applications?.length || 0, + courseHierarchyIncluded: !!(webhookData && webhookData.courseHierarchy), + }, + }; + } catch (error) { + return { + status: 'error', + message: `Failed to sync user ${userId}: ${error.message}`, + }; + } + } + + @Put(':userId') + async updateUser(@Param('userId') userId: string, @Body() userData: any) { + return this.userElasticsearchService.updateUser(userId, userData); + } + + @Delete(':userId') + async deleteUser(@Param('userId') userId: string) { + return this.userElasticsearchService.deleteUser(userId); + } + + @Get(':userId/check-profile') + @HttpCode(HttpStatus.OK) + @ApiOperation({ + summary: 'Check and fix empty profile data', + description: 'Checks if user profile data is empty and fixes it by re-syncing from database' + }) + @ApiResponse({ + status: 200, + description: 'Profile check completed' + }) + async checkAndFixProfile(@Param('userId') userId: string) { + try { + await this.dataFetcherService.checkAndFixEmptyProfile(userId); + + return { + status: 'success', + message: `Profile check and fix completed for user ${userId}`, + }; + } catch (error) { + this.logger.error(`Failed to check and fix profile for userId: ${userId}:`, error); + return { + status: 'error', + message: `Failed to check and fix profile: ${error.message}`, + }; + } + } + + private cleanupDuplicateLessonTrackIds(userData: any) { + if (!userData || !userData.applications) { + return; + } + + this.logger.log(`Starting cleanup of duplicate lessonTrackIds for user ${userData.userId}`); + + userData.applications.forEach((application: any) => { + if (!application.courses || !application.courses.values) { + return; + } + + application.courses.values.forEach((course: any) => { + if (!course.units || !course.units.values) { + return; + } + + this.logger.log(`Processing course ${course.courseId} with ${course.units.values.length} units`); + + // Track unique lessons across ALL units in this course to prevent duplicates + const uniqueLessons = new Map(); + const lessonTrackIdMap = new Map(); + + // First pass: identify all unique lessons and their lessonTrackIds + course.units.values.forEach((unit: any) => { + if (!unit.contents || !unit.contents.values) { + return; + } + + this.logger.log(`Processing unit ${unit.unitId} with ${unit.contents.values.length} contents`); + + unit.contents.values.forEach((content: any) => { + const contentId = content.contentId || content.lessonId; + const lessonId = content.lessonId; + const lessonTrackId = content.lessonTrackId; + + if (!contentId) return; + + this.logger.log(`Processing content ${contentId} with lessonTrackId: ${lessonTrackId}`); + + // Create a unique key for this lesson + const uniqueKey = `${contentId}-${lessonId}`; + + // Track lessonTrackIds to prevent duplicates + if (lessonTrackId) { + if (lessonTrackIdMap.has(lessonTrackId)) { + // This lessonTrackId already exists, remove it from this content + this.logger.warn(`Removing duplicate lessonTrackId ${lessonTrackId} from content ${contentId}`); + delete content.lessonTrackId; + } else { + lessonTrackIdMap.set(lessonTrackId, uniqueKey); + this.logger.log(`Added lessonTrackId ${lessonTrackId} to map for content ${contentId}`); + } + } + + // Track unique lessons + if (uniqueLessons.has(uniqueKey)) { + const existingLesson = uniqueLessons.get(uniqueKey); + + this.logger.log(`Found duplicate lesson for key ${uniqueKey}: existing in unit ${existingLesson.unitId}, new in unit ${unit.unitId}`); + + // Prefer lesson with lessonTrackId + if (content.lessonTrackId && !existingLesson.lessonTrackId) { + uniqueLessons.set(uniqueKey, content); + // Mark existing lesson for removal + existingLesson._shouldRemove = true; + this.logger.log(`Preferring new lesson with lessonTrackId for key ${uniqueKey}`); + } else if (existingLesson.lessonTrackId && !content.lessonTrackId) { + // Keep existing lesson, mark this one for removal + content._shouldRemove = true; + this.logger.log(`Keeping existing lesson with lessonTrackId for key ${uniqueKey}`); + } else if (content.lessonTrackId && existingLesson.lessonTrackId) { + // Both have lessonTrackId, keep the one with higher progress + const existingProgress = existingLesson.tracking?.percentComplete || 0; + const newProgress = content.tracking?.percentComplete || 0; + + if (newProgress > existingProgress) { + uniqueLessons.set(uniqueKey, content); + existingLesson._shouldRemove = true; + this.logger.log(`Preferring new lesson with higher progress for key ${uniqueKey}`); + } else { + content._shouldRemove = true; + this.logger.log(`Keeping existing lesson with higher progress for key ${uniqueKey}`); + } + } else { + // Neither has lessonTrackId, keep the one with higher progress + const existingProgress = existingLesson.tracking?.percentComplete || 0; + const newProgress = content.tracking?.percentComplete || 0; + + if (newProgress > existingProgress) { + uniqueLessons.set(uniqueKey, content); + existingLesson._shouldRemove = true; + this.logger.log(`Preferring new lesson with higher progress for key ${uniqueKey}`); + } else { + content._shouldRemove = true; + this.logger.log(`Keeping existing lesson with higher progress for key ${uniqueKey}`); + } + } + } else { + uniqueLessons.set(uniqueKey, content); + this.logger.log(`Added new unique lesson for key ${uniqueKey}`); + } + }); + }); + + // Second pass: remove duplicate content from units + course.units.values.forEach((unit: any) => { + if (!unit.contents || !unit.contents.values) { + return; + } + + const originalCount = unit.contents.values.length; + + // Filter out content marked for removal + unit.contents.values = unit.contents.values.filter((content: any) => { + if (content._shouldRemove) { + this.logger.warn(`Removing duplicate content ${content.contentId} from unit ${unit.unitId}`); + return false; + } + // Clean up the temporary flag + delete content._shouldRemove; + return true; + }); + + const finalCount = unit.contents.values.length; + if (originalCount !== finalCount) { + this.logger.log(`Removed ${originalCount - finalCount} duplicate contents from unit ${unit.unitId}`); + } + }); + + // Remove empty units + const originalUnitCount = course.units.values.length; + course.units.values = course.units.values.filter((unit: any) => + unit.contents && unit.contents.values && unit.contents.values.length > 0 + ); + const finalUnitCount = course.units.values.length; + + if (originalUnitCount !== finalUnitCount) { + this.logger.log(`Removed ${originalUnitCount - finalUnitCount} empty units from course ${course.courseId}`); + } + }); + }); + + this.logger.log(`Completed cleanup of duplicate lessonTrackIds for user ${userData.userId}`); + } +} \ No newline at end of file diff --git a/src/elasticsearch/elasticsearch-data-fetcher.service.ts b/src/elasticsearch/elasticsearch-data-fetcher.service.ts new file mode 100644 index 00000000..8eeaedca --- /dev/null +++ b/src/elasticsearch/elasticsearch-data-fetcher.service.ts @@ -0,0 +1,1820 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository } from 'typeorm'; +import { User } from '../user/entities/user-entity'; +import { CohortMembers } from '../cohortMembers/entities/cohort-member.entity'; +import { FormSubmission } from '../forms/entities/form-submission.entity'; +import { FieldValues } from '../fields/entities/fields-values.entity'; +import { Cohort } from '../cohort/entities/cohort.entity'; +import { IUser, IProfile } from './interfaces/user.interface'; +import { isElasticsearchEnabled } from '../common/utils/elasticsearch.util'; +import { LoggerUtil } from '../common/logger/LoggerUtil'; +import { PostgresFieldsService } from '../adapters/postgres/fields-adapter'; +import { FormsService } from '../forms/forms.service'; +import { LMSService } from '../common/services/lms.service'; +import axios from 'axios'; +import { UserElasticsearchService } from './user-elasticsearch.service'; + +/** + * Centralized Elasticsearch Data Fetcher Service + * + * This service handles all data fetching operations from database to Elasticsearch. + * It provides reusable functions for fetching user profiles, applications, and cohort details + * that can be used across different adapters and services. + * + * Key Features: + * - Centralized data fetching logic to reduce code duplication + * - Handles user profile data with custom fields + * - Manages application data with form submissions and cohort details + * - Provides course data structure for future use + * - Implements proper error handling and logging + * - Supports dynamic field mapping and schema extraction + */ +@Injectable() +export class ElasticsearchDataFetcherService { + private readonly logger = new Logger(ElasticsearchDataFetcherService.name); + + constructor( + @InjectRepository(User) + private readonly userRepository: Repository, + @InjectRepository(CohortMembers) + private readonly cohortMembersRepository: Repository, + @InjectRepository(FormSubmission) + private readonly formSubmissionRepository: Repository, + @InjectRepository(FieldValues) + private readonly fieldValuesRepository: Repository, + @InjectRepository(Cohort) + private readonly cohortRepository: Repository, + private readonly fieldsService: PostgresFieldsService, + private readonly formsService: FormsService, + private readonly lmsService: LMSService, + private readonly userElasticsearchService: UserElasticsearchService, + ) {} + + /** + * Fetch complete user document for Elasticsearch from database + * This is the main function that fetches all required data for a user + * + * @param userId - The user ID to fetch data for + * @returns Promise - Complete user document or null if user not found + */ + async fetchUserDocumentForElasticsearch(userId: string): Promise { + try { + this.logger.log(`Fetching complete user document for userId: ${userId}`); + + // Fetch user from database first to get tenant/organisation info + const user = await this.userRepository.findOne({ where: { userId } }); + if (!user) { + this.logger.warn(`User not found in database: ${userId}`); + return null; + } + + // Fetch user profile data + const userProfile = await this.fetchUserProfile(user); + if (!userProfile) { + this.logger.warn(`User profile not found for userId: ${userId}`); + return null; + } + + // Fetch applications data + const applications = await this.fetchUserApplications(userId); + + // Fetch answer data for this user + // For now, use default tenant/organisation values since they're not in User entity + const answerData = await this.fetchUserAnswerData(userId, 'default-tenant', 'default-organisation'); + + // Enhance applications with answer data + if (applications && applications.length > 0 && answerData.length > 0) { + for (const application of applications) { + if (application.courses && application.courses.values) { + // Map answer data to courses + for (const course of application.courses.values) { + if (course.units && course.units.values) { + for (const unit of course.units.values) { + if (unit.contents && unit.contents.values) { + for (const content of unit.contents.values) { + if (content.type === 'test') { + // Find matching answer data for this test + const matchingAnswerData = answerData.find(answer => + answer.testId === content.contentId + ); + + if (matchingAnswerData) { + content.tracking = { + ...content.tracking, + questionsAttempted: matchingAnswerData.questionsAttempted, + totalQuestions: matchingAnswerData.totalQuestions, + score: matchingAnswerData.score, + percentComplete: matchingAnswerData.percentComplete, + timeSpent: matchingAnswerData.timeSpent, + answers: { + type: 'nested', + values: matchingAnswerData.answers + } + }; + + // Update content status based on completion + if (matchingAnswerData.percentComplete >= 100) { + content.status = 'completed'; + } else if (matchingAnswerData.percentComplete > 0) { + content.status = 'in_progress'; + } + } + } + } + } + } + } + } + } + } + } + + return { + userId: userProfile.userId, + profile: userProfile, + applications: applications, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString() + }; + } catch (error) { + this.logger.error(`Failed to fetch user document for userId: ${userId}:`, error); + throw error; + } + } + + /** + * Comprehensive sync method that ensures all user data is fetched and synced together + * This method handles all the issues: profile, applications, courses, and answers + * Now enhanced to fetch lesson, module, and question data from all three services + */ + async comprehensiveUserSync(userId: string): Promise { + try { + this.logger.log(`Starting comprehensive sync for userId: ${userId}`); + + // 1. Fetch user from database first to get tenant/organisation info + const user = await this.userRepository.findOne({ where: { userId } }); + if (!user) { + this.logger.warn(`User not found in database: ${userId}`); + return null; + } + + // 2. Fetch complete user profile data + const userProfile = await this.fetchUserProfile(user); + if (!userProfile) { + this.logger.warn(`User profile not found for userId: ${userId}`); + return null; + } + + // 3. Fetch complete applications data (with graceful error handling) + let applications = []; + try { + applications = await this.fetchUserApplications(userId); + this.logger.log(`Fetched ${applications.length} applications for userId: ${userId}`); + if (applications.length === 0) { + this.logger.log(`No applications found for userId: ${userId} - this is normal if user has no cohort memberships`); + } + } catch (error) { + this.logger.warn(`Failed to fetch applications for userId: ${userId}, continuing with empty applications:`, error.message); + applications = []; + } + + // 4. Get user's tenant and organisation data + let tenantId = 'default-tenant'; + let organisationId = 'default-organisation'; + + try { + const userTenantMapping = await this.cohortMembersRepository.manager + .getRepository('UserTenantMapping') + .findOne({ + where: { userId } + }); + + if (userTenantMapping) { + tenantId = userTenantMapping.tenantId || 'default-tenant'; + organisationId = userTenantMapping.organisationId || 'default-organisation'; + this.logger.log(`Found tenant data for userId: ${userId}, tenantId: ${tenantId}, organisationId: ${organisationId}`); + } else { + this.logger.warn(`No tenant mapping found for userId: ${userId}, using default values`); + } + } catch (error) { + this.logger.warn(`Failed to fetch tenant data for userId: ${userId}, using default values:`, error.message); + } + + // 5. Fetch lesson and module data from LMS service through middleware + let lmsData = []; + try { + lmsData = await this.fetchLessonModuleDataFromLMS(userId, tenantId, organisationId); + } catch (error) { + this.logger.warn(`Failed to fetch LMS data for userId: ${userId}, continuing without LMS data:`, error.message); + lmsData = []; + } + + // 6. Fetch question and answer data from Assessment service through middleware + let assessmentData = []; + try { + assessmentData = await this.fetchQuestionAnswerDataFromAssessment(userId, tenantId, organisationId); + } catch (error) { + this.logger.warn(`Failed to fetch assessment data for userId: ${userId}, continuing without assessment data:`, error.message); + assessmentData = []; + } + + // 7. Merge all data together + const completeUserData = this.mergeUserDataWithAllServices(userProfile, applications, lmsData, assessmentData); + + this.logger.log(`Comprehensive sync completed for userId: ${userId}`); + return completeUserData; + } catch (error) { + this.logger.error(`Failed to perform comprehensive sync for userId: ${userId}:`, error); + throw error; + } + } + + /** + * Fetch user profile data including custom fields + * + * @param user - User entity from database + * @returns Promise - User profile with custom fields + */ + private async fetchUserProfile(user: User): Promise { + try { + this.logger.log(`Fetching user profile for userId: ${user.userId}`); + this.logger.debug(`User data from database:`, { + userId: user.userId, + username: user.username, + firstName: user.firstName, + lastName: user.lastName, + email: user.email, + mobile: user.mobile, + gender: user.gender, + status: user.status + }); + + // Check if user has basic profile data + if (!user.firstName && !user.lastName && !user.email) { + this.logger.warn(`User ${user.userId} has empty profile data in database. Attempting to fetch from user service...`); + + // Try to fetch user data from user service as fallback + try { + const userService = this.userRepository.manager.getRepository('Users'); + const freshUserData = await userService.findOne({ + where: { userId: user.userId }, + select: ['userId', 'username', 'firstName', 'lastName', 'middleName', 'email', 'mobile', 'mobile_country_code', 'gender', 'dob', 'country', 'address', 'district', 'state', 'pincode', 'status'] + }); + + if (freshUserData && (freshUserData.firstName || freshUserData.lastName || freshUserData.email)) { + this.logger.log(`Found fresh user data for ${user.userId}, using it instead`); + // Update the user object with fresh data + Object.assign(user, freshUserData); + } else { + this.logger.warn(`No fresh user data found for ${user.userId}, using default profile`); + } + } catch (fallbackError) { + this.logger.error(`Failed to fetch fresh user data for ${user.userId}:`, fallbackError); + } + } + + // Fetch custom fields for the user + const customFields = await this.fetchUserCustomFields(user.userId); + + // Format date of birth + let formattedDob: string | null = null; + if (user.dob instanceof Date) { + formattedDob = user.dob.toISOString(); + } else if (typeof user.dob === 'string') { + formattedDob = user.dob; + } + + // Create profile object with better fallbacks + const profile: IProfile = { + userId: user.userId, + username: user.username || `user-${user.userId}`, + firstName: user.firstName || 'User', + lastName: user.lastName || 'Name', + middleName: user.middleName || '', + email: user.email || `user-${user.userId}@example.com`, + mobile: user.mobile?.toString() || '', + mobile_country_code: user.mobile_country_code || '', + gender: user.gender || '', + dob: formattedDob, + country: user.country || '', + address: user.address || '', + district: user.district || '', + state: user.state || '', + pincode: user.pincode || '', + status: user.status || 'active', + customFields, + }; + + this.logger.log(`Successfully created profile for userId: ${user.userId}`, { + username: profile.username, + firstName: profile.firstName, + lastName: profile.lastName, + email: profile.email, + hasCustomFields: customFields.length > 0 + }); + + return profile; + + } catch (error) { + this.logger.error(`Failed to fetch user profile for ${user.userId}:`, error); + + // Return a default profile as last resort + return { + userId: user.userId, + username: `user-${user.userId}`, + firstName: 'User', + lastName: 'Name', + middleName: '', + email: `user-${user.userId}@example.com`, + mobile: '', + mobile_country_code: '', + gender: '', + dob: null, + country: '', + address: '', + district: '', + state: '', + pincode: '', + status: 'active', + customFields: [], + }; + } + } + + /** + * Fetch user custom fields from database using existing fields service + * This ensures consistency with the existing implementation + * + * @param userId - User ID to fetch custom fields for + * @returns Promise - Array of custom field objects + */ + private async fetchUserCustomFields(userId: string): Promise { + try { + // Use existing fields service to get custom field details + const customFields = await this.fieldsService.getUserCustomFieldDetails(userId); + + this.logger.debug(`Found ${customFields.length} custom fields for user ${userId}`); + + // Get all form submissions for this user to identify form fields + const submissions = await this.formSubmissionRepository.find({ + where: { itemId: userId }, + }); + + this.logger.debug(`Found ${submissions.length} form submissions for filtering custom fields`); + + // Collect all form field IDs to exclude from custom fields + const formFieldIds = new Set(); + + for (const submission of submissions) { + try { + // Extract field IDs from form schema using proper implementation + const formFields = await this.extractFieldIdsFromFormSchema(submission.formId); + formFields.forEach(fieldId => formFieldIds.add(fieldId)); + } catch (error) { + this.logger.warn(`Failed to extract field IDs from form ${submission.formId}:`, error); + } + } + + this.logger.debug(`Form field IDs to exclude: ${Array.from(formFieldIds).join(', ')}`); + + // Filter out fields that are part of form schemas + const filteredCustomFields = customFields.filter((field) => !formFieldIds.has(field.fieldId)); + + this.logger.debug(`After filtering, ${filteredCustomFields.length} custom fields remain`); + + // Transform to match the expected format + const transformedFields = filteredCustomFields.map(field => ({ + fieldId: field.fieldId, + fieldValuesId: field.fieldValuesId || '', + fieldname: field.label || '', + code: field.code || '', + label: field.label || '', + type: field.type || '', + value: this.processFieldValueForElasticsearch(field.value), + context: field.context || '', + contextType: field.contextType || '', + state: field.state || '', + fieldParams: field.fieldParams || {}, + })); + + this.logger.debug(`Returning ${transformedFields.length} transformed custom fields`); + return transformedFields; + + } catch (error) { + this.logger.error(`Failed to fetch custom fields for ${userId}:`, error); + return []; + } + } + + /** + * Fetch user applications with form submissions and cohort details + * + * @param userId - User ID to fetch applications for + * @returns Promise - Array of application objects + */ + async fetchUserApplications(userId: string): Promise { + try { + this.logger.log(`Fetching applications for userId: ${userId}`); + + // Get all cohort members for this user (without relations since they don't exist) + const cohortMembers = await this.cohortMembersRepository.find({ + where: { userId } + }); + + if (cohortMembers.length === 0) { + this.logger.warn(`No cohort members found for userId: ${userId}`); + + // Check if user has form submissions to create a basic application + const submissions = await this.formSubmissionRepository.find({ + where: { itemId: userId } + }); + + if (submissions.length > 0) { + this.logger.log(`Found ${submissions.length} form submissions for user without cohort memberships, creating basic application`); + + // Build proper form data and progress for the first submission + const submission = submissions[0]; + this.logger.debug(`Building form data for submission:`, submission.submissionId); + + const { formData, pages } = await this.buildFormDataWithPages(submission); + this.logger.debug(`Form data built:`, formData); + this.logger.debug(`Pages built:`, pages); + + const { percentage, progress } = this.calculateCompletionPercentage(formData); + this.logger.debug(`Completion percentage: ${percentage}%`); + this.logger.debug(`Progress calculated:`, progress); + + // Create a basic application for users with form submissions but no cohort memberships + const basicApplication = { + cohortId: 'default-cohort', + cohortmemberstatus: 'active', + cohortDetails: { + cohortId: 'default-cohort', + name: 'Default Cohort', + type: 'COHORT', + status: 'active', + }, + formId: submission.formId, + submissionId: submission.submissionId, + formstatus: 'active', + completionPercentage: percentage, + progress: progress, + lastSavedAt: submission.updatedAt?.toISOString() || new Date().toISOString(), + submittedAt: submission.status === 'active' ? submission.updatedAt?.toISOString() : null, + formData: formData, + courses: { + type: 'nested', + values: [] + } + }; + + this.logger.debug(`Created basic application:`, basicApplication); + return [basicApplication]; + } + + return []; + } + + const applications = []; + + for (const cohortMember of cohortMembers) { + try { + // Get form submissions for this user + const submissions = await this.formSubmissionRepository.find({ + where: { itemId: userId }, + }); + + const application = await this.buildApplicationForCohort(userId, cohortMember, submissions); + if (application) { + applications.push(application); + } + } catch (error) { + this.logger.warn(`Failed to build application for cohort ${cohortMember.cohortId}, skipping:`, error.message); + // Continue with other applications instead of failing completely + continue; + } + } + + this.logger.log(`Returning ${applications.length} applications for user ${userId}`); + return applications; + } catch (error) { + this.logger.error(`Failed to fetch applications for userId: ${userId}:`, error); + throw error; + } + } + + /** + * Build application object for a specific cohort + * + * @param userId - User ID + * @param membership - Cohort membership entity + * @param submissions - All form submissions for the user + * @returns Promise - Application object or null + */ + private async buildApplicationForCohort( + userId: string, + membership: CohortMembers, + submissions: FormSubmission[] + ): Promise { + try { + // Always fetch cohort details first (this is reliable) + const cohort = await this.cohortRepository.findOne({ + where: { cohortId: membership.cohortId }, + }); + + // Initialize application with basic cohort data + const application = { + cohortId: membership.cohortId, + cohortmemberstatus: membership.status || 'active', + cohortDetails: { + cohortId: membership.cohortId, + name: cohort?.name || 'Unknown Cohort', + type: 'COHORT', + status: cohort?.status || 'active', + }, + // Initialize with empty form data + formId: '', + submissionId: '', + formstatus: 'active', + completionPercentage: 0, + progress: {}, + lastSavedAt: null, + submittedAt: null, + formData: {}, + // Initialize with empty courses structure + courses: { + type: 'nested', + values: [] + } + }; + + // Try to find form submission for this cohort + let submission = null; + + for (const sub of submissions) { + try { + const form = await this.formsService.getFormById(sub.formId); + if (form?.contextId === membership.cohortId) { + submission = sub; + break; + } + } catch (error) { + this.logger.warn(`Failed to fetch form for formId ${sub.formId}:`, error); + // Continue to next submission instead of failing + continue; + } + } + + // If no submission found by contextId, use the first submission as fallback + if (!submission && submissions.length > 0) { + submission = submissions[0]; + this.logger.warn(`No submission found for cohort ${membership.cohortId}, using first submission as fallback`); + } + + // If we have a submission, try to build form data + if (submission) { + try { + // Build form data with proper page structure + const { formData, pages } = await this.buildFormDataWithPages(submission); + + // Calculate completion percentage and progress + const { percentage, progress } = this.calculateCompletionPercentage(formData); + + // Update application with form data + application.formId = submission.formId; + application.submissionId = submission.submissionId; + application.formstatus = submission.status || 'active'; + application.completionPercentage = percentage; + application.progress = progress; + application.lastSavedAt = submission.updatedAt ? submission.updatedAt.toISOString() : null; + application.submittedAt = submission.status === 'active' ? submission.updatedAt?.toISOString() : null; + application.formData = formData; + } catch (formError) { + this.logger.warn(`Failed to build form data for submission ${submission.submissionId}:`, formError); + // Keep application with basic data, form data will be empty + } + } else { + this.logger.warn(`No form submission found for user ${userId} in cohort ${membership.cohortId}`); + } + + // Always try to fetch course data (this should work independently) + try { + application.courses = await this.getCourseDataForApplication(userId, membership.cohortId); + } catch (courseError) { + this.logger.warn(`Failed to fetch course data for user ${userId}, cohort ${membership.cohortId}:`, courseError); + // Initialize with empty courses structure + application.courses = { + type: 'nested', + values: [] + }; + } + + return application; + + } catch (error) { + this.logger.error(`Failed to build application for cohort ${membership.cohortId}:`, error); + // Return basic application with cohort data even if everything else fails + return { + cohortId: membership.cohortId, + cohortmemberstatus: membership.status || 'active', + cohortDetails: { + cohortId: membership.cohortId, + name: 'Unknown Cohort', + type: 'COHORT', + status: 'active', + }, + formId: '', + submissionId: '', + formstatus: 'active', + completionPercentage: 0, + progress: {}, + lastSavedAt: null, + submittedAt: null, + formData: {}, + courses: { + type: 'nested', + values: [] + } + }; + } + } + + /** + * Build form data from submission + * + * @param submission - Form submission entity + * @returns Promise - Form data object + */ + private async buildFormDataFromSubmission(submission: FormSubmission): Promise { + try { + // Fetch field values for this submission + // Note: FieldValues entity doesn't have formId field, so we'll filter by itemId only + const fieldValues = await this.fieldValuesRepository.find({ + where: { + itemId: submission.itemId, + }, + relations: ['field'], + }); + + // Group field values by page (simplified - you may need to implement proper page mapping) + const formData: any = {}; + + for (const fieldValue of fieldValues) { + // For now, put all fields in a default page + // In a real implementation, you'd need to map fields to pages based on form schema + const pageId = 'default'; + + if (!formData[pageId]) { + formData[pageId] = {}; + } + + formData[pageId][fieldValue.fieldId] = this.processFieldValueForElasticsearch(fieldValue.value); + } + + return formData; + + } catch (error) { + this.logger.error(`Failed to build form data from submission ${submission.submissionId}:`, error); + return {}; + } + } + + /** + * Build form data with proper page structure from submission + * + * @param submission - Form submission entity + * @returns Promise - Form data with page structure + */ + private async buildFormDataWithPages(submission: FormSubmission): Promise { + try { + this.logger.debug(`Building form data with pages for submission ${submission.submissionId}`); + + // Fetch field values for this submission + const fieldValues = await this.fieldValuesRepository.find({ + where: { + itemId: submission.itemId, + }, + relations: ['field'], + }); + + this.logger.debug(`Found ${fieldValues.length} field values for submission ${submission.submissionId}`); + + // Get form schema to build proper page structure + const formSchema = await this.getFormSchema(submission.formId); + this.logger.debug(`Retrieved form schema for formId ${submission.formId}:`, Object.keys(formSchema)); + + const fieldIdToPageName = this.getFieldIdToPageNameMap(formSchema); + this.logger.debug(`Field ID to page name mapping:`, fieldIdToPageName); + + // Build page structure + const formData: any = {}; + const pages: any = {}; + + // Initialize pages from schema + for (const [pageKey, pageSchema] of Object.entries(formSchema)) { + const pageName = pageKey === 'default' ? 'eligibilityCheck' : pageKey; + pages[pageName] = { completed: true, fields: {} }; + formData[pageName] = {}; + } + + this.logger.debug(`Initialized ${Object.keys(pages).length} pages from schema`); + + // Map field values to correct pages + for (const fieldValue of fieldValues) { + const pageName = fieldIdToPageName[fieldValue.fieldId]; + if (!pageName) { + this.logger.warn(`Field ${fieldValue.fieldId} not found in schema mapping, skipping`); + continue; + } + + if (!pages[pageName]) { + pages[pageName] = { completed: true, fields: {} }; + formData[pageName] = {}; + } + + const processedValue = this.processFieldValueForElasticsearch(fieldValue.value); + pages[pageName].fields[fieldValue.fieldId] = processedValue; + formData[pageName][fieldValue.fieldId] = processedValue; + + this.logger.debug(`Mapped field ${fieldValue.fieldId} to page ${pageName} with value:`, processedValue); + } + + // Update page completion status + for (const [pageName, pageData] of Object.entries(pages)) { + const fields = (pageData as any).fields; + const fieldCount = Object.keys(fields).length; + const completedFields = Object.values(fields).filter(value => + value !== null && value !== undefined && value !== '' + ).length; + + pages[pageName].completed = fieldCount > 0 && completedFields === fieldCount; + this.logger.debug(`Page ${pageName}: ${completedFields}/${fieldCount} fields completed`); + } + + this.logger.debug(`Final form data:`, formData); + this.logger.debug(`Final pages:`, pages); + + return { formData, pages }; + + } catch (error) { + this.logger.error(`Failed to build form data with pages for submission ${submission.submissionId}:`, error); + return { formData: {}, pages: {} }; + } + } + + /** + * Calculate completion percentage from form data + * + * @param formData - Form data object + * @returns Object with percentage and progress data + */ + private calculateCompletionPercentage(formData: any): { percentage: number; progress: any } { + let totalFields = 0; + let completedFields = 0; + const pages: any = {}; + + // Process each page + for (const [pageId, pageData] of Object.entries(formData)) { + const pageFields: any = {}; + let pageCompleted = true; + let pageTotal = 0; + let pageCompletedCount = 0; + + // Process each field in the page + for (const [fieldId, value] of Object.entries(pageData as any)) { + pageTotal++; + totalFields++; + + if (value !== null && value !== undefined && value !== '') { + pageCompletedCount++; + completedFields++; + pageFields[fieldId] = value; + } else { + pageCompleted = false; + pageFields[fieldId] = value; + } + } + + // Set page completion status + pages[pageId] = { + completed: pageCompleted, + fields: pageFields, + }; + } + + // Calculate overall percentage + const percentage = totalFields > 0 ? Math.round((completedFields / totalFields) * 100) : 0; + + return { + percentage, + progress: { + pages, + overall: { + completed: completedFields, + total: totalFields, + }, + }, + }; + } + + /** + * Fetch user courses (placeholder for future implementation) + * + * @param userId - User ID to fetch courses for + * @returns Promise - Array of course objects + */ + private async fetchUserCourses(userId: string): Promise { + // Placeholder for future course implementation + // This can be expanded when course functionality is added + return []; + } + + /** + * Get form schema from forms service + * + * @param formId - Form ID + * @returns Promise - Form schema + */ + private async getFormSchema(formId: string): Promise { + try { + const form = await this.formsService.getFormById(formId); + const fieldsObj = form && (form as any).fields ? (form as any).fields : null; + + // Handle different schema structures + let schema: any = {}; + if (fieldsObj) { + // Try different possible schema structures + if ( + Array.isArray(fieldsObj?.result) && + fieldsObj.result[0]?.schema?.properties + ) { + // Structure: { result: [{ schema: { properties: {...} } }] } + schema = fieldsObj.result[0].schema.properties; + } else if (fieldsObj?.schema?.properties) { + // Structure: { schema: { properties: {...} } } + schema = fieldsObj.schema.properties; + } else if (fieldsObj?.properties) { + // Structure: { properties: {...} } + schema = fieldsObj.properties; + } else if (typeof fieldsObj === 'object' && fieldsObj !== null) { + // Try to find schema in nested structure + const findSchema = (obj: any): any => { + if (obj?.schema?.properties) return obj.schema.properties; + if (obj?.properties) return obj.properties; + if (Array.isArray(obj)) { + for (const item of obj) { + const found = findSchema(item); + if (found) return found; + } + } else if (typeof obj === 'object') { + for (const key in obj) { + const found = findSchema(obj[key]); + if (found) return found; + } + } + return null; + }; + schema = findSchema(fieldsObj) || {}; + } + } + + this.logger.debug(`Extracted schema for form ${formId}:`, Object.keys(schema)); + return schema; + } catch (error) { + this.logger.error(`Failed to get form schema for ${formId}:`, error); + return {}; + } + } + + /** + * Extract field IDs from form schema using proper implementation + * + * @param formId - Form ID to extract field IDs from + * @returns Promise - Array of field IDs + */ + private async extractFieldIdsFromFormSchema(formId: string): Promise { + try { + const schema = await this.getFormSchema(formId); + const fieldIds: string[] = []; + + // Extract field IDs from schema structure + for (const [pageKey, pageSchema] of Object.entries(schema)) { + const fieldProps = (pageSchema as any).properties || {}; + + const extractFieldIds = (properties: any) => { + for (const [fieldKey, fieldSchema] of Object.entries(properties)) { + const fieldId = (fieldSchema as any).fieldId; + if (fieldId) { + fieldIds.push(fieldId); + } + + // Handle dependencies + if ((fieldSchema as any).dependencies) { + const dependencies = (fieldSchema as any).dependencies; + for (const depSchema of Object.values(dependencies)) { + if (!depSchema || typeof depSchema !== 'object') continue; + const dep = depSchema as any; + if (dep.oneOf) + dep.oneOf.forEach((item: any) => + item?.properties && extractFieldIds(item.properties) + ); + if (dep.allOf) + dep.allOf.forEach((item: any) => + item?.properties && extractFieldIds(item.properties) + ); + if (dep.anyOf) + dep.anyOf.forEach((item: any) => + item?.properties && extractFieldIds(item.properties) + ); + if (dep.properties) extractFieldIds(dep.properties); + } + } + } + }; + + extractFieldIds(fieldProps); + + // Handle page-level dependencies + const pageDependencies = (pageSchema as any).dependencies || {}; + for (const depSchema of Object.values(pageDependencies)) { + if (!depSchema || typeof depSchema !== 'object') continue; + const dep = depSchema as any; + if (dep.oneOf) + dep.oneOf.forEach((item: any) => + item?.properties && extractFieldIds(item.properties) + ); + if (dep.allOf) + dep.allOf.forEach((item: any) => + item?.properties && extractFieldIds(item.properties) + ); + if (dep.anyOf) + dep.anyOf.forEach((item: any) => + item?.properties && extractFieldIds(item.properties) + ); + if (dep.properties) extractFieldIds(dep.properties); + } + } + + this.logger.debug(`Extracted ${fieldIds.length} field IDs from form ${formId}`); + return fieldIds; + + } catch (error) { + this.logger.error(`Failed to extract field IDs from form ${formId}:`, error); + return []; + } + } + + /** + * Get field ID to page name mapping from schema + * + * @param schema - Form schema + * @returns Record - Field ID to page name mapping + */ + private getFieldIdToPageNameMap(schema: any): Record { + const fieldIdToPageName: Record = {}; + + function extract(properties: any, currentPage: string) { + if (!properties || typeof properties !== 'object') return; + for (const [fieldKey, fieldSchema] of Object.entries(properties)) { + if (!fieldSchema || typeof fieldSchema !== 'object') continue; + const fieldId = (fieldSchema as any).fieldId; + if (fieldId) fieldIdToPageName[fieldId] = currentPage; + + // Traverse dependencies + if ((fieldSchema as any).dependencies) { + for (const depSchema of Object.values((fieldSchema as any).dependencies)) { + if (!depSchema || typeof depSchema !== 'object') continue; + const dep = depSchema as any; + if (dep.oneOf) + dep.oneOf.forEach((item: any) => + item?.properties && extract(item.properties, currentPage) + ); + if (dep.allOf) + dep.allOf.forEach((item: any) => + item?.properties && extract(item.properties, currentPage) + ); + if (dep.anyOf) + dep.anyOf.forEach((item: any) => + item?.properties && extract(item.properties, currentPage) + ); + if (dep.properties) extract(dep.properties, currentPage); + } + } + } + } + + for (const [pageKey, pageSchema] of Object.entries(schema)) { + const pageName = pageKey === 'default' ? 'eligibilityCheck' : pageKey; + extract((pageSchema as any).properties, pageName); + + // Also check for page-level dependencies + if ((pageSchema as any).dependencies) { + for (const depSchema of Object.values((pageSchema as any).dependencies)) { + if (!depSchema || typeof depSchema !== 'object') continue; + const dep = depSchema as any; + if (dep.oneOf) + dep.oneOf.forEach((item: any) => + item?.properties && extract(item.properties, pageName) + ); + if (dep.allOf) + dep.allOf.forEach((item: any) => + item?.properties && extract(item.properties, pageName) + ); + if (dep.anyOf) + dep.anyOf.forEach((item: any) => + item?.properties && extract(item.properties, pageName) + ); + if (dep.properties) extract(dep.properties, pageName); + } + } + } + + return fieldIdToPageName; + } + + /** + * Process field value for Elasticsearch storage + * Converts array values to comma-separated strings for multiselect fields + * + * @param value - The field value to process + * @returns Processed value (array becomes comma-separated string, other types unchanged) + */ + private processFieldValueForElasticsearch(value: any): any { + // If value is an array, convert to comma-separated string + if (Array.isArray(value)) { + return value.join(', '); + } + // Return value as-is for non-array values + return value; + } + + /** + * Check if Elasticsearch is enabled + * + * @returns boolean - True if Elasticsearch is enabled + */ + isElasticsearchEnabled(): boolean { + return isElasticsearchEnabled(); + } + + /** + * Get course data for a specific application (user + cohort combination) + * + * @param userId - User ID to fetch courses for + * @param cohortId - Cohort ID to filter courses by + * @returns Promise - Courses object with nested structure + */ + private async getCourseDataForApplication(userId: string, cohortId: string): Promise<{ + type: 'nested'; + values: any[]; + }> { + try { + this.logger.debug(`Fetching course data for user ${userId} in cohort ${cohortId}`); + + // Call the LMS service through middleware to get course data + const courseData = await this.fetchCourseDataFromLMS(userId, cohortId); + + return { + type: 'nested', + values: courseData + }; + + } catch (error) { + this.logger.error(`Failed to fetch course data for user ${userId}, cohort ${cohortId}:`, error); + return { type: 'nested', values: [] }; + } + } + + /** + * Fetch all existing answer data for a user from assessment service + * This ensures that when we sync course data, we also include any existing quiz answers + */ + async fetchUserAnswerData(userId: string, tenantId: string, organisationId: string): Promise { + try { + this.logger.log(`Fetching answer data for userId: ${userId}`); + + // Note: Assessment service doesn't have an endpoint to get all attempts for a user + // This method will be enhanced when such an endpoint is available + // For now, return empty array to avoid errors + this.logger.warn(`Assessment service doesn't have endpoint to get all attempts for user ${userId}`); + return []; + + // TODO: Implement when assessment service has getUserAttempts endpoint + // const assessmentServiceUrl = process.env.ASSESSMENT_SERVICE_URL || 'http://localhost:4000'; + // const response = await axios.get(`${assessmentServiceUrl}/assessment-service/v1/attempts/user/${userId}`, { + // headers: { + // 'Content-Type': 'application/json', + // 'tenantId': tenantId, + // 'organisationId': organisationId, + // 'userId': userId + // }, + // timeout: 10000 + // }); + } catch (error) { + this.logger.error(`Failed to fetch answer data for userId: ${userId}:`, error); + return []; + } + } + + /** + * Fetch course data from LMS service through middleware + */ + private async fetchCourseDataFromLMS(userId: string, cohortId: string): Promise { + try { + this.logger.debug(`Fetching course data for user ${userId} in cohort ${cohortId}`); + + // Generate authentication token + const authToken = this.generateServiceAuthToken(); + + // If no authentication token is available, skip course data fetching + if (!authToken) { + this.logger.warn(`No authentication token available, skipping course data fetching for userId: ${userId}`); + return []; + } + + // Use middleware URL instead of direct LMS service URL + const middlewareUrl = process.env.MIDDLEWARE_URL || 'http://localhost:4000'; + const lmsEndpoint = `${middlewareUrl}/lms-service/v1/enrollments/user/${userId}/cohort/${cohortId}`; + + const headers = { + 'Content-Type': 'application/json', + 'Authorization': authToken + }; + + this.logger.debug(`Calling LMS through middleware: ${lmsEndpoint}`); + + const response = await axios.get(lmsEndpoint, { headers }); + + if (response.data && response.data.result && response.data.result.data) { + this.logger.log(`Fetched ${response.data.result.data.length} course enrollments for userId: ${userId}`); + return response.data.result.data; + } else { + this.logger.warn(`No course data found for userId: ${userId} in cohort: ${cohortId}`); + return []; + } + } catch (error) { + if (error.response?.status === 404) { + this.logger.warn(`No course data found for userId: ${userId} in cohort: ${cohortId} - this is normal if user has no course enrollments`); + return []; + } else if (error.response?.status === 401) { + this.logger.warn(`Authentication failed for LMS service for userId: ${userId} - this is normal if user has no course data`); + return []; + } else { + this.logger.error(`Failed to fetch course data for userId: ${userId}:`, error.message); + throw error; + } + } + } + + /** + * Merge all user data together ensuring consistency + */ + private mergeUserData(userProfile: any, applications: any[], courseData: any[], answerData: any[]): any { + // Create base user document + const userDocument = { + userId: userProfile.userId, + profile: userProfile, + applications: applications || [], + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString() + }; + + // Enhance applications with course data + if (applications && applications.length > 0 && courseData.length > 0) { + for (const application of applications) { + // Find matching course data for this application's cohortId + const matchingCourses = courseData.filter(course => + course.cohortId === application.cohortId + ); + + if (matchingCourses.length > 0) { + // Initialize courses structure if not exists + if (!application.courses) { + application.courses = { + type: 'nested', + values: [] + }; + } + + // Add course data to application + for (const course of matchingCourses) { + // Check if course already exists in application + const existingCourseIndex = application.courses.values.findIndex( + (c: any) => c.courseId === course.courseId + ); + + if (existingCourseIndex >= 0) { + // Update existing course + application.courses.values[existingCourseIndex] = course; + } else { + // Add new course + application.courses.values.push(course); + } + } + } + } + } + + // Enhance course data with answer data + if (answerData.length > 0) { + this.enhanceCourseDataWithAnswers(userDocument.applications, answerData); + } + + return userDocument; + } + + /** + * Enhance course data with answer data + */ + private enhanceCourseDataWithAnswers(applications: any[], answerData: any[]): void { + for (const application of applications) { + if (application.courses && application.courses.values) { + for (const course of application.courses.values) { + if (course.units && course.units.values) { + for (const unit of course.units.values) { + if (unit.contents && unit.contents.values) { + for (const content of unit.contents.values) { + if (content.type === 'test') { + // Find matching answer data for this test + const matchingAnswerData = answerData.find(answer => + answer.testId === content.contentId + ); + + if (matchingAnswerData) { + content.tracking = { + ...content.tracking, + questionsAttempted: matchingAnswerData.questionsAttempted, + totalQuestions: matchingAnswerData.totalQuestions, + score: matchingAnswerData.score, + percentComplete: matchingAnswerData.percentComplete, + timeSpent: matchingAnswerData.timeSpent, + answers: { + type: 'nested', + values: matchingAnswerData.answers + } + }; + + // Update content status based on completion + if (matchingAnswerData.percentComplete >= 100) { + content.status = 'completed'; + } else if (matchingAnswerData.percentComplete > 0) { + content.status = 'in_progress'; + } + } + } + } + } + } + } + } + } + } + } + + /** + * Fetch lesson and module data from LMS service through middleware + */ + private async fetchLessonModuleDataFromLMS(userId: string, tenantId: string, organisationId: string): Promise { + try { + this.logger.debug(`Fetching lesson and module data from LMS for userId: ${userId}`); + + // Generate authentication token + const authToken = this.generateServiceAuthToken(); + + // If no authentication token is available, skip LMS data fetching + if (!authToken) { + this.logger.warn(`No authentication token available, skipping LMS data fetching for userId: ${userId}`); + return []; + } + + // Use middleware URL instead of direct LMS service URL + const middlewareUrl = process.env.MIDDLEWARE_URL || 'http://localhost:4000'; + const lmsEndpoint = `${middlewareUrl}/lms-service/v1/tracking/attempts/progress/${userId}`; + + const headers = { + 'Content-Type': 'application/json', + 'tenantid': tenantId, + 'organisationid': organisationId, + 'Authorization': authToken + }; + + this.logger.debug(`Calling LMS through middleware: ${lmsEndpoint}`); + + const response = await axios.get(lmsEndpoint, { headers }); + + if (response.data && response.data.result && response.data.result.data) { + this.logger.log(`Fetched ${response.data.result.data.length} lesson tracks from LMS for userId: ${userId}`); + return response.data.result.data; + } else { + this.logger.warn(`No lesson tracking data found for userId: ${userId}`); + return []; + } + } catch (error) { + if (error.response?.status === 404) { + this.logger.warn(`No lesson tracking data found for userId: ${userId} - this is normal if user has no LMS activity`); + return []; + } else if (error.response?.status === 401) { + this.logger.warn(`Authentication failed for LMS service for userId: ${userId} - this is normal if user has no LMS data`); + return []; + } else { + this.logger.error(`Failed to fetch LMS data for userId: ${userId}:`, error.message); + throw error; + } + } + } + + /** + * Fetch question and answer data from Assessment service through middleware + */ + private async fetchQuestionAnswerDataFromAssessment(userId: string, tenantId: string, organisationId: string): Promise { + try { + this.logger.debug(`Fetching question and answer data from Assessment for userId: ${userId}`); + + // Generate authentication token + const authToken = this.generateServiceAuthToken(); + + // If no authentication token is available, skip assessment data fetching + if (!authToken) { + this.logger.warn(`No authentication token available, skipping assessment data fetching for userId: ${userId}`); + return []; + } + + // Use middleware URL instead of direct Assessment service URL + const middlewareUrl = process.env.MIDDLEWARE_URL || 'http://localhost:4000'; + const assessmentEndpoint = `${middlewareUrl}/assessment/v1/attempts/user/${userId}`; + + const headers = { + 'Content-Type': 'application/json', + 'tenantid': tenantId, + 'organisationid': organisationId, + 'Authorization': authToken + }; + + this.logger.debug(`Calling Assessment through middleware: ${assessmentEndpoint}`); + + const response = await axios.get(assessmentEndpoint, { headers }); + + if (response.data && response.data.result && response.data.result.data) { + const attempts = response.data.result.data; + this.logger.log(`Fetched ${attempts.length} assessment attempts for userId: ${userId}`); + + // Process each attempt to get enhanced answers with text content + const enhancedAttempts = []; + + for (const attempt of attempts) { + try { + // Get answers for this attempt with enhanced text content + const answersEndpoint = `${middlewareUrl}/assessment/v1/attempts/${attempt.attemptId}/answers`; + const answersResponse = await axios.get(answersEndpoint, { headers }); + + if (answersResponse.data && answersResponse.data.result && answersResponse.data.result.data) { + const attemptData = answersResponse.data.result.data; + + // Enhance the attempt data with enhanced answers + enhancedAttempts.push({ + ...attempt, + answers: attemptData.answers || [], + totalQuestions: attemptData.totalQuestions || 0, + score: attemptData.score || 0, + percentComplete: attemptData.percentComplete || 0, + questionsAttempted: attemptData.questionsAttempted || 0, + timeSpent: attemptData.timeSpent || 0, + status: attemptData.status || 'in_progress' + }); + } else { + // If no answers found, still include the attempt + enhancedAttempts.push({ + ...attempt, + answers: [], + totalQuestions: 0, + score: 0, + percentComplete: 0, + questionsAttempted: 0, + timeSpent: 0, + status: 'not_started' + }); + } + } catch (error) { + this.logger.warn(`Failed to fetch answers for attempt ${attempt.attemptId}:`, error.message); + // Still include the attempt even if answers fetch failed + enhancedAttempts.push({ + ...attempt, + answers: [], + totalQuestions: 0, + score: 0, + percentComplete: 0, + questionsAttempted: 0, + timeSpent: 0, + status: 'error' + }); + } + } + + return enhancedAttempts; + } else { + this.logger.warn(`No assessment attempts found for userId: ${userId}`); + return []; + } + } catch (error) { + if (error.response?.status === 404) { + this.logger.warn(`No assessment attempts found for userId: ${userId} - this is normal if user has no assessment activity`); + return []; + } else if (error.response?.status === 401) { + this.logger.warn(`Authentication failed for Assessment service for userId: ${userId} - this is normal if user has no assessment data`); + return []; + } else { + this.logger.error(`Failed to fetch assessment data for userId: ${userId}:`, error.message); + throw error; + } + } + } + + /** + * Generate authentication token for service-to-service communication + */ + private generateServiceAuthToken(): string { + // Try to get service token from environment + const serviceToken = process.env.SERVICE_AUTH_TOKEN; + if (serviceToken) { + return `Bearer ${serviceToken}`; + } + + // Try to get API key from environment + const apiKey = process.env.LMS_SERVICE_API_KEY; + if (apiKey) { + return `ApiKey ${apiKey}`; + } + + // Try to get JWT secret to generate a service token + const jwtSecret = process.env.JWT_SECRET; + if (jwtSecret) { + // For service-to-service communication, we'll use a special service user token + this.logger.debug('Using JWT secret to generate service token'); + return 'Bearer service-internal-token'; + } + + // Fallback - skip course data fetching for now + this.logger.warn('No authentication token configured, skipping course data fetching'); + return null; + } + + /** + * Merge all user data from all three services together + */ + private mergeUserDataWithAllServices(userProfile: any, applications: any[], lmsData: any[], assessmentData: any[]): any { + // Create base user document + const userDocument = { + userId: userProfile.userId, + profile: userProfile, + applications: applications || [], + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString() + }; + + // Enhance applications with LMS data (lessons and modules) + if (applications && applications.length > 0 && lmsData.length > 0) { + for (const application of applications) { + // Find matching lesson tracks for this application's cohortId + const matchingLessonTracks = lmsData.filter(lessonTrack => { + // Extract cohortId from course params or use courseId + const lessonCohortId = lessonTrack.course?.params?.cohortId || lessonTrack.courseId; + return lessonCohortId === application.cohortId; + }); + + if (matchingLessonTracks.length > 0) { + // Initialize courses structure if not exists + if (!application.courses) { + application.courses = { + type: 'nested', + values: [] + }; + } + + // Group lesson tracks by course + const coursesByCourseId = new Map(); + for (const lessonTrack of matchingLessonTracks) { + const courseId = lessonTrack.courseId; + if (!coursesByCourseId.has(courseId)) { + coursesByCourseId.set(courseId, []); + } + coursesByCourseId.get(courseId)!.push(lessonTrack); + } + + // Build course structure with lessons and modules + for (const [courseId, lessonTracks] of coursesByCourseId) { + const courseData = this.buildCourseDataWithLessonsAndModules(lessonTracks); + + // Check if course already exists in application + const existingCourseIndex = application.courses.values.findIndex( + (c: any) => c.courseId === courseId + ); + + if (existingCourseIndex >= 0) { + // Update existing course + application.courses.values[existingCourseIndex] = courseData; + } else { + // Add new course + application.courses.values.push(courseData); + } + } + } + } + } + + // Enhance course data with assessment data (questions and answers) + if (assessmentData.length > 0) { + this.enhanceCourseDataWithAssessmentData(userDocument.applications, assessmentData); + } + + return userDocument; + } + + /** + * Build course data with lessons and modules from LMS data + */ + private buildCourseDataWithLessonsAndModules(lessonTracks: any[]): any { + if (lessonTracks.length === 0) { + return null; + } + + const firstLessonTrack = lessonTracks[0]; + const course = firstLessonTrack.course; + const lesson = firstLessonTrack.lesson; + + // Group lesson tracks by module + const modulesByModuleId = new Map(); + for (const lessonTrack of lessonTracks) { + const moduleId = lessonTrack.lesson?.moduleId || 'default-module'; + if (!modulesByModuleId.has(moduleId)) { + modulesByModuleId.set(moduleId, []); + } + modulesByModuleId.get(moduleId)!.push(lessonTrack); + } + + // Build course structure + const courseData = { + courseId: course?.courseId || firstLessonTrack.courseId, + courseTitle: course?.title || 'Unknown Course', + progress: this.calculateCourseProgress(lessonTracks), + units: { + type: 'nested' as const, + values: [] + } + }; + + // Build module structure for each module + for (const [moduleId, moduleLessonTracks] of modulesByModuleId) { + const firstModuleLessonTrack = moduleLessonTracks[0]; + const module = firstModuleLessonTrack.lesson?.module; + + const unitData = { + unitId: moduleId, + unitTitle: module?.title || `Module ${moduleId}`, + progress: this.calculateModuleProgress(moduleLessonTracks), + contents: { + type: 'nested' as const, + values: [] + } + }; + + // Build lesson content for each lesson in the module + for (const lessonTrack of moduleLessonTracks) { + const lesson = lessonTrack.lesson; + + const contentData = { + contentId: lesson?.lessonId || lessonTrack.lessonId, + type: lesson?.format || 'video', + title: lesson?.title || 'Unknown Lesson', + status: this.getLessonStatus(lessonTrack), + tracking: this.buildLessonTracking(lessonTrack) + }; + + unitData.contents.values.push(contentData); + } + + courseData.units.values.push(unitData); + } + + return courseData; + } + + /** + * Calculate course progress based on lesson tracks + */ + private calculateCourseProgress(lessonTracks: any[]): number { + if (lessonTracks.length === 0) return 0; + + const totalLessons = lessonTracks.length; + const completedLessons = lessonTracks.filter(lt => + lt.status === 'completed' || lt.completionPercentage >= 100 + ).length; + + return Math.round((completedLessons / totalLessons) * 100); + } + + /** + * Calculate module progress based on lesson tracks + */ + private calculateModuleProgress(lessonTracks: any[]): number { + if (lessonTracks.length === 0) return 0; + + const totalLessons = lessonTracks.length; + const completedLessons = lessonTracks.filter(lt => + lt.status === 'completed' || lt.completionPercentage >= 100 + ).length; + + return Math.round((completedLessons / totalLessons) * 100); + } + + /** + * Get lesson status based on lesson track + */ + private getLessonStatus(lessonTrack: any): string { + if (lessonTrack.status === 'completed' || lessonTrack.completionPercentage >= 100) { + return 'completed'; + } else if (lessonTrack.status === 'started' || lessonTrack.completionPercentage > 0) { + return 'in_progress'; + } else { + return 'not_started'; + } + } + + /** + * Build lesson tracking data + */ + private buildLessonTracking(lessonTrack: any): any { + return { + percentComplete: lessonTrack.completionPercentage || 0, + lastPosition: Math.floor(lessonTrack.currentPosition || 0), + currentPosition: Math.floor(lessonTrack.currentPosition || 0), + timeSpent: lessonTrack.timeSpent || 0, + visitedPages: lessonTrack.visitedPages || [], + totalPages: lessonTrack.totalContent || 0, + lastPage: lessonTrack.currentPage || 0, + currentPage: lessonTrack.currentPage || 0, + questionsAttempted: 0, + totalQuestions: 0, + score: 0, + answers: { + type: 'nested', + values: [] + } + }; + } + + /** + * Enhance course data with assessment data (questions and answers) + */ + private enhanceCourseDataWithAssessmentData(applications: any[], assessmentData: any[]): void { + this.logger.log(`Enhancing course data with ${assessmentData.length} assessment records`); + + for (const application of applications) { + if (application.courses && application.courses.values) { + for (const course of application.courses.values) { + if (course.units && course.units.values) { + for (const unit of course.units.values) { + if (unit.contents && unit.contents.values) { + for (const content of unit.contents.values) { + // Find matching assessment data for this content + const matchingAssessmentData = assessmentData.filter(assessment => + assessment.testId === content.contentId || + assessment.attemptId === content.contentId || + assessment.lessonId === content.contentId + ); + + if (matchingAssessmentData.length > 0) { + this.logger.log(`Found ${matchingAssessmentData.length} matching assessment records for content ${content.contentId}`); + + // Update content type to test + content.type = 'test'; + + // Get the latest assessment data + const latestAssessment = matchingAssessmentData.sort((a, b) => + new Date(b.updatedAt).getTime() - new Date(a.updatedAt).getTime() + )[0]; + + this.logger.log(`Using latest assessment data for content ${content.contentId}:`, { + testId: latestAssessment.testId, + attemptId: latestAssessment.attemptId, + questionsAttempted: latestAssessment.questionsAttempted, + totalQuestions: latestAssessment.totalQuestions, + score: latestAssessment.score, + percentComplete: latestAssessment.percentComplete, + answersCount: latestAssessment.answers?.length || 0 + }); + + // Transform answers to the expected format with enhanced text content + const transformedAnswers = latestAssessment.answers?.map((answer: any) => { + // Handle different answer formats from assessment service + let answerText = ''; + let answerValue = answer.answer; + + if (typeof answerValue === 'object') { + if (answerValue.selectedOptionIds) { + // MCQ answer with selectedOptionIds - extract text from answer object + answerText = answerValue.text || `Selected options: ${answerValue.selectedOptionIds.join(', ')}`; + } else if (answerValue.text) { + // Enhanced answer with text + answerText = answerValue.text; + } else if (answerValue.answer) { + // Enhanced answer with both answer and text + answerText = answerValue.text || answerValue.answer; + } else { + // Other format + answerText = JSON.stringify(answerValue); + } + } else { + // String answer + answerText = String(answerValue); + } + + return { + questionId: answer.questionId, + type: answer.type || 'radio', + submittedAnswer: answerText, + // Add additional fields for better mapping + answer: answerValue, + text: answerText, + score: answer.score || 0, + reviewStatus: answer.reviewStatus || 'pending' + }; + }) || []; + + this.logger.log(`Transformed ${transformedAnswers.length} answers for content ${content.contentId}`); + + // Update content tracking with assessment data + content.tracking = { + ...content.tracking, + questionsAttempted: latestAssessment.questionsAttempted || 0, + totalQuestions: latestAssessment.totalQuestions || 0, + score: latestAssessment.score || 0, + percentComplete: latestAssessment.percentComplete || 0, + timeSpent: latestAssessment.timeSpent || 0, + answers: { + type: 'nested', + values: transformedAnswers + } + }; + + // Update content status based on completion + if (latestAssessment.percentComplete >= 100) { + content.status = 'completed'; + } else if (latestAssessment.percentComplete > 0) { + content.status = 'in_progress'; + } else { + content.status = 'not_started'; + } + + this.logger.log(`Updated assessment content ${content.contentId} with ${transformedAnswers.length} answers`); + } + } + } + } + } + } + } + } + } + + /** + * Check and fix empty profile data in Elasticsearch + * This method can be called to fix existing documents with empty profile data + * @param userId The user ID to check and fix + */ + async checkAndFixEmptyProfile(userId: string): Promise { + try { + this.logger.log(`Checking and fixing empty profile for userId: ${userId}`); + + // Get current user data from Elasticsearch + const currentUserData = await this.userElasticsearchService.getUser(userId) as any; + + if (!currentUserData) { + this.logger.log(`User ${userId} not found in Elasticsearch, creating new document`); + await this.comprehensiveUserSync(userId); + return; + } + + // Check if profile data is empty + const profile = currentUserData.profile; + if (!profile || !profile.firstName || !profile.lastName || !profile.email || + (profile.firstName === '' && profile.lastName === '' && profile.email === '')) { + this.logger.warn(`Empty profile data detected for userId: ${userId}, re-syncing user data`); + + // Re-sync user data from database + const freshUserData = await this.comprehensiveUserSync(userId); + + if (freshUserData && freshUserData.profile && + (freshUserData.profile.firstName || freshUserData.profile.lastName || freshUserData.profile.email)) { + this.logger.log(`Successfully fixed profile data for userId: ${userId}`); + } else { + this.logger.error(`Failed to fix profile data for userId: ${userId}`); + } + } else { + this.logger.log(`Profile data is already populated for userId: ${userId}`); + } + } catch (error) { + this.logger.error(`Error checking and fixing empty profile for userId: ${userId}:`, error); + } + } +} \ No newline at end of file diff --git a/src/elasticsearch/elasticsearch-sync.service.ts b/src/elasticsearch/elasticsearch-sync.service.ts new file mode 100644 index 00000000..b41a5221 --- /dev/null +++ b/src/elasticsearch/elasticsearch-sync.service.ts @@ -0,0 +1,423 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { UserElasticsearchService } from './user-elasticsearch.service'; +import { ElasticsearchDataFetcherService } from './elasticsearch-data-fetcher.service'; + +export enum SyncSection { + PROFILE = 'profile', + APPLICATIONS = 'applications', + COURSES = 'courses', + ASSESSMENT = 'assessment', + ALL = 'all' +} + +export interface SyncOptions { + section: SyncSection; + forceFullSync?: boolean; + skipExternalServices?: boolean; +} + +@Injectable() +export class ElasticsearchSyncService { + private readonly logger = new Logger(ElasticsearchSyncService.name); + + constructor( + private readonly userElasticsearchService: UserElasticsearchService, + private readonly elasticsearchDataFetcherService: ElasticsearchDataFetcherService, + ) {} + + + /** + * Centralized function to sync user data to Elasticsearch + * + * @param userId - User ID to sync + * @param options - Sync options including which section to update + * @returns Promise + */ + async syncUserToElasticsearch(userId: string, options: SyncOptions = { section: SyncSection.ALL }): Promise { + try { + this.logger.log(`Starting centralized sync for userId: ${userId}, section: ${options.section}`); + + // Check if user exists in Elasticsearch + const existingUser = await this.userElasticsearchService.getUser(userId); + + if (existingUser) { + this.logger.log(`User ${userId} exists in Elasticsearch, updating specific section: ${options.section}`); + await this.updateExistingUser(userId, options); + } else { + this.logger.log(`User ${userId} does not exist in Elasticsearch, creating full document`); + await this.createNewUser(userId, options); + } + + this.logger.log(`Centralized sync completed for userId: ${userId}`); + } catch (error) { + this.logger.error(`Failed to sync user ${userId} to Elasticsearch:`, error); + throw error; + } + } + + /** + * Update existing user document in Elasticsearch + */ + private async updateExistingUser(userId: string, options: SyncOptions): Promise { + try { + // Get current document from Elasticsearch + const currentDoc = await this.userElasticsearchService.getUser(userId); + if (!currentDoc) { + throw new Error(`User ${userId} not found in Elasticsearch during update`); + } + + const currentData = currentDoc._source; + let updatedData: any = {}; + + // Update specific section based on service calling + switch (options.section) { + case SyncSection.PROFILE: + updatedData = await this.updateProfileSection(userId, currentData); + break; + + case SyncSection.APPLICATIONS: + updatedData = await this.updateApplicationsSection(userId, currentData); + break; + + case SyncSection.COURSES: + updatedData = await this.updateCoursesSection(userId, currentData); + break; + + case SyncSection.ASSESSMENT: + updatedData = await this.updateAssessmentSection(userId, currentData); + break; + + case SyncSection.ALL: + default: + updatedData = await this.updateAllSections(userId, currentData, options); + break; + } + + // Update the document in Elasticsearch + await this.userElasticsearchService.updateUser( + userId, + { doc: updatedData }, + async (userId: string) => { + return await this.elasticsearchDataFetcherService.comprehensiveUserSync(userId); + } + ); + + this.logger.log(`Updated user ${userId} in Elasticsearch for section: ${options.section}`); + } catch (error) { + this.logger.error(`Failed to update existing user ${userId}:`, error); + throw error; + } + } + + /** + * Create new user document in Elasticsearch + */ + private async createNewUser(userId: string, options: SyncOptions): Promise { + try { + // Fetch complete user data from all services + const completeUserData = await this.elasticsearchDataFetcherService.comprehensiveUserSync(userId); + + if (!completeUserData) { + throw new Error(`Failed to fetch complete user data for ${userId}`); + } + + // Create new document in Elasticsearch + await this.userElasticsearchService.createUser(completeUserData); + + this.logger.log(`Created new user document for ${userId} in Elasticsearch`); + } catch (error) { + this.logger.error(`Failed to create new user ${userId}:`, error); + throw error; + } + } + + /** + * Modular subfunction: Get user profile from database + */ + private async getUserProfileFromDB(userId: string): Promise { + try { + this.logger.log(`Fetching user profile from DB for userId: ${userId}`); + + const user = await this.elasticsearchDataFetcherService['userRepository'].findOne({ + where: { userId } + }); + + if (!user) { + throw new Error(`User ${userId} not found in database`); + } + + const profile = await this.elasticsearchDataFetcherService['fetchUserProfile'](user); + this.logger.log(`Successfully fetched user profile for userId: ${userId}`); + + return profile; + } catch (error) { + this.logger.error(`Failed to get user profile from DB for userId: ${userId}:`, error); + throw error; + } + } + + /** + * Modular subfunction: Get user applications from database + */ + private async getUserApplicationsFromDB(userId: string): Promise { + try { + this.logger.log(`Fetching user applications from DB for userId: ${userId}`); + + const applications = await this.elasticsearchDataFetcherService.fetchUserApplications(userId); + this.logger.log(`Successfully fetched ${applications.length} applications for userId: ${userId}`); + + return applications; + } catch (error) { + this.logger.error(`Failed to get user applications from DB for userId: ${userId}:`, error); + throw error; + } + } + + /** + * Modular subfunction: Get user courses from LMS service + */ + private async getUserCoursesFromLMS(userId: string, tenantId: string, organisationId: string): Promise { + try { + this.logger.log(`Fetching user courses from LMS for userId: ${userId}`); + + const lmsData = await this.elasticsearchDataFetcherService['fetchLessonModuleDataFromLMS']( + userId, tenantId, organisationId + ); + + this.logger.log(`Successfully fetched LMS data for userId: ${userId}`); + return lmsData; + } catch (error) { + this.logger.error(`Failed to get user courses from LMS for userId: ${userId}:`, error); + throw error; + } + } + + + /** + * Modular subfunction: Get user answers from Assessment service + */ + private async getUserAnswersFromAssessment(userId: string, tenantId: string, organisationId: string): Promise { + try { + this.logger.log(`Fetching user answers from Assessment for userId: ${userId}`); + + const assessmentData = await this.elasticsearchDataFetcherService['fetchQuestionAnswerDataFromAssessment']( + userId, tenantId, organisationId + ); + + this.logger.log(`Successfully fetched assessment data for userId: ${userId}`); + return assessmentData; + } catch (error) { + this.logger.error(`Failed to get user answers from Assessment for userId: ${userId}:`, error); + throw error; + } + } + + /** + * Update profile section only + */ + private async updateProfileSection(userId: string, currentData: any): Promise { + this.logger.log(`Updating profile section for userId: ${userId}`); + + // Use modular subfunction to get profile data + const profile = await this.getUserProfileFromDB(userId); + + // Use deep merge to preserve existing structure + return this.deepMerge(currentData, { profile }); + } + + /** + * Update applications section only + */ + private async updateApplicationsSection(userId: string, currentData: any): Promise { + this.logger.log(`Updating applications section for userId: ${userId}`); + + // Use modular subfunction to get applications data + const applications = await this.getUserApplicationsFromDB(userId); + + // Use deep merge to preserve existing structure + return this.deepMerge(currentData, { applications }); + } + + /** + * Update courses section only + */ + private async updateCoursesSection(userId: string, currentData: any): Promise { + this.logger.log(`Updating courses section for userId: ${userId}`); + + // Get user's tenant and organisation data + let tenantId = 'default-tenant'; + let organisationId = 'default-organisation'; + + try { + const userTenantMapping = await this.elasticsearchDataFetcherService['cohortMembersRepository'].manager + .getRepository('UserTenantMapping') + .findOne({ where: { userId } }); + + if (userTenantMapping) { + tenantId = userTenantMapping.tenantId || 'default-tenant'; + organisationId = userTenantMapping.organisationId || 'default-organisation'; + } + } catch (error) { + this.logger.warn(`Failed to fetch tenant data for userId: ${userId}, using default values`); + } + + // Use modular subfunction to get courses data + const lmsData = await this.getUserCoursesFromLMS(userId, tenantId, organisationId); + + // Update applications with new courses data + const updatedApplications = currentData.applications || []; + this.elasticsearchDataFetcherService['enhanceCourseDataWithAssessmentData'](updatedApplications, []); + + // Use deep merge to preserve existing structure + return this.deepMerge(currentData, { applications: updatedApplications }); + } + + + /** + * Update assessment section only + */ + private async updateAssessmentSection(userId: string, currentData: any): Promise { + this.logger.log(`Updating assessment section for userId: ${userId}`); + + // Get user's tenant and organisation data + let tenantId = 'default-tenant'; + let organisationId = 'default-organisation'; + + try { + const userTenantMapping = await this.elasticsearchDataFetcherService['cohortMembersRepository'].manager + .getRepository('UserTenantMapping') + .findOne({ where: { userId } }); + + if (userTenantMapping) { + tenantId = userTenantMapping.tenantId || 'default-tenant'; + organisationId = userTenantMapping.organisationId || 'default-organisation'; + } + } catch (error) { + this.logger.warn(`Failed to fetch tenant data for userId: ${userId}, using default values`); + } + + // Use modular subfunction to get assessment data + const assessmentData = await this.getUserAnswersFromAssessment(userId, tenantId, organisationId); + + // Update applications with new assessment data + const updatedApplications = currentData.applications || []; + this.elasticsearchDataFetcherService['enhanceCourseDataWithAssessmentData'](updatedApplications, assessmentData); + + // Use deep merge to preserve existing structure + return this.deepMerge(currentData, { applications: updatedApplications }); + } + + /** + * Update all sections (comprehensive sync) + */ + private async updateAllSections(userId: string, currentData: any, options: SyncOptions): Promise { + this.logger.log(`Updating all sections for userId: ${userId}`); + + // Fetch complete user data + const completeUserData = await this.elasticsearchDataFetcherService.comprehensiveUserSync(userId); + + if (!completeUserData) { + throw new Error(`Failed to fetch complete user data for ${userId}`); + } + + return completeUserData; + } + + /** + * Improved deep merge function for nested structures + * Handles arrays, objects, and primitive values + */ + private deepMerge(existing: any, updates: any): any { + if (!existing) return updates; + if (!updates) return existing; + + // Handle arrays + if (Array.isArray(existing) && Array.isArray(updates)) { + return updates; // Replace arrays completely + } + + // Handle objects + if (typeof existing === 'object' && typeof updates === 'object' && !Array.isArray(existing) && !Array.isArray(updates)) { + const merged = { ...existing }; + + for (const key in updates) { + if (updates.hasOwnProperty(key)) { + if (existing.hasOwnProperty(key) && + typeof existing[key] === 'object' && + typeof updates[key] === 'object' && + !Array.isArray(existing[key]) && + !Array.isArray(updates[key])) { + // Recursively merge nested objects + merged[key] = this.deepMerge(existing[key], updates[key]); + } else { + // Replace or add new values + merged[key] = updates[key]; + } + } + } + + return merged; + } + + // Handle primitives - updates take precedence + return updates; + } + + /** + * Check if specific section exists in current document + */ + private hasSection(currentData: any, section: SyncSection): boolean { + switch (section) { + case SyncSection.PROFILE: + return !!currentData.profile; + case SyncSection.APPLICATIONS: + return !!(currentData.applications && currentData.applications.length > 0); + case SyncSection.COURSES: + return !!(currentData.applications && + currentData.applications.some((app: any) => + app.courses && app.courses.values && app.courses.values.length > 0 + )); + case SyncSection.ASSESSMENT: + return !!(currentData.applications && + currentData.applications.some((app: any) => + app.courses && app.courses.values && + app.courses.values.some((course: any) => + course.units && course.units.values && + course.units.values.some((unit: any) => + unit.contents && unit.contents.values && + unit.contents.values.some((content: any) => + content.tracking && content.tracking.answers + ) + ) + ) + )); + default: + return true; + } + } + + /** + * Get missing sections that need to be fetched + */ + private getMissingSections(currentData: any): SyncSection[] { + const missingSections: SyncSection[] = []; + + if (!this.hasSection(currentData, SyncSection.PROFILE)) { + missingSections.push(SyncSection.PROFILE); + } + + if (!this.hasSection(currentData, SyncSection.APPLICATIONS)) { + missingSections.push(SyncSection.APPLICATIONS); + } + + if (!this.hasSection(currentData, SyncSection.COURSES)) { + missingSections.push(SyncSection.COURSES); + } + + if (!this.hasSection(currentData, SyncSection.ASSESSMENT)) { + missingSections.push(SyncSection.ASSESSMENT); + } + + return missingSections; + } +} \ No newline at end of file diff --git a/src/elasticsearch/elasticsearch.module.ts b/src/elasticsearch/elasticsearch.module.ts index 436affb4..a435ca44 100644 --- a/src/elasticsearch/elasticsearch.module.ts +++ b/src/elasticsearch/elasticsearch.module.ts @@ -1,17 +1,58 @@ // src/elasticsearch/elasticsearch.module.ts import { Module } from '@nestjs/common'; -import { ElasticsearchService } from './elasticsearch.service'; +import { TypeOrmModule } from '@nestjs/typeorm'; +import { ConfigService } from '@nestjs/config'; import { UserElasticsearchService } from './user-elasticsearch.service'; -import { ElasticsearchConfig } from './elasticsearch.config'; +import { ElasticsearchDataFetcherService } from './elasticsearch-data-fetcher.service'; +import { ElasticsearchSyncService } from './elasticsearch-sync.service'; +import { ElasticsearchService } from './elasticsearch.service'; import { UserElasticsearchController } from './user-elasticsearch.controller'; +import { ElasticsearchController } from './controllers/elasticsearch.controller'; +import { User } from '../user/entities/user-entity'; +import { CohortMembers } from '../cohortMembers/entities/cohort-member.entity'; +import { FormSubmission } from '../forms/entities/form-submission.entity'; +import { Form } from '../forms/entities/form.entity'; +import { FieldValues } from '../fields/entities/fields-values.entity'; +import { Fields } from '../fields/entities/fields.entity'; +import { Cohort } from '../cohort/entities/cohort.entity'; +import { PostgresFieldsService } from '../adapters/postgres/fields-adapter'; +import { FormsService } from '../forms/forms.service'; +import { LMSService } from '../common/services/lms.service'; +import { HttpService } from '../common/utils/http-service'; @Module({ - controllers: [UserElasticsearchController], + imports: [ + TypeOrmModule.forFeature([ + User, + CohortMembers, + FormSubmission, + Form, + FieldValues, + Fields, + Cohort, + ]), + ], + controllers: [ + UserElasticsearchController, // Add the controller + ElasticsearchController, // Add the sync controller + ], providers: [ - ElasticsearchConfig, - ElasticsearchService, - UserElasticsearchService + ConfigService, + ElasticsearchService, // Add this missing service + UserElasticsearchService, + ElasticsearchDataFetcherService, + ElasticsearchSyncService, + PostgresFieldsService, + FormsService, + LMSService, + HttpService, + ], + exports: [ + ConfigService, + ElasticsearchService, // Export it as well + UserElasticsearchService, + ElasticsearchDataFetcherService, + ElasticsearchSyncService, ], - exports: [ElasticsearchService, UserElasticsearchService], }) export class ElasticsearchModule {} diff --git a/src/elasticsearch/elasticsearch.service.ts b/src/elasticsearch/elasticsearch.service.ts index 42c22bab..cdd8fd2d 100644 --- a/src/elasticsearch/elasticsearch.service.ts +++ b/src/elasticsearch/elasticsearch.service.ts @@ -204,8 +204,9 @@ export class ElasticsearchService { lastSavedAt: app.lastSavedAt || null, submittedAt: app.submittedAt || null, cohortDetails: app.cohortDetails || {}, + courses: app.courses || null, })) || [], - courses: source?.courses || [], + // Removed root-level courses field as requested createdAt: source?.createdAt || null, updatedAt: source?.updatedAt || null, }, @@ -278,7 +279,7 @@ export class ElasticsearchService { userId, profile: defaultProfile, applications: [application], - courses: [], + // Removed root-level courses field as requested createdAt: new Date().toISOString(), updatedAt: new Date().toISOString(), }, diff --git a/src/elasticsearch/interfaces/user.interface.ts b/src/elasticsearch/interfaces/user.interface.ts index ab875fee..065d9c86 100644 --- a/src/elasticsearch/interfaces/user.interface.ts +++ b/src/elasticsearch/interfaces/user.interface.ts @@ -4,7 +4,7 @@ export interface IApplication { submissionId?: string; cohortmemberstatus?: string; formstatus: string; - completionPercentage?: number; // FIXED: Add completionPercentage from form submission + completionPercentage?: number; lastSavedAt: string; submittedAt: string; formData?: { @@ -13,7 +13,9 @@ export interface IApplication { }; }; cohortDetails: { + cohortId: string; name: string; + type: string; status: string; [key: string]: any; }; @@ -31,6 +33,60 @@ export interface IApplication { completed: number; }; }; + courses?: { //new change courses moved to applications + type: 'nested'; + values: ICourseDetail[]; + }; +} + +export interface ICourseDetail { + courseId: string; + courseTitle: string; + progress: number; + units: { + type: 'nested'; + values: IUnitDetail[]; + }; +} + +export interface IUnitDetail { + unitId: string; + unitTitle: string; + progress: number; + contents: { + type: 'nested'; + values: IContentDetail[]; + }; +} + +export interface IContentDetail { + contentId: string; + type: string; + title: string; + status: string; + tracking: { + percentComplete?: number; + lastPosition?: number; + currentPosition?: number; + timeSpent?: number; + visitedPages?: number[]; + totalPages?: number; + lastPage?: number; + currentPage?: number; + questionsAttempted?: number; + totalQuestions?: number; + score?: number; + answers?: { + type: 'nested'; + values: IAnswerDetail[]; + }; + }; +} + +export interface IAnswerDetail { + questionId: string; + type: string; + submittedAnswer: string | string[]; } export interface ICourse { @@ -82,7 +138,7 @@ export interface IUser { userId: string; profile: IProfile; applications: IApplication[]; - courses: ICourse[]; + courses?: ICourse[]; // Made optional since we're removing root-level courses createdAt: string; updatedAt: string; } diff --git a/src/elasticsearch/services/course-elasticsearch.service.ts b/src/elasticsearch/services/course-elasticsearch.service.ts new file mode 100644 index 00000000..6fefc275 --- /dev/null +++ b/src/elasticsearch/services/course-elasticsearch.service.ts @@ -0,0 +1,508 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository } from 'typeorm'; +import { ElasticsearchService } from '../elasticsearch.service'; +import { User } from '../../user/entities/user-entity'; +import { CohortMembers } from '../../cohortMembers/entities/cohort-member.entity'; + +export interface CourseProgressUpdate { + userId: string; + cohortId: string; + courseId: string; + courseTitle: string; + progress: number; + unitId?: string; + unitTitle?: string; + unitProgress?: number; + contentId?: string; + contentType?: string; + contentTitle?: string; + contentStatus?: string; + tracking?: { + percentComplete?: number; + lastPosition?: number; + currentPosition?: number; + timeSpent?: number; + visitedPages?: number[]; + totalPages?: number; + lastPage?: number; + currentPage?: number; + questionsAttempted?: number; + totalQuestions?: number; + score?: number; + answers?: Array<{ + questionId: string; + type: string; + submittedAnswer: string | string[]; + }>; + }; +} + +export interface QuizAnswerUpdate { + userId: string; + cohortId: string; + courseId: string; + unitId: string; + contentId: string; + attemptId: string; + answers: Array<{ + questionId: string; + type: string; + submittedAnswer: string | string[]; + }>; + score?: number; + questionsAttempted: number; + totalQuestions: number; + percentComplete: number; + timeSpent: number; +} + +@Injectable() +export class CourseElasticsearchService { + private readonly logger = new Logger(CourseElasticsearchService.name); + + constructor( + private readonly elasticsearchService: ElasticsearchService, + @InjectRepository(User) + private readonly userRepository: Repository, + @InjectRepository(CohortMembers) + private readonly cohortMembersRepository: Repository, + ) {} + + /** + * Update course progress in Elasticsearch + * Called when lesson tracking is updated via LMS service + * + * @param progressUpdate - Course progress update data + */ + async updateCourseProgress(progressUpdate: CourseProgressUpdate): Promise { + try { + this.logger.debug(`Updating course progress for user ${progressUpdate.userId}, course ${progressUpdate.courseId}`); + + // Get the user's document from Elasticsearch + const userDoc = await this.elasticsearchService.get('users', progressUpdate.userId); + + if (!userDoc) { + this.logger.warn(`User document not found in Elasticsearch: ${progressUpdate.userId}`); + return; + } + + const sourceData = userDoc._source as any; + const applications = sourceData.applications || []; + + // Find the relevant application (user + cohort combination) + const appIndex = applications.findIndex((app: any) => + app.cohortId === progressUpdate.cohortId + ); + + if (appIndex === -1) { + this.logger.warn(`Application not found for user ${progressUpdate.userId}, cohort ${progressUpdate.cohortId}`); + return; + } + + // Initialize courses structure if it doesn't exist + if (!applications[appIndex].courses) { + applications[appIndex].courses = { + type: 'nested', + values: [] + }; + } + + // Update or create course data + this.updateCourseData(applications[appIndex].courses.values, progressUpdate); + + // Update the document in Elasticsearch + await this.elasticsearchService.update('users', progressUpdate.userId, { + applications: applications, + updatedAt: new Date().toISOString() + }); + + this.logger.log(`Successfully updated course progress for user ${progressUpdate.userId}`); + + } catch (error) { + this.logger.error(`Failed to update course progress for user ${progressUpdate.userId}:`, error); + throw error; + } + } + + /** + * Update quiz answers in Elasticsearch + * Called when quiz answers are submitted via LMS service + * + * @param quizUpdate - Quiz answer update data + */ + async updateQuizAnswers(quizUpdate: QuizAnswerUpdate): Promise { + try { + this.logger.debug(`Updating quiz answers for user ${quizUpdate.userId}, course ${quizUpdate.courseId}`); + + // Get the user's document from Elasticsearch + const userDoc = await this.elasticsearchService.get('users', quizUpdate.userId); + + if (!userDoc) { + this.logger.warn(`User document not found in Elasticsearch: ${quizUpdate.userId}`); + return; + } + + const sourceData = userDoc._source as any; + const applications = sourceData.applications || []; + + // Find the relevant application + // First try to find by exact cohortId match + let appIndex = applications.findIndex((app: any) => + app.cohortId === quizUpdate.cohortId + ); + + // If not found, try to find by testId in the course structure + if (appIndex === -1) { + this.logger.debug(`No exact cohortId match for ${quizUpdate.cohortId}, searching in course structure`); + + for (let i = 0; i < applications.length; i++) { + const app = applications[i]; + if (app.courses && app.courses.values) { + for (const course of app.courses.values) { + if (course.units && course.units.values) { + for (const unit of course.units.values) { + if (unit.contents && unit.contents.values) { + for (const content of unit.contents.values) { + // Check if this content matches the testId + if (content.contentId === quizUpdate.contentId || + content.contentId === quizUpdate.courseId || + content.contentId === quizUpdate.unitId) { + appIndex = i; + this.logger.debug(`Found matching content in application ${i}, cohortId: ${app.cohortId}`); + break; + } + } + if (appIndex !== -1) break; + } + } + if (appIndex !== -1) break; + } + } + if (appIndex !== -1) break; + } + } + } + + if (appIndex === -1) { + this.logger.warn(`Application not found for user ${quizUpdate.userId}, cohort ${quizUpdate.cohortId}. Available cohorts: ${applications.map((app: any) => app.cohortId).join(', ')}`); + return; + } + + // Initialize courses structure if it doesn't exist + if (!applications[appIndex].courses) { + applications[appIndex].courses = { + type: 'nested', + values: [] + }; + } + + // Update quiz data in the course structure + this.updateQuizData(applications[appIndex].courses.values, quizUpdate); + + // Update the document in Elasticsearch + await this.elasticsearchService.update('users', quizUpdate.userId, { + applications: applications, + updatedAt: new Date().toISOString() + }); + + this.logger.log(`Successfully updated quiz answers for user ${quizUpdate.userId}`); + + } catch (error) { + this.logger.error(`Failed to update quiz answers for user ${quizUpdate.userId}:`, error); + throw error; + } + } + + /** + * Update course data structure with new progress information + * + * @param courses - Existing courses array + * @param progressUpdate - Progress update data + */ + private updateCourseData(courses: any[], progressUpdate: CourseProgressUpdate): void { + // Find existing course or create new one + let course = courses.find(c => c.courseId === progressUpdate.courseId); + + if (!course) { + course = { + courseId: progressUpdate.courseId, + courseTitle: progressUpdate.courseTitle, + progress: 0, + units: { + type: 'nested', + values: [] + } + }; + courses.push(course); + } + + // Update course-level progress + course.progress = progressUpdate.progress; + course.courseTitle = progressUpdate.courseTitle; + + // If unit-specific update + if (progressUpdate.unitId) { + this.updateUnitData(course.units.values, progressUpdate); + } + } + + /** + * Update unit data structure with new progress information + * + * @param units - Existing units array + * @param progressUpdate - Progress update data + */ + private updateUnitData(units: any[], progressUpdate: CourseProgressUpdate): void { + if (!progressUpdate.unitId) return; + + // Find existing unit or create new one + let unit = units.find(u => u.unitId === progressUpdate.unitId); + + if (!unit) { + unit = { + unitId: progressUpdate.unitId, + unitTitle: progressUpdate.unitTitle || 'Untitled Unit', + progress: 0, + contents: { + type: 'nested', + values: [] + } + }; + units.push(unit); + } + + // Update unit-level progress + if (progressUpdate.unitProgress !== undefined) { + unit.progress = progressUpdate.unitProgress; + } + unit.unitTitle = progressUpdate.unitTitle || unit.unitTitle; + + // If content-specific update + if (progressUpdate.contentId) { + this.updateContentData(unit.contents.values, progressUpdate); + } + } + + /** + * Update content data structure with new progress information + * + * @param contents - Existing contents array + * @param progressUpdate - Progress update data + */ + private updateContentData(contents: any[], progressUpdate: CourseProgressUpdate): void { + if (!progressUpdate.contentId) return; + + // Find existing content or create new one + let content = contents.find(c => c.contentId === progressUpdate.contentId); + + if (!content) { + content = { + contentId: progressUpdate.contentId, + type: progressUpdate.contentType || 'unknown', + title: progressUpdate.contentTitle || 'Untitled Content', + status: 'not_started', + tracking: {} + }; + contents.push(content); + } + + // Update content data + content.type = progressUpdate.contentType || content.type; + content.title = progressUpdate.contentTitle || content.title; + content.status = progressUpdate.contentStatus || content.status; + + // Update tracking data + if (progressUpdate.tracking) { + content.tracking = { + ...content.tracking, + ...progressUpdate.tracking + }; + } + } + + /** + * Update quiz data in the course structure + * + * @param courses - Existing courses array + * @param quizUpdate - Quiz update data + */ + private updateQuizData(courses: any[], quizUpdate: QuizAnswerUpdate): void { + // Find the course + let course = courses.find(c => c.courseId === quizUpdate.courseId); + if (!course) { + this.logger.debug(`Course ${quizUpdate.courseId} not found, creating new course structure`); + course = { + courseId: quizUpdate.courseId, + courseTitle: `Assessment ${quizUpdate.courseId}`, + progress: 0, + units: { + type: 'nested', + values: [] + } + }; + courses.push(course); + } + + // Find the unit + let unit = course.units.values.find((u: any) => u.unitId === quizUpdate.unitId); + if (!unit) { + this.logger.debug(`Unit ${quizUpdate.unitId} not found, creating new unit structure`); + unit = { + unitId: quizUpdate.unitId, + unitTitle: `Assessment Unit ${quizUpdate.unitId}`, + progress: 0, + contents: { + type: 'nested', + values: [] + } + }; + course.units.values.push(unit); + } + + // Find the content (assessment) + let content = unit.contents.values.find((c: any) => c.contentId === quizUpdate.contentId); + if (!content) { + this.logger.debug(`Content ${quizUpdate.contentId} not found, creating new content structure`); + content = { + contentId: quizUpdate.contentId, + type: 'test', + title: `Assessment ${quizUpdate.contentId}`, + status: 'incomplete', + tracking: { + timeSpent: 0, + currentPosition: 0, + lastPosition: 0, + percentComplete: 0, + questionsAttempted: 0, + totalQuestions: 0, + score: 0, + answers: { + type: 'nested', + values: [] + } + } + }; + unit.contents.values.push(content); + } + + // Update quiz tracking data + content.tracking = { + ...content.tracking, + questionsAttempted: quizUpdate.questionsAttempted, + totalQuestions: quizUpdate.totalQuestions, + score: quizUpdate.score || 0, + percentComplete: quizUpdate.percentComplete, + timeSpent: quizUpdate.timeSpent, + answers: { + type: 'nested', + values: quizUpdate.answers || [] + } + }; + + // Update content status based on completion + if (quizUpdate.percentComplete >= 100) { + content.status = 'completed'; + } else if (quizUpdate.percentComplete > 0) { + content.status = 'in_progress'; + } + + this.logger.debug(`Successfully updated quiz data for content ${quizUpdate.contentId} with ${quizUpdate.answers?.length || 0} answers`); + } + + /** + * Initialize course structure for a user's application + * Called when a user is enrolled in a course + * + * @param userId - User ID + * @param cohortId - Cohort ID + * @param courseData - Initial course data from LMS + */ + async initializeCourseStructure( + userId: string, + cohortId: string, + courseData: any + ): Promise { + try { + this.logger.debug(`Initializing course structure for user ${userId}, cohort ${cohortId}`); + + // Get the user's document from Elasticsearch + const userDoc = await this.elasticsearchService.get('users', userId); + + if (!userDoc) { + this.logger.warn(`User document not found in Elasticsearch: ${userId}`); + return; + } + + const sourceData = userDoc._source as any; + const applications = sourceData.applications || []; + + // Find the relevant application + const appIndex = applications.findIndex((app: any) => + app.cohortId === cohortId + ); + + if (appIndex === -1) { + this.logger.warn(`Application not found for user ${userId}, cohort ${cohortId}`); + return; + } + + // Initialize courses structure + applications[appIndex].courses = { + type: 'nested', + values: this.transformLMSCourseData(courseData) + }; + + // Update the document in Elasticsearch + await this.elasticsearchService.update('users', userId, { + applications: applications, + updatedAt: new Date().toISOString() + }); + + this.logger.log(`Successfully initialized course structure for user ${userId}`); + + } catch (error) { + this.logger.error(`Failed to initialize course structure for user ${userId}:`, error); + throw error; + } + } + + /** + * Transform LMS course data to Elasticsearch format + * + * @param courseData - Raw course data from LMS + * @returns Transformed course data + */ + private transformLMSCourseData(courseData: any): any[] { + if (!Array.isArray(courseData)) { + return []; + } + + return courseData.map(course => ({ + courseId: course.id || course.courseId, + courseTitle: course.title || course.name || 'Untitled Course', + progress: 0, // Initialize with 0 progress + units: { + type: 'nested', + values: (course.units || []).map((unit: any) => ({ + unitId: unit.id || unit.unitId, + unitTitle: unit.title || unit.name || 'Untitled Unit', + progress: 0, + contents: { + type: 'nested', + values: (unit.contents || []).map((content: any) => ({ + contentId: content.id || content.contentId, + type: content.type || 'unknown', + title: content.title || content.name || 'Untitled Content', + status: 'not_started', + tracking: { + percentComplete: 0, + timeSpent: 0 + } + })) + } + })) + } + })); + } +} \ No newline at end of file diff --git a/src/elasticsearch/user-elasticsearch.controller.ts b/src/elasticsearch/user-elasticsearch.controller.ts index 6ff5e7a9..11c26354 100644 --- a/src/elasticsearch/user-elasticsearch.controller.ts +++ b/src/elasticsearch/user-elasticsearch.controller.ts @@ -41,11 +41,10 @@ export class UserElasticsearchController { lastSavedAt: application.lastSavedAt || new Date().toISOString(), submittedAt: application.submittedAt || new Date().toISOString(), cohortDetails: { + cohortId: application.cohortDetails?.cohortId || cohortId, name: application.cohortDetails?.name || '', - description: application.cohortDetails?.description ?? '', - startDate: application.cohortDetails?.startDate ?? '', - endDate: application.cohortDetails?.endDate ?? '', - status: application.cohortDetails?.status || '', + type: application.cohortDetails?.type || 'COHORT', + status: application.cohortDetails?.status || 'active', }, progress: application.progress || { pages: {}, diff --git a/src/elasticsearch/user-elasticsearch.service.ts b/src/elasticsearch/user-elasticsearch.service.ts index 0362cc79..2dad93b5 100644 --- a/src/elasticsearch/user-elasticsearch.service.ts +++ b/src/elasticsearch/user-elasticsearch.service.ts @@ -84,9 +84,11 @@ export class UserElasticsearchService implements OnModuleInit { type: 'nested', properties: { cohortId: { type: 'keyword' }, + formId: { type: 'keyword' }, + submissionId: { type: 'keyword' }, cohortmemberstatus: { type: 'keyword' }, formstatus: { type: 'keyword' }, - completionPercentage: { type: 'float' }, // FIXED: Add completionPercentage field + completionPercentage: { type: 'float' }, progress: { properties: { pages: { @@ -108,32 +110,72 @@ export class UserElasticsearchService implements OnModuleInit { submittedAt: { type: 'date', null_value: null }, cohortDetails: { properties: { + cohortId: { type: 'keyword' }, name: { type: 'text' }, - description: { type: 'text' }, - startDate: { type: 'date', null_value: null }, - endDate: { type: 'date', null_value: null }, + type: { type: 'keyword' }, status: { type: 'keyword' }, }, }, - }, - }, - courses: { - type: 'nested', - properties: { - courseId: { type: 'keyword' }, - progress: { type: 'float' }, - lessonsCompleted: { type: 'keyword' }, - lastLessonAt: { type: 'date', null_value: null }, - courseDetails: { + courses: { + type: 'nested', properties: { - name: { type: 'text' }, - description: { type: 'text' }, - duration: { type: 'integer' }, - status: { type: 'keyword' }, + courseId: { type: 'keyword' }, + courseTitle: { type: 'text' }, + progress: { type: 'float' }, + units: { + type: 'nested', + properties: { + unitId: { type: 'keyword' }, + unitTitle: { type: 'text' }, + progress: { type: 'float' }, + contents: { + type: 'nested', + properties: { + contentId: { type: 'keyword' }, + type: { type: 'keyword' }, + title: { type: 'text' }, + status: { type: 'keyword' }, + tracking: { + properties: { + percentComplete: { type: 'float' }, + lastPosition: { type: 'float' }, + currentPosition: { type: 'float' }, + timeSpent: { type: 'integer' }, + visitedPages: { type: 'integer' }, + totalPages: { type: 'integer' }, + lastPage: { type: 'integer' }, + currentPage: { type: 'integer' }, + questionsAttempted: { type: 'integer' }, + totalQuestions: { type: 'integer' }, + score: { type: 'float' }, + answers: { + type: 'nested', + properties: { + questionId: { type: 'keyword' }, + type: { type: 'keyword' }, + submittedAnswer: { type: 'text' }, + answer: { + properties: { + answer: { type: 'keyword' }, + text: { type: 'text' }, + }, + }, + text: { type: 'text' }, + score: { type: 'float' }, + reviewStatus: { type: 'keyword' }, + }, + }, + }, + }, + }, + }, + }, + }, }, }, }, }, + // Removed root-level courses mapping as requested - courses now only in applications createdAt: { type: 'date', null_value: null }, updatedAt: { type: 'date', null_value: null }, }, @@ -188,7 +230,7 @@ export class UserElasticsearchService implements OnModuleInit { customFields: user.profile.customFields || {}, }, applications: user.applications || [], - courses: user.courses || [], + // Removed root-level courses field as requested createdAt: user.createdAt, updatedAt: user.updatedAt, }; @@ -352,6 +394,16 @@ export class UserElasticsearchService implements OnModuleInit { filter: [], }, }; + + // Add userId filter if provided + if (query.userId) { + searchQuery.bool.filter.push({ + term: { + userId: query.userId, + }, + }); + } + // Add text search with partial matching support if (query.q) { if (typeof query.q !== 'string') { @@ -491,10 +543,21 @@ export class UserElasticsearchService implements OnModuleInit { } } if (query.filters && typeof query.filters === 'object') { - // Special handling for cohortId and cohortmemberstatus in applications + // Special handling for cohortId, cohortmemberstatus, completionPercentage, and courses in applications const appFilters: any = {}; + const courseFilters: any = {}; Object.entries(query.filters).forEach(([field, value]) => { if (value !== undefined && value !== null && value !== '') { + // Handle userId filter + if (field === 'userId') { + searchQuery.bool.filter.push({ + term: { + userId: value, + }, + }); + return; + } + // Handle cohortId, cohortmemberstatus, and completionPercentage as nested application filters if ( field === 'cohortId' || @@ -505,6 +568,13 @@ export class UserElasticsearchService implements OnModuleInit { return; } + // Handle courses fields as nested course filters + if (field.startsWith('courses.')) { + const courseField = field.replace('courses.', ''); + courseFilters[courseField] = value; + return; + } + // Handle custom fields filtering if (field.startsWith('customFields.')) { const customFieldName = field.replace('customFields.', ''); @@ -717,6 +787,323 @@ export class UserElasticsearchService implements OnModuleInit { }, }); } + + // Handle course filters if present + if (Object.keys(courseFilters).length > 0) { + const courseMust: any[] = []; + + Object.entries(courseFilters).forEach(([field, value]) => { + if (value !== undefined && value !== null && value !== '') { + // Handle different course field types + if (field === 'courseId') { + courseMust.push({ + term: { + 'applications.courses.courseId': value, + }, + }); + } else if (field === 'courseTitle') { + courseMust.push({ + wildcard: { + 'applications.courses.courseTitle': `*${String( + value + ).toLowerCase()}*`, + }, + }); + } else if (field === 'progress') { + const progressValue = Number(value); + if (!isNaN(progressValue)) { + courseMust.push({ + range: { + 'applications.courses.progress': { + gte: progressValue, + lte: progressValue, + }, + }, + }); + } + } else if (field.startsWith('units.')) { + // Handle nested unit fields + const unitField = field.replace('units.', ''); + if (unitField === 'unitId') { + courseMust.push({ + nested: { + path: 'applications.courses.units', + query: { + term: { + 'applications.courses.units.unitId': value, + }, + }, + }, + }); + } else if (unitField === 'unitTitle') { + courseMust.push({ + nested: { + path: 'applications.courses.units', + query: { + wildcard: { + 'applications.courses.units.unitTitle': `*${String( + value + ).toLowerCase()}*`, + }, + }, + }, + }); + } else if (unitField === 'progress') { + const unitProgressValue = Number(value); + if (!isNaN(unitProgressValue)) { + courseMust.push({ + nested: { + path: 'applications.courses.units', + query: { + range: { + 'applications.courses.units.progress': { + gte: unitProgressValue, + lte: unitProgressValue, + }, + }, + }, + }, + }); + } + } else if (unitField.startsWith('contents.')) { + // Handle nested content fields within units + const contentField = unitField.replace('contents.', ''); + if (contentField === 'contentId') { + courseMust.push({ + nested: { + path: 'applications.courses.units.contents', + query: { + term: { + 'applications.courses.units.contents.contentId': + value, + }, + }, + }, + }); + } else if (contentField === 'lessonId') { + courseMust.push({ + nested: { + path: 'applications.courses.units.contents', + query: { + term: { + 'applications.courses.units.contents.lessonId': + value, + }, + }, + }, + }); + } else if (contentField === 'title') { + courseMust.push({ + nested: { + path: 'applications.courses.units.contents', + query: { + wildcard: { + 'applications.courses.units.contents.title': `*${String( + value + ).toLowerCase()}*`, + }, + }, + }, + }); + } else if (contentField === 'type') { + courseMust.push({ + nested: { + path: 'applications.courses.units.contents', + query: { + term: { + 'applications.courses.units.contents.type': value, + }, + }, + }, + }); + } else if (contentField === 'status') { + courseMust.push({ + nested: { + path: 'applications.courses.units.contents', + query: { + term: { + 'applications.courses.units.contents.status': value, + }, + }, + }, + }); + } else if (contentField.startsWith('tracking.')) { + // Handle nested tracking fields within contents + const trackingField = contentField.replace('tracking.', ''); + if (trackingField === 'percentComplete') { + const trackingValue = Number(value); + if (!isNaN(trackingValue)) { + courseMust.push({ + nested: { + path: 'applications.courses.units.contents', + query: { + range: { + 'applications.courses.units.contents.tracking.percentComplete': + { + gte: trackingValue, + lte: trackingValue, + }, + }, + }, + }, + }); + } + } else if (trackingField === 'score') { + const trackingValue = Number(value); + if (!isNaN(trackingValue)) { + courseMust.push({ + nested: { + path: 'applications.courses.units.contents', + query: { + range: { + 'applications.courses.units.contents.tracking.score': + { + gte: trackingValue, + lte: trackingValue, + }, + }, + }, + }, + }); + } + } else if (trackingField === 'timeSpent') { + const trackingValue = Number(value); + if (!isNaN(trackingValue)) { + courseMust.push({ + nested: { + path: 'applications.courses.units.contents', + query: { + range: { + 'applications.courses.units.contents.tracking.timeSpent': + { + gte: trackingValue, + lte: trackingValue, + }, + }, + }, + }, + }); + } + } else if (trackingField === 'questionsAttempted') { + const trackingValue = Number(value); + if (!isNaN(trackingValue)) { + courseMust.push({ + nested: { + path: 'applications.courses.units.contents', + query: { + range: { + 'applications.courses.units.contents.tracking.questionsAttempted': + { + gte: trackingValue, + lte: trackingValue, + }, + }, + }, + }, + }); + } + } else if (trackingField === 'totalQuestions') { + const trackingValue = Number(value); + if (!isNaN(trackingValue)) { + courseMust.push({ + nested: { + path: 'applications.courses.units.contents', + query: { + range: { + 'applications.courses.units.contents.tracking.totalQuestions': + { + gte: trackingValue, + lte: trackingValue, + }, + }, + }, + }, + }); + } + } else if (trackingField.startsWith('answers.')) { + // Handle nested answers fields within tracking + const answerField = trackingField.replace('answers.', ''); + if (answerField === 'questionId') { + courseMust.push({ + nested: { + path: 'applications.courses.units.contents.tracking.answers', + query: { + term: { + 'applications.courses.units.contents.tracking.answers.questionId': + value, + }, + }, + }, + }); + } else if (answerField === 'answer') { + courseMust.push({ + nested: { + path: 'applications.courses.units.contents.tracking.answers', + query: { + wildcard: { + 'applications.courses.units.contents.tracking.answers.answer': `*${String( + value + ).toLowerCase()}*`, + }, + }, + }, + }); + } else if (answerField === 'score') { + const answerScoreValue = Number(value); + if (!isNaN(answerScoreValue)) { + courseMust.push({ + nested: { + path: 'applications.courses.units.contents.tracking.answers', + query: { + range: { + 'applications.courses.units.contents.tracking.answers.score': + { + gte: answerScoreValue, + lte: answerScoreValue, + }, + }, + }, + }, + }); + } + } else if (answerField === 'reviewStatus') { + courseMust.push({ + nested: { + path: 'applications.courses.units.contents.tracking.answers', + query: { + term: { + 'applications.courses.units.contents.tracking.answers.reviewStatus': + value, + }, + }, + }, + }); + } + } + } + } + } else { + // Default to wildcard for other course fields + courseMust.push({ + wildcard: { + [`applications.courses.${field}`]: `*${String( + value + ).toLowerCase()}*`, + }, + }); + } + } + }); + + if (courseMust.length > 0) { + searchQuery.bool.filter.push({ + nested: { + path: 'applications.courses', + query: { bool: { must: courseMust } }, + }, + }); + } + } } if (query.cohortId && typeof query.cohortId === 'string') { searchQuery.bool.filter.push({ @@ -853,10 +1240,17 @@ export class UserElasticsearchService implements OnModuleInit { lastSavedAt: application.lastSavedAt || new Date().toISOString(), submittedAt: application.submittedAt || new Date().toISOString(), cohortDetails: application.cohortDetails || { + cohortId: '', name: '', + type: 'COHORT', status: 'active', }, formData: {}, + // Preserve existing courses data or initialize empty courses structure + courses: application.courses || { + type: 'nested', + values: [], + }, }; // If application has formData, map it to pages structure @@ -992,37 +1386,14 @@ export class UserElasticsearchService implements OnModuleInit { course: Partial ): Promise { try { - const script = { - source: ` - if (ctx._source.courses == null) { - ctx._source.courses = []; - } - boolean found = false; - for (int i = 0; i < ctx._source.courses.length; i++) { - if (ctx._source.courses[i].courseId == params.courseId) { - ctx._source.courses[i] = params.course; - found = true; - break; - } - } - if (!found) { - ctx._source.courses.add(params.course); - } - `, - lang: 'painless', - params: { - courseId, - course, - }, - }; - - const result = await this.elasticsearchService.update( - this.indexName, - userId, - { script }, - { retry_on_conflict: 3 } + // For now, courses will be null as per requirements + // This method is kept for backward compatibility + this.logger.log( + `Course update requested for user ${userId}, course ${courseId} - currently disabled as courses are null` ); - return result; + + // Return success without actually updating anything + return { acknowledged: true }; } catch (error) { this.logger.error('Error updating course in Elasticsearch:', error); throw new Error( @@ -1031,6 +1402,39 @@ export class UserElasticsearchService implements OnModuleInit { } } + async updateApplicationCourse( + userId: string, + cohortId: string, + courseData: { + courseId: string; + courseTitle: string; + progress: number; + units?: { + type: 'nested'; + values: any[]; + }; + } + ): Promise { + try { + // For now, courses will be null as per requirements + // This method is prepared for future implementation + this.logger.log( + `Application course update requested for user ${userId}, cohort ${cohortId}, course ${courseData.courseId} - currently disabled as courses are null` + ); + + // Return success without actually updating anything + return { acknowledged: true }; + } catch (error) { + this.logger.error( + 'Error updating application course in Elasticsearch:', + error + ); + throw new Error( + `Failed to update application course in Elasticsearch: ${error.message}` + ); + } + } + async updateApplicationPage( userId: string, cohortId: string, diff --git a/src/forms/services/form-submission.service.ts b/src/forms/services/form-submission.service.ts index 4417db5d..a075c74f 100644 --- a/src/forms/services/form-submission.service.ts +++ b/src/forms/services/form-submission.service.ts @@ -29,6 +29,7 @@ import { FieldsSearchDto } from '../../fields/dto/fields-search.dto'; import jwt_decode from 'jwt-decode'; import { Form } from '../entities/form.entity'; import { UserElasticsearchService } from '../../elasticsearch/user-elasticsearch.service'; +import { ElasticsearchDataFetcherService } from '../../elasticsearch/elasticsearch-data-fetcher.service'; import { FormsService } from '../../forms/forms.service'; import { PostgresCohortService } from 'src/adapters/postgres/cohort-adapter'; import { IUser } from '../../elasticsearch/interfaces/user.interface'; @@ -36,6 +37,10 @@ import { LoggerUtil } from 'src/common/logger/LoggerUtil'; import { isElasticsearchEnabled } from 'src/common/utils/elasticsearch.util'; import { CohortMembers } from 'src/cohortMembers/entities/cohort-member.entity'; import { Cohort } from 'src/cohort/entities/cohort.entity'; +import { + ElasticsearchSyncService, + SyncSection, +} from '../../elasticsearch/elasticsearch-sync.service'; interface DateRange { start: string; @@ -76,6 +81,8 @@ interface FieldSearchResponse { @Injectable() export class FormSubmissionService { + private readonly logger = new Logger(FormSubmissionService.name); + constructor( @InjectRepository(FormSubmission) private formSubmissionRepository: Repository, @@ -89,7 +96,9 @@ export class FormSubmissionService { private cohortRepository: Repository, private readonly fieldsService: FieldsService, private readonly userElasticsearchService: UserElasticsearchService, + private readonly elasticsearchDataFetcherService: ElasticsearchDataFetcherService, private readonly formsService: FormsService, + private readonly elasticsearchSyncService: ElasticsearchSyncService, @Inject(forwardRef(() => PostgresCohortService)) private readonly postgresCohortService: PostgresCohortService ) {} @@ -195,14 +204,11 @@ export class FormSubmissionService { createFormSubmissionDto.formSubmission.formId // Pass formId for checkbox processing ); - // Update Elasticsearch with complete field values from database (includes dependencies) - // Only update if Elasticsearch is enabled + // Update Elasticsearch using centralized service if (isElasticsearchEnabled()) { - await this.updateApplicationInElasticsearch( - userId, - savedSubmission, - customFields // Fixed: now passes all fields from DB - ); + await this.elasticsearchSyncService.syncUserToElasticsearch(userId, { + section: SyncSection.APPLICATIONS, + }); } // Create response object @@ -357,7 +363,7 @@ export class FormSubmissionService { Array.isArray(completionPercentage) && completionPercentage.length > 0 ) { - // Multiple ranges - use OR conditions (union) with proper casting + // Multiple ranges - use OR conditions (union) with proper numeric casting const rangeConditions = completionPercentage.map((range, index) => { const [min, max] = range.split('-').map(Number); return `(CAST(fs.completionPercentage AS DECIMAL(5,2)) >= :min${index} AND CAST(fs.completionPercentage AS DECIMAL(5,2)) <= :max${index})`; @@ -1000,14 +1006,11 @@ export class FormSubmissionService { updatedSubmission.formId // Pass formId for checkbox processing ); - // Update Elasticsearch after successful form submission update - // Only update if Elasticsearch is enabled + // Update Elasticsearch using centralized service if (isElasticsearchEnabled()) { - await this.updateApplicationInElasticsearch( - userId, - updatedSubmission, - completeFieldValues // Fixed: now passes all fields from DB - ); + await this.elasticsearchSyncService.syncUserToElasticsearch(userId, { + section: SyncSection.APPLICATIONS, + }); } const successResponse = { id: 'api.form.submission.update', @@ -1159,9 +1162,9 @@ export class FormSubmissionService { } /** - * Update the user's applications array in Elasticsearch after a form submission update. - * This will upsert (update or create) the user document in Elasticsearch if missing. - * If the document is missing, it will fetch the user from the database and create it. + * Update application in Elasticsearch using centralized service + * This method is now deprecated in favor of the centralized service. + * @deprecated Use elasticsearchSyncService.syncUserToElasticsearch instead */ private async updateApplicationInElasticsearch( userId: string, @@ -1169,338 +1172,12 @@ export class FormSubmissionService { updatedFieldValues: any[] ): Promise { try { - // Get the existing user document from Elasticsearch - const userDoc = await this.userElasticsearchService.getUser(userId); - - // Prepare the applications array (existing or new) - let applications: any[] = []; - if (userDoc && userDoc._source) { - const userSource = userDoc._source as IUser; - applications = userSource.applications || []; - } - - // Use the actual submissionId and formId from the updatedSubmission - const formIdToMatch = updatedSubmission.formId; - const submissionIdToMatch = updatedSubmission.submissionId; - - // --- NEW LOGIC: Use the robust getFieldIdToPageNameMap utility function --- - let fieldIdToPageName: Record = {}; - let fieldIdToFieldName: Record = {}; - let formFieldsOnly: any[] = []; - try { - const form = await this.formsService.getFormById(formIdToMatch); - const fieldsObj = form && form.fields ? (form.fields as any) : null; - - // Handle different schema structures - let schema: any = {}; - if (fieldsObj) { - // Try different possible schema structures - if ( - Array.isArray(fieldsObj?.result) && - fieldsObj.result[0]?.schema?.properties - ) { - // Structure: { result: [{ schema: { properties: {...} } }] } - schema = fieldsObj.result[0].schema.properties; - } else if (fieldsObj?.schema?.properties) { - // Structure: { schema: { properties: {...} } } - schema = fieldsObj.schema.properties; - } else if (fieldsObj?.properties) { - // Structure: { properties: {...} } - schema = fieldsObj.properties; - } else if (typeof fieldsObj === 'object' && fieldsObj !== null) { - // Try to find schema in nested structure - const findSchema = (obj: any): any => { - if (obj?.schema?.properties) return obj.schema.properties; - if (obj?.properties) return obj.properties; - if (Array.isArray(obj)) { - for (const item of obj) { - const found = findSchema(item); - if (found) return found; - } - } else if (typeof obj === 'object') { - for (const key in obj) { - const found = findSchema(obj[key]); - if (found) return found; - } - } - return null; - }; - schema = findSchema(fieldsObj) || {}; - } - } - - // Use the robust utility function to get field mappings - fieldIdToPageName = getFieldIdToPageNameMap(schema); - - // Also build fieldIdToFieldName mapping for logging - for (const [pageKey, pageSchema] of Object.entries(schema)) { - const pageName = pageKey === 'default' ? 'eligibilityCheck' : pageKey; - const fieldProps = (pageSchema as any).properties || {}; - - const extractFieldNames = (properties: any) => { - for (const [fieldKey, fieldSchema] of Object.entries(properties)) { - const fieldId = (fieldSchema as any).fieldId; - const fieldTitle = (fieldSchema as any).title || fieldKey; - if (fieldId) { - fieldIdToFieldName[fieldId] = fieldTitle; - } - - // Handle dependencies - if ((fieldSchema as any).dependencies) { - const dependencies = (fieldSchema as any).dependencies; - for (const depSchema of Object.values(dependencies)) { - if (!depSchema || typeof depSchema !== 'object') continue; - const dep = depSchema as any; - if (dep.oneOf) - dep.oneOf.forEach( - (item: any) => - item?.properties && extractFieldNames(item.properties) - ); - if (dep.allOf) - dep.allOf.forEach( - (item: any) => - item?.properties && extractFieldNames(item.properties) - ); - if (dep.anyOf) - dep.anyOf.forEach( - (item: any) => - item?.properties && extractFieldNames(item.properties) - ); - if (dep.properties) extractFieldNames(dep.properties); - } - } - } - }; - - extractFieldNames(fieldProps); - - // Handle page-level dependencies - const pageDependencies = (pageSchema as any).dependencies || {}; - for (const depSchema of Object.values(pageDependencies)) { - if (!depSchema || typeof depSchema !== 'object') continue; - const dep = depSchema as any; - if (dep.oneOf) - dep.oneOf.forEach( - (item: any) => - item?.properties && extractFieldNames(item.properties) - ); - if (dep.allOf) - dep.allOf.forEach( - (item: any) => - item?.properties && extractFieldNames(item.properties) - ); - if (dep.anyOf) - dep.anyOf.forEach( - (item: any) => - item?.properties && extractFieldNames(item.properties) - ); - if (dep.properties) extractFieldNames(dep.properties); - } - } - - for (const [fieldId, pageName] of Object.entries(fieldIdToPageName)) { - const fieldName = fieldIdToFieldName[fieldId] || fieldId; - } - - // Filter updatedFieldValues to only include form fields (not custom fields) - const formFieldIds = new Set(); - for (const [pageKey, pageSchema] of Object.entries(schema)) { - const fieldProps = (pageSchema as any).properties || {}; - - const extractFormFieldIds = (properties: any) => { - for (const [fieldKey, fieldSchema] of Object.entries(properties)) { - const fieldId = (fieldSchema as any).fieldId; - if (fieldId) { - formFieldIds.add(fieldId); - } - - // Handle nested dependencies structure - ensure dependency fields are included - if ((fieldSchema as any).dependencies) { - const dependencies = (fieldSchema as any).dependencies; - // Check if dependencies is an object - for (const [depKey, depSchema] of Object.entries( - dependencies - )) { - if ((depSchema as any).oneOf) { - // Handle oneOf dependencies - for (const oneOfItem of (depSchema as any).oneOf) { - if (oneOfItem.properties) { - extractFormFieldIds(oneOfItem.properties); - } - } - } else if ((depSchema as any).properties) { - // Handle direct properties dependencies - extractFormFieldIds((depSchema as any).properties); - } - } - } - } - }; - - extractFormFieldIds(fieldProps); - } - // Filter updatedFieldValues to only include form fields - // TEMPORARY FIX: If a field is not in formFieldIds but exists in fieldIdToPageName, include it - formFieldsOnly = updatedFieldValues.filter((field) => { - const isInFormFieldIds = formFieldIds.has(field.fieldId); - const isInFieldMapping = fieldIdToPageName[field.fieldId]; - - return isInFormFieldIds || isInFieldMapping; - }); - } catch (err) { - const logger = new Logger('FormSubmissionService'); - logger.error('Schema fetch failed, cannot proceed', err); - // If schema fetch fails, fallback to empty map (all fields go to 'default') - fieldIdToPageName = {}; - fieldIdToFieldName = {}; - // If schema fetch fails, don't filter fields (use all updatedFieldValues) - formFieldsOnly = updatedFieldValues; - } - // --- END NEW LOGIC --- - - // Always fetch cohortId from the related Form entity - // Fetch cohortId from the related Form entity with proper error handling - let cohortId = ''; - try { - const form = await this.formsService.getFormById(formIdToMatch); - cohortId = form?.contextId || ''; - } catch (error) { - LoggerUtil.warn( - `Failed to fetch cohortId for formId ${formIdToMatch}:`, - error - ); - cohortId = ''; - } - let existingAppIndex = -1; - if (cohortId) { - existingAppIndex = applications.findIndex( - (app) => app.cohortId === cohortId - ); - // If not found by cohortId, don't fallback to avoid inconsistencies - // Log this scenario for investigation - if (existingAppIndex === -1) { - } - } else { - // Use formId/submissionId only when cohortId is not available - existingAppIndex = applications.findIndex( - (app) => - app.formId === formIdToMatch && - app.submissionId === submissionIdToMatch - ); - } - - // Prepare the updated fields data using the robust mapping - const updatedFields = {}; - formFieldsOnly.forEach((field) => { - const pageKey = fieldIdToPageName[field.fieldId]; - if (!pageKey) { - // Log warning for unmapped fields instead of fallback - LoggerUtil.warn( - `FieldId ${field.fieldId} not found in schema mapping! Skipping field.` - ); - return; // Skip this field instead of assigning to wrong page - } - updatedFields[pageKey] ??= { - completed: true, - fields: {}, - }; - // Process field value for Elasticsearch - convert arrays to comma-separated strings - updatedFields[pageKey].fields[field.fieldId] = this.processFieldValueForElasticsearch(field.value); - }); - - - - if (existingAppIndex !== -1) { - // COMPLETELY REPLACE old pages structure instead of merging - const mergedPages = {}; - - // Only use the new schema-based mapping, don't merge with old data - for (const [pageKey, pageValue] of Object.entries(updatedFields)) { - const newPage = pageValue as { - completed: boolean; - fields: { [key: string]: any }; - }; - mergedPages[pageKey] = newPage; - } - - // Merge overall progress - const mergedOverall = applications[existingAppIndex]?.progress?.overall - ? { ...applications[existingAppIndex].progress.overall } - : { - completed: updatedFieldValues.length, - total: updatedFieldValues.length, - }; - - // Use completionPercentage from the updatedSubmission (payload value) instead of calculating - const completionPercentage = - updatedSubmission.completionPercentage ?? 0; - - // --- Update cohortmemberstatus and cohortDetails logic --- - // cohortmemberstatus is not a property of FormSubmission; set as empty string or fetch from CohortMembers if needed - applications[existingAppIndex].cohortmemberstatus = - applications[existingAppIndex].cohortmemberstatus ?? ''; // preserve if already set - // Ensure cohortDetails is populated; if missing or empty, fetch from DB - if ( - !applications[existingAppIndex].cohortDetails || - Object.keys(applications[existingAppIndex].cohortDetails).length === 0 - ) { - applications[existingAppIndex].cohortDetails = - await this.fetchCohortDetailsFromDB(updatedSubmission); - } - // --- End cohortmemberstatus and cohortDetails logic --- - - applications[existingAppIndex] = { - ...applications[existingAppIndex], - formId: formIdToMatch, - submissionId: submissionIdToMatch, - formstatus: - updatedSubmission.status ?? - applications[existingAppIndex].formstatus, - // Use completionPercentage from payload (updatedSubmission) - completionPercentage: completionPercentage, - progress: { - pages: mergedPages, - overall: mergedOverall, - }, - lastSavedAt: new Date().toISOString(), - submittedAt: new Date().toISOString(), - }; - - // Also update the FormSubmission entity in the database - try { - const submissionToUpdate = - await this.formSubmissionRepository.findOne({ - where: { submissionId: submissionIdToMatch }, - }); - if (submissionToUpdate) { - submissionToUpdate.completionPercentage = completionPercentage; - await this.formSubmissionRepository.save(submissionToUpdate); - } - } catch (error) { - LoggerUtil.warn( - 'Failed to update FormSubmission completionPercentage:', - error - ); - } - } else { - // If no existing application found, build from DB with correct cohortDetails - const newApp = await this.buildApplicationFromDB(updatedSubmission); - applications.push(newApp); - } - - // Upsert (update or create) the user document in Elasticsearch if (isElasticsearchEnabled()) { - await this.userElasticsearchService.updateUser( - userId, - { doc: { applications: applications } }, - async (userId: string) => { - // Build the full user document for Elasticsearch, including profile and all applications - return await this.buildUserDocumentForElasticsearch(userId); - } - ); + await this.elasticsearchSyncService.syncUserToElasticsearch(userId, { + section: SyncSection.APPLICATIONS, + }); } } catch (elasticError) { - // Log Elasticsearch error but don't fail the request LoggerUtil.warn('Failed to update Elasticsearch:', elasticError); } } @@ -1570,8 +1247,20 @@ export class FormSubmissionService { const form = await this.formsService.getFormById(submission.formId); const fieldsObj = form && form.fields ? (form.fields as any) : null; - // Get cohortId from form context - cohortId = form?.contextId || ''; + // Get cohortId from form's contextId + try { + // Get form to find contextId (which is the cohortId) + cohortId = form?.contextId || ''; + LoggerUtil.warn( + `Using form contextId as cohortId: ${cohortId} for formId: ${submission.formId}` + ); + } catch (error) { + LoggerUtil.warn( + `Failed to fetch form for formId ${submission.formId}:`, + error + ); + cohortId = ''; + } // Handle different schema structures if (fieldsObj) { @@ -1878,8 +1567,10 @@ export class FormSubmissionService { } // Process field value for Elasticsearch - convert arrays to comma-separated strings - pages[pageName].fields[field.fieldId] = this.processFieldValueForElasticsearch(field.value); - formData[pageName][field.fieldId] = this.processFieldValueForElasticsearch(field.value); + pages[pageName].fields[field.fieldId] = + this.processFieldValueForElasticsearch(field.value); + formData[pageName][field.fieldId] = + this.processFieldValueForElasticsearch(field.value); } if (Object.keys(pages).length === 0) { @@ -1887,7 +1578,9 @@ export class FormSubmissionService { completed: true, fields: formFieldsOnly.reduce((acc, field) => { // Process field value for Elasticsearch - convert arrays to comma-separated strings - acc[field.fieldId] = this.processFieldValueForElasticsearch(field.value); + acc[field.fieldId] = this.processFieldValueForElasticsearch( + field.value + ); return acc; }, {}), }; @@ -1980,8 +1673,28 @@ export class FormSubmissionService { * * Made public so it can be used as an upsert callback from other services (e.g., cohortMembers-adapter). */ + /** + * Build user document for Elasticsearch using centralized data fetcher. + * This method provides a centralized way to fetch user data for Elasticsearch. + * + * @param userId - User ID to fetch data for + * @returns Promise - Complete user document or null if user not found + */ public async buildUserDocumentForElasticsearch( userId: string + ): Promise { + // Use centralized data fetcher service for consistent data structure + return await this.elasticsearchDataFetcherService.fetchUserDocumentForElasticsearch( + userId + ); + } + + /** + * Legacy method - kept for backward compatibility. + * @deprecated Use buildUserDocumentForElasticsearch instead + */ + public async buildUserDocumentForElasticsearchLegacy( + userId: string ): Promise { // Fetch user profile from Users table const userRepo = this.formRepository.manager.getRepository('Users'); @@ -2597,15 +2310,19 @@ export class FormSubmissionService { } // Process field value for Elasticsearch - convert arrays to comma-separated strings - pages[pageName].fields[field.fieldId] = this.processFieldValueForElasticsearch(field.value); - formData[pageName][field.fieldId] = this.processFieldValueForElasticsearch(field.value); + pages[pageName].fields[field.fieldId] = + this.processFieldValueForElasticsearch(field.value); + formData[pageName][field.fieldId] = + this.processFieldValueForElasticsearch(field.value); } if (Object.keys(pages).length === 0) { pages['eligibilityCheck'] = { completed: true, fields: formFieldsOnly.reduce((acc, field) => { // Process field value for Elasticsearch - convert arrays to comma-separated strings - acc[field.fieldId] = this.processFieldValueForElasticsearch(field.value); + acc[field.fieldId] = this.processFieldValueForElasticsearch( + field.value + ); return acc; }, {}), }; @@ -3017,7 +2734,7 @@ export class FormSubmissionService { customFields: profileCustomFields, // Only user profile custom fields }, applications, - courses: [], + // Removed root-level courses field as requested createdAt: user.createdAt ? user.createdAt.toISOString() : new Date().toISOString(), @@ -3041,6 +2758,60 @@ export class FormSubmissionService { // Return value as-is for non-array values return value; } + + /** + * Helper function to build fieldId to fieldName mapping from form schema + * @param schema - Form schema object + * @returns Record - Mapping of fieldId to fieldName + */ + private getFieldIdToFieldNameMap(schema: any): Record { + const fieldIdToFieldName: Record = {}; + + function extract(properties: any, currentPage: string) { + for (const [fieldKey, fieldSchema] of Object.entries(properties)) { + const fieldId = (fieldSchema as any).fieldId; + const fieldTitle = (fieldSchema as any).title || fieldKey; + if (fieldId) { + fieldIdToFieldName[fieldId] = fieldTitle; + } + + // Handle dependencies + if ((fieldSchema as any).dependencies) { + const dependencies = (fieldSchema as any).dependencies; + for (const depSchema of Object.values(dependencies)) { + if (!depSchema || typeof depSchema !== 'object') continue; + const dep = depSchema as any; + if (dep.oneOf) + dep.oneOf.forEach( + (item: any) => + item?.properties && extract(item.properties, currentPage) + ); + if (dep.allOf) + dep.allOf.forEach( + (item: any) => + item?.properties && extract(item.properties, currentPage) + ); + if (dep.anyOf) + dep.anyOf.forEach( + (item: any) => + item?.properties && extract(item.properties, currentPage) + ); + if (dep.properties) extract(dep.properties, currentPage); + } + } + } + } + + // Handle different schema structures + if (schema?.properties) { + for (const [pageKey, pageSchema] of Object.entries(schema.properties)) { + const fieldProps = (pageSchema as any).properties || {}; + extract(fieldProps, pageKey); + } + } + + return fieldIdToFieldName; + } } /** diff --git a/src/shared/shared-elasticsearch.config.ts b/src/shared/shared-elasticsearch.config.ts new file mode 100644 index 00000000..17631e8c --- /dev/null +++ b/src/shared/shared-elasticsearch.config.ts @@ -0,0 +1,144 @@ +export const SHARED_ELASTICSEARCH_CONFIG = { + indexName: 'users', + node: process.env.ELASTICSEARCH_HOST || 'http://localhost:9200', + mappings: { + mappings: { + properties: { + userId: { type: 'keyword' }, + profile: { + properties: { + userId: { type: 'keyword' }, + username: { type: 'keyword' }, + firstName: { type: 'text' }, + lastName: { type: 'text' }, + middleName: { type: 'text' }, + email: { type: 'keyword' }, + mobile: { type: 'keyword' }, + mobile_country_code: { type: 'keyword' }, + gender: { type: 'keyword' }, + dob: { type: 'date', null_value: null }, + address: { type: 'text' }, + state: { type: 'keyword' }, + district: { type: 'keyword' }, + country: { type: 'keyword' }, + pincode: { type: 'keyword' }, + status: { type: 'keyword' }, + customFields: { + type: 'nested', + properties: { + fieldId: { type: 'keyword' }, + fieldValuesId: { type: 'keyword' }, + fieldname: { type: 'text' }, + code: { type: 'keyword' }, + label: { type: 'text' }, + type: { type: 'keyword' }, + value: { type: 'text' }, + context: { type: 'keyword' }, + contextType: { type: 'keyword' }, + state: { type: 'keyword' }, + fieldParams: { type: 'object', dynamic: true }, + }, + }, + }, + }, + applications: { + type: 'nested', + properties: { + cohortId: { type: 'keyword' }, + formId: { type: 'keyword' }, + submissionId: { type: 'keyword' }, + cohortmemberstatus: { type: 'keyword' }, + formstatus: { type: 'keyword' }, + completionPercentage: { type: 'float' }, + progress: { + properties: { + pages: { + type: 'object', + properties: { + completed: { type: 'boolean' }, + fields: { type: 'object', dynamic: true }, + }, + }, + overall: { + properties: { + completed: { type: 'integer' }, + total: { type: 'integer' }, + }, + }, + }, + }, + lastSavedAt: { type: 'date', null_value: null }, + submittedAt: { type: 'date', null_value: null }, + cohortDetails: { + properties: { + cohortId: { type: 'keyword' }, + name: { type: 'text' }, + type: { type: 'keyword' }, + status: { type: 'keyword' }, + }, + }, + courses: { + type: 'nested', + properties: { + courseId: { type: 'keyword' }, + courseTitle: { type: 'text' }, + progress: { type: 'float' }, + units: { + type: 'nested', + properties: { + unitId: { type: 'keyword' }, + unitTitle: { type: 'text' }, + progress: { type: 'float' }, + contents: { + type: 'nested', + properties: { + contentId: { type: 'keyword' }, + type: { type: 'keyword' }, + title: { type: 'text' }, + status: { type: 'keyword' }, + tracking: { + properties: { + percentComplete: { type: 'float' }, + lastPosition: { type: 'float' }, + currentPosition: { type: 'float' }, + timeSpent: { type: 'integer' }, + visitedPages: { type: 'integer' }, + totalPages: { type: 'integer' }, + lastPage: { type: 'integer' }, + currentPage: { type: 'integer' }, + questionsAttempted: { type: 'integer' }, + totalQuestions: { type: 'integer' }, + score: { type: 'float' }, + answers: { + type: 'nested', + properties: { + questionId: { type: 'keyword' }, + type: { type: 'keyword' }, + submittedAnswer: { type: 'text' }, + answer: { + properties: { + answer: { type: 'keyword' }, + text: { type: 'text' } + } + }, + text: { type: 'text' }, + score: { type: 'float' }, + reviewStatus: { type: 'keyword' } + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + createdAt: { type: 'date', null_value: null }, + updatedAt: { type: 'date', null_value: null }, + }, + }, + }, +}; \ No newline at end of file