From b6a1ea1f52e1bd877422e7dda7eaeeab8a21cb9f Mon Sep 17 00:00:00 2001 From: Tushar Date: Thu, 24 Jul 2025 18:20:53 +0530 Subject: [PATCH 1/2] Issue #000000 Fix: Elasticsearch code improved --- .../postgres/cohortMembers-adapter.ts | 7 +- src/adapters/postgres/user-adapter.ts | 90 +- .../elasticsearch-data-fetcher.service.ts | 769 ++++++++++++++++++ src/elasticsearch/elasticsearch.module.ts | 35 +- src/forms/services/form-submission.service.ts | 33 +- 5 files changed, 851 insertions(+), 83 deletions(-) create mode 100644 src/elasticsearch/elasticsearch-data-fetcher.service.ts diff --git a/src/adapters/postgres/cohortMembers-adapter.ts b/src/adapters/postgres/cohortMembers-adapter.ts index 0e2d1da2..fe67d4ed 100644 --- a/src/adapters/postgres/cohortMembers-adapter.ts +++ b/src/adapters/postgres/cohortMembers-adapter.ts @@ -34,6 +34,7 @@ import { FormSubmissionSearchDto } from 'src/forms/dto/form-submission-search.dt import { FormsService } from 'src/forms/forms.service'; import { isElasticsearchEnabled } from 'src/common/utils/elasticsearch.util'; import { UserElasticsearchService } from 'src/elasticsearch/user-elasticsearch.service'; +import { ElasticsearchDataFetcherService } from 'src/elasticsearch/elasticsearch-data-fetcher.service'; @Injectable() export class PostgresCohortMembersService { constructor( @@ -55,7 +56,8 @@ export class PostgresCohortMembersService { private readonly userService: PostgresUserService, private readonly formsService: FormsService, private readonly formSubmissionService: FormSubmissionService, - private readonly userElasticsearchService: UserElasticsearchService + private readonly userElasticsearchService: UserElasticsearchService, + private readonly elasticsearchDataFetcherService: ElasticsearchDataFetcherService ) {} //Get cohort member @@ -760,7 +762,8 @@ export class PostgresCohortMembersService { cohortMembers.userId, { doc: { ...baseDoc, applications } }, async (userId: string) => { - return await this.formSubmissionService.buildUserDocumentForElasticsearch( + // Use centralized service to fetch complete user document + return await this.elasticsearchDataFetcherService.fetchUserDocumentForElasticsearch( userId ); } diff --git a/src/adapters/postgres/user-adapter.ts b/src/adapters/postgres/user-adapter.ts index b4438e33..0fe2fedd 100644 --- a/src/adapters/postgres/user-adapter.ts +++ b/src/adapters/postgres/user-adapter.ts @@ -48,6 +48,7 @@ import config from '../../common/config'; import { CalendarField } from 'src/fields/fieldValidators/fieldTypeClasses'; import { UserCreateSsoDto } from 'src/user/dto/user-create-sso.dto'; import { UserElasticsearchService } from '../../elasticsearch/user-elasticsearch.service'; +import { ElasticsearchDataFetcherService } from '../../elasticsearch/elasticsearch-data-fetcher.service'; import { IUser } from '../../elasticsearch/interfaces/user.interface'; import { isElasticsearchEnabled } from 'src/common/utils/elasticsearch.util'; @@ -94,7 +95,8 @@ export class PostgresUserService implements IServicelocator { private postgresAcademicYearService: PostgresAcademicYearService, private readonly cohortAcademicYearService: CohortAcademicYearService, private readonly authUtils: AuthUtils, - private readonly userElasticsearchService: UserElasticsearchService + private readonly userElasticsearchService: UserElasticsearchService, + private readonly elasticsearchDataFetcherService: ElasticsearchDataFetcherService ) { this.jwt_secret = this.configService.get('RBAC_JWT_SECRET'); this.jwt_password_reset_expires_In = this.configService.get( @@ -1031,8 +1033,9 @@ export class PostgresUserService implements IServicelocator { const updatedUser = await this.updateBasicUserDetails(userId, userDto); // Sync to Elasticsearch - await this.syncUserToElasticsearch(updatedUser); - + if (isElasticsearchEnabled()) { + await this.syncUserToElasticsearch(updatedUser); + } return await APIResponse.success( response, apiId, @@ -2771,89 +2774,22 @@ export class PostgresUserService implements IServicelocator { } /** - * Sync user profile to Elasticsearch. + * Sync user profile to Elasticsearch using centralized data fetcher. * This will upsert (update or create) the user document in Elasticsearch. * If the document is missing, it will fetch the user from the database and create it. */ private async syncUserToElasticsearch(user: User) { try { - const customFields = await this.getFilteredCustomFields(user.userId); - - let formattedDob: string | null = null; - if (user.dob instanceof Date) { - formattedDob = user.dob.toISOString(); - } else if (typeof user.dob === 'string') { - formattedDob = user.dob; - } - // Prepare the profile data - const profile = { - userId: user.userId, - username: user.username, - firstName: user.firstName, - lastName: user.lastName, - middleName: user.middleName || '', - email: user.email || '', - mobile: user.mobile?.toString() || '', - mobile_country_code: user.mobile_country_code || '', - dob: formattedDob, - country: user.country, - gender: user.gender, - address: user.address || '', - district: user.district || '', - state: user.state || '', - pincode: user.pincode || '', - status: user.status, - customFields, // Now filtered to exclude form schema fields - }; - - // Upsert (update or create) the user profile in Elasticsearch + // Use centralized data fetcher to get complete user document if (isElasticsearchEnabled()) { await this.userElasticsearchService.updateUserProfile( user.userId, - profile, + { userId: user.userId }, // Minimal profile update async (userId: string) => { - // Fetch the latest user from the database for upsert - const dbUser = await this.usersRepository.findOne({ - where: { userId }, - }); - if (!dbUser) return null; - const customFields = await this.getFilteredCustomFields(userId); - let formattedDob: string | null = null; - if (dbUser.dob instanceof Date) { - formattedDob = dbUser.dob.toISOString(); - } else if (typeof dbUser.dob === 'string') { - formattedDob = dbUser.dob; - } - return { - userId: dbUser.userId, - profile: { - userId: dbUser.userId, - username: dbUser.username, - firstName: dbUser.firstName, - lastName: dbUser.lastName, - middleName: dbUser.middleName || '', - email: dbUser.email || '', - mobile: dbUser.mobile?.toString() || '', - mobile_country_code: dbUser.mobile_country_code || '', - dob: formattedDob, - gender: dbUser.gender, - country: dbUser.country || '', - address: dbUser.address || '', - district: dbUser.district || '', - state: dbUser.state || '', - pincode: dbUser.pincode || '', - status: dbUser.status, - customFields, - }, - applications: [], - courses: [], - createdAt: dbUser.createdAt - ? dbUser.createdAt.toISOString() - : new Date().toISOString(), - updatedAt: dbUser.updatedAt - ? dbUser.updatedAt.toISOString() - : new Date().toISOString(), - }; + // Use centralized service to fetch complete user document + return await this.elasticsearchDataFetcherService.fetchUserDocumentForElasticsearch( + userId + ); } ); } diff --git a/src/elasticsearch/elasticsearch-data-fetcher.service.ts b/src/elasticsearch/elasticsearch-data-fetcher.service.ts new file mode 100644 index 00000000..3ae58a15 --- /dev/null +++ b/src/elasticsearch/elasticsearch-data-fetcher.service.ts @@ -0,0 +1,769 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository } from 'typeorm'; +import { User } from '../user/entities/user-entity'; +import { CohortMembers } from '../cohortMembers/entities/cohort-member.entity'; +import { FormSubmission } from '../forms/entities/form-submission.entity'; +import { FieldValues } from '../fields/entities/fields-values.entity'; +import { Cohort } from '../cohort/entities/cohort.entity'; +import { IUser, IProfile } from './interfaces/user.interface'; +import { isElasticsearchEnabled } from '../common/utils/elasticsearch.util'; +import { LoggerUtil } from '../common/logger/LoggerUtil'; +import { PostgresFieldsService } from '../adapters/postgres/fields-adapter'; +import { FormsService } from '../forms/forms.service'; + +/** + * Centralized Elasticsearch Data Fetcher Service + * + * This service handles all data fetching operations from database to Elasticsearch. + * It provides reusable functions for fetching user profiles, applications, and cohort details + * that can be used across different adapters and services. + * + * Key Features: + * - Centralized data fetching logic to reduce code duplication + * - Handles user profile data with custom fields + * - Manages application data with form submissions and cohort details + * - Provides course data structure for future use + * - Implements proper error handling and logging + * - Supports dynamic field mapping and schema extraction + */ +@Injectable() +export class ElasticsearchDataFetcherService { + private readonly logger = new Logger(ElasticsearchDataFetcherService.name); + + constructor( + @InjectRepository(User) + private readonly userRepository: Repository, + @InjectRepository(CohortMembers) + private readonly cohortMembersRepository: Repository, + @InjectRepository(FormSubmission) + private readonly formSubmissionRepository: Repository, + @InjectRepository(FieldValues) + private readonly fieldValuesRepository: Repository, + @InjectRepository(Cohort) + private readonly cohortRepository: Repository, + private readonly fieldsService: PostgresFieldsService, + private readonly formsService: FormsService, + ) {} + + /** + * Fetch complete user document for Elasticsearch from database + * This is the main function that fetches all required data for a user + * + * @param userId - The user ID to fetch data for + * @returns Promise - Complete user document or null if user not found + */ + async fetchUserDocumentForElasticsearch(userId: string): Promise { + try { + this.logger.debug(`Fetching user document for Elasticsearch: ${userId}`); + + // Fetch user from database + const user = await this.userRepository.findOne({ where: { userId } }); + if (!user) { + this.logger.warn(`User not found in database: ${userId}`); + return null; + } + + // Fetch profile data (including custom fields) + const profile = await this.fetchUserProfile(user); + + // Fetch applications data + const applications = await this.fetchUserApplications(userId); + + // Fetch courses data (placeholder for future implementation) + const courses = await this.fetchUserCourses(userId); + + // Create complete user document + const userDocument: IUser = { + userId: user.userId, + profile, + applications, + courses, + createdAt: user.createdAt ? user.createdAt.toISOString() : new Date().toISOString(), + updatedAt: user.updatedAt ? user.updatedAt.toISOString() : new Date().toISOString(), + }; + + this.logger.debug(`Successfully fetched user document for: ${userId}`); + return userDocument; + + } catch (error) { + this.logger.error(`Failed to fetch user document for ${userId}:`, error); + throw new Error(`Failed to fetch user document: ${error.message}`); + } + } + + /** + * Fetch user profile data including custom fields + * + * @param user - User entity from database + * @returns Promise - User profile with custom fields + */ + private async fetchUserProfile(user: User): Promise { + try { + // Fetch custom fields for the user + const customFields = await this.fetchUserCustomFields(user.userId); + + // Format date of birth + let formattedDob: string | null = null; + if (user.dob instanceof Date) { + formattedDob = user.dob.toISOString(); + } else if (typeof user.dob === 'string') { + formattedDob = user.dob; + } + + // Create profile object + const profile: IProfile = { + userId: user.userId, + username: user.username || '', + firstName: user.firstName || '', + lastName: user.lastName || '', + middleName: user.middleName || '', + email: user.email || '', + mobile: user.mobile?.toString() || '', + mobile_country_code: user.mobile_country_code || '', + gender: user.gender || '', + dob: formattedDob, + country: user.country || '', + address: user.address || '', + district: user.district || '', + state: user.state || '', + pincode: user.pincode || '', + status: user.status || 'active', + customFields, + }; + + return profile; + + } catch (error) { + this.logger.error(`Failed to fetch user profile for ${user.userId}:`, error); + throw new Error(`Failed to fetch user profile: ${error.message}`); + } + } + + /** + * Fetch user custom fields from database using existing fields service + * This ensures consistency with the existing implementation + * + * @param userId - User ID to fetch custom fields for + * @returns Promise - Array of custom field objects + */ + private async fetchUserCustomFields(userId: string): Promise { + try { + // Use existing fields service to get custom field details + const customFields = await this.fieldsService.getUserCustomFieldDetails(userId); + + this.logger.debug(`Found ${customFields.length} custom fields for user ${userId}`); + + // Get all form submissions for this user to identify form fields + const submissions = await this.formSubmissionRepository.find({ + where: { itemId: userId }, + }); + + this.logger.debug(`Found ${submissions.length} form submissions for filtering custom fields`); + + // Collect all form field IDs to exclude from custom fields + const formFieldIds = new Set(); + + for (const submission of submissions) { + try { + // Extract field IDs from form schema using proper implementation + const formFields = await this.extractFieldIdsFromFormSchema(submission.formId); + formFields.forEach(fieldId => formFieldIds.add(fieldId)); + } catch (error) { + this.logger.warn(`Failed to extract field IDs from form ${submission.formId}:`, error); + } + } + + this.logger.debug(`Form field IDs to exclude: ${Array.from(formFieldIds).join(', ')}`); + + // Filter out fields that are part of form schemas + const filteredCustomFields = customFields.filter((field) => !formFieldIds.has(field.fieldId)); + + this.logger.debug(`After filtering, ${filteredCustomFields.length} custom fields remain`); + + // Transform to match the expected format + const transformedFields = filteredCustomFields.map(field => ({ + fieldId: field.fieldId, + fieldValuesId: field.fieldValuesId || '', + fieldname: field.label || '', + code: field.code || '', + label: field.label || '', + type: field.type || '', + value: this.processFieldValueForElasticsearch(field.value), + context: field.context || '', + contextType: field.contextType || '', + state: field.state || '', + fieldParams: field.fieldParams || {}, + })); + + this.logger.debug(`Returning ${transformedFields.length} transformed custom fields`); + return transformedFields; + + } catch (error) { + this.logger.error(`Failed to fetch custom fields for ${userId}:`, error); + return []; + } + } + + /** + * Fetch user applications with form submissions and cohort details + * + * @param userId - User ID to fetch applications for + * @returns Promise - Array of application objects + */ + private async fetchUserApplications(userId: string): Promise { + try { + // Fetch all cohort memberships for this user + const cohortMemberships = await this.cohortMembersRepository.find({ + where: { userId }, + }); + + this.logger.debug(`Found ${cohortMemberships.length} cohort memberships for user ${userId}`); + + // Log cohort membership details for debugging + if (cohortMemberships.length > 0) { + this.logger.debug(`Cohort membership details:`, cohortMemberships.map(membership => ({ + cohortId: membership.cohortId, + userId: membership.userId, + status: membership.status, + cohortMembershipId: membership.cohortMembershipId + }))); + } + + // Fetch all form submissions for this user + const submissions = await this.formSubmissionRepository.find({ + where: { itemId: userId }, + }); + + this.logger.debug(`Found ${submissions.length} form submissions for user ${userId}`); + + // Log submission details for debugging + if (submissions.length > 0) { + this.logger.debug(`Submission details:`, submissions.map(sub => ({ + formId: sub.formId, + itemId: sub.itemId, + status: sub.status, + updatedAt: sub.updatedAt + }))); + } + + const applications: any[] = []; + + // Process each cohort membership + for (const membership of cohortMemberships) { + const application = await this.buildApplicationForCohort( + userId, + membership, + submissions + ); + + if (application) { + applications.push(application); + } + } + + // If no cohort memberships but form submissions exist, create a default application + if (applications.length === 0 && submissions.length > 0) { + this.logger.debug(`Creating default application for user ${userId} with form submissions`); + + for (const submission of submissions) { + const defaultApplication = { + cohortId: submission.formId || 'default', + cohortmemberstatus: 'active', + formstatus: submission.status || 'inactive', + completionPercentage: 0, + progress: { + pages: {}, + overall: { + completed: 0, + total: 0, + }, + }, + lastSavedAt: submission.updatedAt ? submission.updatedAt.toISOString() : null, + submittedAt: null, + cohortDetails: { + name: `Form ${submission.formId}`, + description: '', + startDate: null, + endDate: null, + status: 'active', + }, + formData: await this.buildFormDataFromSubmission(submission), + }; + + applications.push(defaultApplication); + } + } + + this.logger.debug(`Returning ${applications.length} applications for user ${userId}`); + return applications; + + } catch (error) { + this.logger.error(`Failed to fetch applications for ${userId}:`, error); + return []; + } + } + + /** + * Build application object for a specific cohort + * + * @param userId - User ID + * @param membership - Cohort membership entity + * @param submissions - All form submissions for the user + * @returns Promise - Application object or null + */ + private async buildApplicationForCohort( + userId: string, + membership: CohortMembers, + submissions: FormSubmission[] + ): Promise { + try { + // Find form submission for this cohort + // Note: FormSubmission doesn't have cohortId, so we'll use itemId to match user + const submission = submissions.find(sub => sub.itemId === userId); + + if (!submission) { + this.logger.warn(`No form submission found for user ${userId} in cohort ${membership.cohortId}`); + return null; + } + + // Build form data with proper page structure + const formData = await this.buildFormDataWithPages(submission); + + // Calculate completion percentage and progress + const { percentage, progress } = this.calculateCompletionPercentage(formData); + + // Fetch cohort details + const cohort = await this.cohortRepository.findOne({ + where: { cohortId: membership.cohortId }, + }); + + return { + cohortId: membership.cohortId, + formId: submission.formId, + submissionId: submission.submissionId, + cohortmemberstatus: membership.status || 'active', + formstatus: submission.status || 'active', + completionPercentage: percentage, + progress, + lastSavedAt: submission.updatedAt ? submission.updatedAt.toISOString() : null, + submittedAt: submission.status === 'active' ? submission.updatedAt?.toISOString() : null, + cohortDetails: { + name: cohort?.name || 'Unknown Cohort', + description: '', // Cohort entity doesn't have description field + startDate: null, // Cohort entity doesn't have startDate field + endDate: null, // Cohort entity doesn't have endDate field + status: cohort?.status || 'active', + }, + formData, + }; + + } catch (error) { + this.logger.error(`Failed to build application for cohort ${membership.cohortId}:`, error); + return null; + } + } + + /** + * Build form data from submission + * + * @param submission - Form submission entity + * @returns Promise - Form data object + */ + private async buildFormDataFromSubmission(submission: FormSubmission): Promise { + try { + // Fetch field values for this submission + // Note: FieldValues entity doesn't have formId field, so we'll filter by itemId only + const fieldValues = await this.fieldValuesRepository.find({ + where: { + itemId: submission.itemId, + }, + relations: ['field'], + }); + + // Group field values by page (simplified - you may need to implement proper page mapping) + const formData: any = {}; + + for (const fieldValue of fieldValues) { + // For now, put all fields in a default page + // In a real implementation, you'd need to map fields to pages based on form schema + const pageId = 'default'; + + if (!formData[pageId]) { + formData[pageId] = {}; + } + + formData[pageId][fieldValue.fieldId] = this.processFieldValueForElasticsearch(fieldValue.value); + } + + return formData; + + } catch (error) { + this.logger.error(`Failed to build form data from submission ${submission.submissionId}:`, error); + return {}; + } + } + + /** + * Build form data with proper page structure from submission + * + * @param submission - Form submission entity + * @returns Promise - Form data with page structure + */ + private async buildFormDataWithPages(submission: FormSubmission): Promise { + try { + // Fetch field values for this submission + const fieldValues = await this.fieldValuesRepository.find({ + where: { + itemId: submission.itemId, + }, + relations: ['field'], + }); + + // Get form schema to build proper page structure + const formSchema = await this.getFormSchema(submission.formId); + const fieldIdToPageName = this.getFieldIdToPageNameMap(formSchema); + + // Build page structure + const formData: any = {}; + const pages: any = {}; + + // Initialize pages from schema + for (const [pageKey, pageSchema] of Object.entries(formSchema)) { + const pageName = pageKey === 'default' ? 'eligibilityCheck' : pageKey; + pages[pageName] = { completed: true, fields: {} }; + formData[pageName] = {}; + } + + // Map field values to correct pages + for (const fieldValue of fieldValues) { + const pageName = fieldIdToPageName[fieldValue.fieldId]; + if (!pageName) { + this.logger.warn(`Field ${fieldValue.fieldId} not found in schema mapping, skipping`); + continue; + } + + if (!pages[pageName]) { + pages[pageName] = { completed: true, fields: {} }; + formData[pageName] = {}; + } + + const processedValue = this.processFieldValueForElasticsearch(fieldValue.value); + pages[pageName].fields[fieldValue.fieldId] = processedValue; + formData[pageName][fieldValue.fieldId] = processedValue; + } + + // Update page completion status + for (const [pageName, pageData] of Object.entries(pages)) { + const fields = (pageData as any).fields; + const fieldCount = Object.keys(fields).length; + const completedFields = Object.values(fields).filter(value => + value !== null && value !== undefined && value !== '' + ).length; + + pages[pageName].completed = fieldCount > 0 && completedFields === fieldCount; + } + + return formData; + + } catch (error) { + this.logger.error(`Failed to build form data with pages for submission ${submission.submissionId}:`, error); + return {}; + } + } + + /** + * Calculate completion percentage from form data + * + * @param formData - Form data object + * @returns Object with percentage and progress data + */ + private calculateCompletionPercentage(formData: any): { percentage: number; progress: any } { + let totalFields = 0; + let completedFields = 0; + const pages: any = {}; + + // Process each page + for (const [pageId, pageData] of Object.entries(formData)) { + const pageFields: any = {}; + let pageCompleted = true; + let pageTotal = 0; + let pageCompletedCount = 0; + + // Process each field in the page + for (const [fieldId, value] of Object.entries(pageData as any)) { + pageTotal++; + totalFields++; + + if (value !== null && value !== undefined && value !== '') { + pageCompletedCount++; + completedFields++; + pageFields[fieldId] = value; + } else { + pageCompleted = false; + pageFields[fieldId] = value; + } + } + + // Set page completion status + pages[pageId] = { + completed: pageCompleted, + fields: pageFields, + }; + } + + // Calculate overall percentage + const percentage = totalFields > 0 ? Math.round((completedFields / totalFields) * 100) : 0; + + return { + percentage, + progress: { + pages, + overall: { + completed: completedFields, + total: totalFields, + }, + }, + }; + } + + /** + * Fetch user courses (placeholder for future implementation) + * + * @param userId - User ID to fetch courses for + * @returns Promise - Array of course objects + */ + private async fetchUserCourses(userId: string): Promise { + // Placeholder for future course implementation + // This can be expanded when course functionality is added + return []; + } + + /** + * Get form schema from forms service + * + * @param formId - Form ID + * @returns Promise - Form schema + */ + private async getFormSchema(formId: string): Promise { + try { + const form = await this.formsService.getFormById(formId); + const fieldsObj = form && (form as any).fields ? (form as any).fields : null; + + // Handle different schema structures + let schema: any = {}; + if (fieldsObj) { + // Try different possible schema structures + if ( + Array.isArray(fieldsObj?.result) && + fieldsObj.result[0]?.schema?.properties + ) { + // Structure: { result: [{ schema: { properties: {...} } }] } + schema = fieldsObj.result[0].schema.properties; + } else if (fieldsObj?.schema?.properties) { + // Structure: { schema: { properties: {...} } } + schema = fieldsObj.schema.properties; + } else if (fieldsObj?.properties) { + // Structure: { properties: {...} } + schema = fieldsObj.properties; + } else if (typeof fieldsObj === 'object' && fieldsObj !== null) { + // Try to find schema in nested structure + const findSchema = (obj: any): any => { + if (obj?.schema?.properties) return obj.schema.properties; + if (obj?.properties) return obj.properties; + if (Array.isArray(obj)) { + for (const item of obj) { + const found = findSchema(item); + if (found) return found; + } + } else if (typeof obj === 'object') { + for (const key in obj) { + const found = findSchema(obj[key]); + if (found) return found; + } + } + return null; + }; + schema = findSchema(fieldsObj) || {}; + } + } + + this.logger.debug(`Extracted schema for form ${formId}:`, Object.keys(schema)); + return schema; + } catch (error) { + this.logger.error(`Failed to get form schema for ${formId}:`, error); + return {}; + } + } + + /** + * Extract field IDs from form schema using proper implementation + * + * @param formId - Form ID to extract field IDs from + * @returns Promise - Array of field IDs + */ + private async extractFieldIdsFromFormSchema(formId: string): Promise { + try { + const schema = await this.getFormSchema(formId); + const fieldIds: string[] = []; + + // Extract field IDs from schema structure + for (const [pageKey, pageSchema] of Object.entries(schema)) { + const fieldProps = (pageSchema as any).properties || {}; + + const extractFieldIds = (properties: any) => { + for (const [fieldKey, fieldSchema] of Object.entries(properties)) { + const fieldId = (fieldSchema as any).fieldId; + if (fieldId) { + fieldIds.push(fieldId); + } + + // Handle dependencies + if ((fieldSchema as any).dependencies) { + const dependencies = (fieldSchema as any).dependencies; + for (const depSchema of Object.values(dependencies)) { + if (!depSchema || typeof depSchema !== 'object') continue; + const dep = depSchema as any; + if (dep.oneOf) + dep.oneOf.forEach((item: any) => + item?.properties && extractFieldIds(item.properties) + ); + if (dep.allOf) + dep.allOf.forEach((item: any) => + item?.properties && extractFieldIds(item.properties) + ); + if (dep.anyOf) + dep.anyOf.forEach((item: any) => + item?.properties && extractFieldIds(item.properties) + ); + if (dep.properties) extractFieldIds(dep.properties); + } + } + } + }; + + extractFieldIds(fieldProps); + + // Handle page-level dependencies + const pageDependencies = (pageSchema as any).dependencies || {}; + for (const depSchema of Object.values(pageDependencies)) { + if (!depSchema || typeof depSchema !== 'object') continue; + const dep = depSchema as any; + if (dep.oneOf) + dep.oneOf.forEach((item: any) => + item?.properties && extractFieldIds(item.properties) + ); + if (dep.allOf) + dep.allOf.forEach((item: any) => + item?.properties && extractFieldIds(item.properties) + ); + if (dep.anyOf) + dep.anyOf.forEach((item: any) => + item?.properties && extractFieldIds(item.properties) + ); + if (dep.properties) extractFieldIds(dep.properties); + } + } + + this.logger.debug(`Extracted ${fieldIds.length} field IDs from form ${formId}`); + return fieldIds; + + } catch (error) { + this.logger.error(`Failed to extract field IDs from form ${formId}:`, error); + return []; + } + } + + /** + * Get field ID to page name mapping from schema + * + * @param schema - Form schema + * @returns Record - Field ID to page name mapping + */ + private getFieldIdToPageNameMap(schema: any): Record { + const fieldIdToPageName: Record = {}; + + function extract(properties: any, currentPage: string) { + if (!properties || typeof properties !== 'object') return; + for (const [fieldKey, fieldSchema] of Object.entries(properties)) { + if (!fieldSchema || typeof fieldSchema !== 'object') continue; + const fieldId = (fieldSchema as any).fieldId; + if (fieldId) fieldIdToPageName[fieldId] = currentPage; + + // Traverse dependencies + if ((fieldSchema as any).dependencies) { + for (const depSchema of Object.values((fieldSchema as any).dependencies)) { + if (!depSchema || typeof depSchema !== 'object') continue; + const dep = depSchema as any; + if (dep.oneOf) + dep.oneOf.forEach((item: any) => + item?.properties && extract(item.properties, currentPage) + ); + if (dep.allOf) + dep.allOf.forEach((item: any) => + item?.properties && extract(item.properties, currentPage) + ); + if (dep.anyOf) + dep.anyOf.forEach((item: any) => + item?.properties && extract(item.properties, currentPage) + ); + if (dep.properties) extract(dep.properties, currentPage); + } + } + } + } + + for (const [pageKey, pageSchema] of Object.entries(schema)) { + const pageName = pageKey === 'default' ? 'eligibilityCheck' : pageKey; + extract((pageSchema as any).properties, pageName); + + // Also check for page-level dependencies + if ((pageSchema as any).dependencies) { + for (const depSchema of Object.values((pageSchema as any).dependencies)) { + if (!depSchema || typeof depSchema !== 'object') continue; + const dep = depSchema as any; + if (dep.oneOf) + dep.oneOf.forEach((item: any) => + item?.properties && extract(item.properties, pageName) + ); + if (dep.allOf) + dep.allOf.forEach((item: any) => + item?.properties && extract(item.properties, pageName) + ); + if (dep.anyOf) + dep.anyOf.forEach((item: any) => + item?.properties && extract(item.properties, pageName) + ); + if (dep.properties) extract(dep.properties, pageName); + } + } + } + + return fieldIdToPageName; + } + + /** + * Process field value for Elasticsearch storage + * Converts array values to comma-separated strings for multiselect fields + * + * @param value - The field value to process + * @returns Processed value (array becomes comma-separated string, other types unchanged) + */ + private processFieldValueForElasticsearch(value: any): any { + // If value is an array, convert to comma-separated string + if (Array.isArray(value)) { + return value.join(', '); + } + // Return value as-is for non-array values + return value; + } + + /** + * Check if Elasticsearch is enabled + * + * @returns boolean - True if Elasticsearch is enabled + */ + isElasticsearchEnabled(): boolean { + return isElasticsearchEnabled(); + } +} \ No newline at end of file diff --git a/src/elasticsearch/elasticsearch.module.ts b/src/elasticsearch/elasticsearch.module.ts index 436affb4..bb00cbb5 100644 --- a/src/elasticsearch/elasticsearch.module.ts +++ b/src/elasticsearch/elasticsearch.module.ts @@ -1,17 +1,46 @@ // src/elasticsearch/elasticsearch.module.ts import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; +import { ElasticsearchConfig } from './elasticsearch.config'; import { ElasticsearchService } from './elasticsearch.service'; import { UserElasticsearchService } from './user-elasticsearch.service'; -import { ElasticsearchConfig } from './elasticsearch.config'; import { UserElasticsearchController } from './user-elasticsearch.controller'; +import { ElasticsearchDataFetcherService } from './elasticsearch-data-fetcher.service'; +import { User } from '../user/entities/user-entity'; +import { CohortMembers } from '../cohortMembers/entities/cohort-member.entity'; +import { FormSubmission } from '../forms/entities/form-submission.entity'; +import { FieldValues } from '../fields/entities/fields-values.entity'; +import { Fields } from '../fields/entities/fields.entity'; +import { Cohort } from '../cohort/entities/cohort.entity'; +import { PostgresFieldsService } from '../adapters/postgres/fields-adapter'; +import { FormsService } from '../forms/forms.service'; +import { Form } from '../forms/entities/form.entity'; @Module({ + imports: [ + TypeOrmModule.forFeature([ + User, + CohortMembers, + FormSubmission, + FieldValues, + Fields, + Cohort, + Form, + ]), + ], controllers: [UserElasticsearchController], providers: [ ElasticsearchConfig, ElasticsearchService, - UserElasticsearchService + UserElasticsearchService, + ElasticsearchDataFetcherService, + PostgresFieldsService, + FormsService, + ], + exports: [ + ElasticsearchService, + UserElasticsearchService, + ElasticsearchDataFetcherService ], - exports: [ElasticsearchService, UserElasticsearchService], }) export class ElasticsearchModule {} diff --git a/src/forms/services/form-submission.service.ts b/src/forms/services/form-submission.service.ts index 6955d4ac..f44a110f 100644 --- a/src/forms/services/form-submission.service.ts +++ b/src/forms/services/form-submission.service.ts @@ -29,6 +29,7 @@ import { FieldsSearchDto } from '../../fields/dto/fields-search.dto'; import jwt_decode from 'jwt-decode'; import { Form } from '../entities/form.entity'; import { UserElasticsearchService } from '../../elasticsearch/user-elasticsearch.service'; +import { ElasticsearchDataFetcherService } from '../../elasticsearch/elasticsearch-data-fetcher.service'; import { FormsService } from '../../forms/forms.service'; import { PostgresCohortService } from 'src/adapters/postgres/cohort-adapter'; import { IUser } from '../../elasticsearch/interfaces/user.interface'; @@ -76,6 +77,8 @@ interface FieldSearchResponse { @Injectable() export class FormSubmissionService { + private readonly logger = new Logger(FormSubmissionService.name); + constructor( @InjectRepository(FormSubmission) private formSubmissionRepository: Repository, @@ -89,6 +92,7 @@ export class FormSubmissionService { private cohortRepository: Repository, private readonly fieldsService: FieldsService, private readonly userElasticsearchService: UserElasticsearchService, + private readonly elasticsearchDataFetcherService: ElasticsearchDataFetcherService, private readonly formsService: FormsService, @Inject(forwardRef(() => PostgresCohortService)) private readonly postgresCohortService: PostgresCohortService @@ -1162,6 +1166,7 @@ export class FormSubmissionService { * Update the user's applications array in Elasticsearch after a form submission update. * This will upsert (update or create) the user document in Elasticsearch if missing. * If the document is missing, it will fetch the user from the database and create it. + * Uses centralized data fetcher for consistent data structure. */ private async updateApplicationInElasticsearch( userId: string, @@ -1169,6 +1174,14 @@ export class FormSubmissionService { updatedFieldValues: any[] ): Promise { try { + // Use centralized service to get complete user document + const userDocument = await this.elasticsearchDataFetcherService.fetchUserDocumentForElasticsearch(userId); + + if (!userDocument) { + this.logger.warn(`User document not found for ${userId}, skipping Elasticsearch update`); + return; + } + // Get the existing user document from Elasticsearch const userDoc = await this.userElasticsearchService.getUser(userId); @@ -1948,7 +1961,7 @@ export class FormSubmissionService { parentId: cohortDetails.parentId, type: cohortDetails.type, status: cohortDetails.status, - customFields: cohortCustomFields, + // Removed customFields from cohortDetails as per requirement }; } } catch (error) { @@ -1980,8 +1993,26 @@ export class FormSubmissionService { * * Made public so it can be used as an upsert callback from other services (e.g., cohortMembers-adapter). */ + /** + * Build user document for Elasticsearch using centralized data fetcher. + * This method provides a centralized way to fetch user data for Elasticsearch. + * + * @param userId - User ID to fetch data for + * @returns Promise - Complete user document or null if user not found + */ public async buildUserDocumentForElasticsearch( userId: string + ): Promise { + // Use centralized data fetcher service for consistent data structure + return await this.elasticsearchDataFetcherService.fetchUserDocumentForElasticsearch(userId); + } + + /** + * Legacy method - kept for backward compatibility. + * @deprecated Use buildUserDocumentForElasticsearch instead + */ + public async buildUserDocumentForElasticsearchLegacy( + userId: string ): Promise { // Fetch user profile from Users table const userRepo = this.formRepository.manager.getRepository('Users'); From d0f58ed27cdbd78d8c0fbc940caae072b6291c93 Mon Sep 17 00:00:00 2001 From: Tushar Date: Tue, 12 Aug 2025 18:03:15 +0530 Subject: [PATCH 2/2] elasticsearch changes with new LMS and assessment --- .../postgres/cohortMembers-adapter.ts | 57 +- src/adapters/postgres/user-adapter.ts | 47 +- src/app.controller.ts | 9 + src/app.module.ts | 2 + .../services/bulk-import.service.ts | 1 + .../controllers/elasticsearch.controller.ts | 92 +- .../elasticsearch-data-fetcher.service.ts | 1311 +++++++++++++++-- .../elasticsearch-sync.service.ts | 1 + src/elasticsearch/elasticsearch.module.ts | 32 +- src/elasticsearch/elasticsearch.service.ts | 5 +- .../interfaces/user.interface.ts | 60 +- .../user-elasticsearch.controller.ts | 7 +- .../user-elasticsearch.service.ts | 504 ++++++- src/forms/services/form-submission.service.ts | 478 ++---- 14 files changed, 2006 insertions(+), 600 deletions(-) diff --git a/src/adapters/postgres/cohortMembers-adapter.ts b/src/adapters/postgres/cohortMembers-adapter.ts index 4930602b..878d8752 100644 --- a/src/adapters/postgres/cohortMembers-adapter.ts +++ b/src/adapters/postgres/cohortMembers-adapter.ts @@ -36,6 +36,11 @@ import { isElasticsearchEnabled } from 'src/common/utils/elasticsearch.util'; import { UserElasticsearchService } from 'src/elasticsearch/user-elasticsearch.service'; import { ElasticsearchDataFetcherService } from 'src/elasticsearch/elasticsearch-data-fetcher.service'; import axios from 'axios'; +import { + ElasticsearchSyncService, + SyncSection, +} from '../../elasticsearch/elasticsearch-sync.service'; + @Injectable() export class PostgresCohortMembersService { constructor( @@ -58,7 +63,8 @@ export class PostgresCohortMembersService { private readonly formsService: FormsService, private readonly formSubmissionService: FormSubmissionService, private readonly userElasticsearchService: UserElasticsearchService, - private readonly elasticsearchDataFetcherService: ElasticsearchDataFetcherService + private readonly elasticsearchDataFetcherService: ElasticsearchDataFetcherService, + private readonly elasticsearchSyncService: ElasticsearchSyncService ) {} //Get cohort member @@ -722,7 +728,20 @@ export class PostgresCohortMembersService { // Update Elasticsearch with cohort member status if (isElasticsearchEnabled()) { try { - // First get the existing user document from Elasticsearch + // Use comprehensive sync to get complete user document including courses and assessment data + const userDocument = + await this.elasticsearchDataFetcherService.comprehensiveUserSync( + cohortMembers.userId + ); + + if (!userDocument) { + LoggerUtil.warn( + `User document not found for ${cohortMembers.userId}, skipping Elasticsearch update` + ); + return; + } + + // Get the existing user document from Elasticsearch const userDoc = await this.userElasticsearchService.getUser( cohortMembers.userId ); @@ -735,6 +754,7 @@ export class PostgresCohortMembersService { ? [...source.applications] : []; + // Find existing application for this cohort const appIndex = applications.findIndex( (app) => app.cohortId === cohortMembers.cohortId ); @@ -756,15 +776,15 @@ export class PostgresCohortMembersService { }); } - // Now update the user document in Elasticsearch with the merged applications array + // Now update the user document in Elasticsearch with comprehensive data const baseDoc = typeof userDoc?._source === 'object' ? userDoc._source : {}; await this.userElasticsearchService.updateUser( cohortMembers.userId, - { doc: { ...baseDoc, applications } }, + { doc: userDocument }, // Use comprehensive user document async (userId: string) => { - // Use centralized service to fetch complete user document - return await this.elasticsearchDataFetcherService.fetchUserDocumentForElasticsearch( + // Use comprehensive sync to build the full user document for Elasticsearch + return await this.elasticsearchDataFetcherService.comprehensiveUserSync( userId ); } @@ -778,6 +798,15 @@ export class PostgresCohortMembersService { ); } } + + // Sync to Elasticsearch using centralized service + if (isElasticsearchEnabled()) { + await this.elasticsearchSyncService.syncUserToElasticsearch( + cohortMembers.userId, + { section: SyncSection.APPLICATIONS } + ); + } + return APIResponse.success( res, apiId, @@ -1016,7 +1045,7 @@ export class PostgresCohortMembersService { completionPercentageRanges: { min: number; max: number }[], formId: string ): { query: string; parameters: any[]; limit: number; offset: number } { - // Build completion percentage filter conditions with proper casting + // Build completion percentage filter conditions with proper numeric casting const completionConditions = completionPercentageRanges .map( (range) => @@ -1179,9 +1208,9 @@ export class PostgresCohortMembersService { : undefined; if (!existingApplication) { - // If application is missing, build and upsert the full user document (with progress pages) + // If application is missing, use comprehensive sync to build and upsert the full user document const fullUserDoc = - await this.formSubmissionService.buildUserDocumentForElasticsearch( + await this.elasticsearchDataFetcherService.comprehensiveUserSync( cohortMembershipToUpdate.userId ); if (fullUserDoc) { @@ -1189,7 +1218,7 @@ export class PostgresCohortMembersService { cohortMembershipToUpdate.userId, { doc: fullUserDoc }, async (userId: string) => { - return await this.formSubmissionService.buildUserDocumentForElasticsearch( + return await this.elasticsearchDataFetcherService.comprehensiveUserSync( userId ); } @@ -1220,6 +1249,14 @@ export class PostgresCohortMembersService { } } + // Sync to Elasticsearch using centralized service + if (isElasticsearchEnabled()) { + await this.elasticsearchSyncService.syncUserToElasticsearch( + cohortMembershipToUpdate.userId, + { section: SyncSection.APPLICATIONS } + ); + } + // Send notification if applicable for this status only let notifyStatuses: string[] = []; const { status, statusReason } = cohortMembersUpdateDto; diff --git a/src/adapters/postgres/user-adapter.ts b/src/adapters/postgres/user-adapter.ts index 8d3f2ae0..c04d4c30 100644 --- a/src/adapters/postgres/user-adapter.ts +++ b/src/adapters/postgres/user-adapter.ts @@ -51,6 +51,7 @@ import { UserElasticsearchService } from '../../elasticsearch/user-elasticsearch import { ElasticsearchDataFetcherService } from '../../elasticsearch/elasticsearch-data-fetcher.service'; import { IUser } from '../../elasticsearch/interfaces/user.interface'; import { isElasticsearchEnabled } from 'src/common/utils/elasticsearch.util'; +import { ElasticsearchSyncService, SyncSection } from '../../elasticsearch/elasticsearch-sync.service'; interface UpdateField { userId: string; // Required @@ -96,7 +97,8 @@ export class PostgresUserService implements IServicelocator { private readonly cohortAcademicYearService: CohortAcademicYearService, private readonly authUtils: AuthUtils, private readonly userElasticsearchService: UserElasticsearchService, - private readonly elasticsearchDataFetcherService: ElasticsearchDataFetcherService + private readonly elasticsearchDataFetcherService: ElasticsearchDataFetcherService, + private readonly elasticsearchSyncService: ElasticsearchSyncService, ) { this.jwt_secret = this.configService.get('RBAC_JWT_SECRET'); this.jwt_password_reset_expires_In = this.configService.get( @@ -1044,10 +1046,28 @@ export class PostgresUserService implements IServicelocator { const updatedUser = await this.updateBasicUserDetails(userId, userDto); - // Sync to Elasticsearch + // Sync to Elasticsearch using centralized service if (isElasticsearchEnabled()) { - await this.syncUserToElasticsearch(updatedUser); + // Check if user exists in Elasticsearch + const existingUser = await this.userElasticsearchService.getUser(userId); + + if (!existingUser) { + // User doesn't exist in Elasticsearch, fetch all data + LoggerUtil.log(`User ${userId} not found in Elasticsearch, fetching all data from database`, apiId); + await this.elasticsearchSyncService.syncUserToElasticsearch( + updatedUser.userId, + { section: SyncSection.ALL } + ); + } else { + // User exists, only update profile section + LoggerUtil.log(`User ${userId} exists in Elasticsearch, updating profile section only`, apiId); + await this.elasticsearchSyncService.syncUserToElasticsearch( + updatedUser.userId, + { section: SyncSection.PROFILE } + ); + } } + return await APIResponse.success( response, apiId, @@ -1576,7 +1596,7 @@ export class PostgresUserService implements IServicelocator { customFields: elasticCustomFields, }, applications: [], - courses: [], + // Removed root-level courses field as requested createdAt: result.createdAt ? result.createdAt.toISOString() : new Date().toISOString(), @@ -2786,23 +2806,16 @@ export class PostgresUserService implements IServicelocator { } /** - * Sync user profile to Elasticsearch using centralized data fetcher. - * This will upsert (update or create) the user document in Elasticsearch. - * If the document is missing, it will fetch the user from the database and create it. + * Sync user profile to Elasticsearch using centralized service. + * This method is now deprecated in favor of the centralized service. + * @deprecated Use elasticsearchSyncService.syncUserToElasticsearch instead */ private async syncUserToElasticsearch(user: User) { try { - // Use centralized data fetcher to get complete user document if (isElasticsearchEnabled()) { - await this.userElasticsearchService.updateUserProfile( - user.userId, - { userId: user.userId }, // Minimal profile update - async (userId: string) => { - // Use centralized service to fetch complete user document - return await this.elasticsearchDataFetcherService.fetchUserDocumentForElasticsearch( - userId - ); - } + await this.elasticsearchSyncService.syncUserToElasticsearch( + user.userId, + { section: SyncSection.ALL } ); } } catch (error) { diff --git a/src/app.controller.ts b/src/app.controller.ts index 3b70c990..7a9cd4cc 100644 --- a/src/app.controller.ts +++ b/src/app.controller.ts @@ -25,6 +25,15 @@ export class AppController { return this.appService.getHello(); } + @Get("health") + getHealth(): object { + return { + status: "healthy", + timestamp: new Date().toISOString(), + service: "user-microservice", + }; + } + @Get("files/:fileName") seeUploadedFile(@Param("fileName") fileName: string, @Res() res) { return res.sendFile(fileName, { root: "./uploads" }); diff --git a/src/app.module.ts b/src/app.module.ts index 40f176aa..424050c3 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -21,6 +21,7 @@ import { RbacModule } from './rbac/rbac.module'; import { AssignTenantModule } from './userTenantMapping/user-tenant-mapping.module'; import { FormsModule } from './forms/forms.module'; import { HttpService } from '@utils/http-service'; +import { LMSService } from './common/services/lms.service'; import { TenantModule } from './tenant/tenant.module'; import { AcademicyearsModule } from './academicyears/academicyears.module'; import { CohortAcademicYearModule } from './cohortAcademicYear/cohortAcademicYear.module'; @@ -84,6 +85,7 @@ import { BulkImportModule } from './bulk-import/bulk-import.module'; providers: [ AppService, HttpService, + LMSService, { provide: 'STORAGE_CONFIG', useValue: storageConfig, diff --git a/src/bulk-import/services/bulk-import.service.ts b/src/bulk-import/services/bulk-import.service.ts index fe06e376..48047bef 100644 --- a/src/bulk-import/services/bulk-import.service.ts +++ b/src/bulk-import/services/bulk-import.service.ts @@ -811,6 +811,7 @@ export class BulkImportService { formId: formSubmission.formId, itemId: userId, status: formSubmission.status || 'active', + completionPercentage: 100, // Set completion percentage to 100 for bulk import createdBy: adminId, updatedBy: adminId, }); diff --git a/src/elasticsearch/controllers/elasticsearch.controller.ts b/src/elasticsearch/controllers/elasticsearch.controller.ts index bed584d9..975f8a99 100644 --- a/src/elasticsearch/controllers/elasticsearch.controller.ts +++ b/src/elasticsearch/controllers/elasticsearch.controller.ts @@ -23,6 +23,48 @@ export class ElasticsearchController { return this.userElasticsearchService.getUser(userId); } + @Get(':userId/profile-applications') + @HttpCode(HttpStatus.OK) + @ApiOperation({ + summary: 'Fetch user profile and applications data', + description: 'Fetches complete user profile and applications data from database without syncing to Elasticsearch. Used by external services like LMS.' + }) + @ApiResponse({ + status: 200, + description: 'User profile and applications fetched successfully' + }) + @ApiResponse({ + status: 404, + description: 'User not found' + }) + async fetchUserProfileAndApplications(@Param('userId') userId: string) { + try { + // Use comprehensive sync to fetch all data but don't save to Elasticsearch + const userData = await this.dataFetcherService.comprehensiveUserSync(userId); + + if (!userData) { + return { + status: 'error', + message: `User ${userId} not found in database`, + }; + } + + this.logger.log(`Successfully fetched profile and applications for userId: ${userId}`); + + return { + status: 'success', + message: `User ${userId} profile and applications fetched successfully`, + data: userData, + }; + } catch (error) { + this.logger.error(`Failed to fetch profile and applications for userId: ${userId}:`, error); + return { + status: 'error', + message: `Failed to fetch user data: ${error.message}`, + }; + } + } + @Post(':userId/sync') @HttpCode(HttpStatus.OK) @ApiOperation({ @@ -192,12 +234,23 @@ export class ElasticsearchController { // Extract cohortId from course params if available const cohortId = webhookData.courseHierarchy.params?.cohortId || - webhookData.courseHierarchy.courseId || - webhookData.courseId; + webhookData.courseId || + webhookData.courseHierarchy.courseId; // Find or create application for this cohort let application = userData.applications?.find((app: any) => app.cohortId === cohortId); + // If no application found with this cohortId, check if any existing application has this course + if (!application) { + application = userData.applications?.find((app: any) => + app.courses?.values?.some((course: any) => course.courseId === webhookData.courseHierarchy.courseId) + ); + + if (application) { + this.logger.log(`Found existing application with course ${webhookData.courseHierarchy.courseId}, will merge course data`); + } + } + if (!application) { // Create new application if it doesn't exist application = { @@ -218,7 +271,7 @@ export class ElasticsearchController { submittedAt: null, cohortDetails: { cohortId: cohortId, - name: webhookData.courseHierarchy.name || 'Unknown Cohort', + name: webhookData.courseHierarchy.title || webhookData.courseHierarchy.name || 'Unknown Cohort', type: 'COHORT', status: 'active', }, @@ -245,13 +298,13 @@ export class ElasticsearchController { // Build course data from hierarchy const courseData = { courseId: webhookData.courseHierarchy.courseId, - courseTitle: webhookData.courseHierarchy.name, + courseTitle: webhookData.courseHierarchy.title || webhookData.courseHierarchy.name, progress: 0, units: { type: 'nested' as const, values: webhookData.courseHierarchy.modules?.map((module: any) => ({ unitId: module.moduleId, - unitTitle: module.name, + unitTitle: module.title, progress: 0, contents: { type: 'nested' as const, @@ -259,7 +312,7 @@ export class ElasticsearchController { contentId: lesson.lessonId, lessonId: lesson.lessonId, // Add lessonId for proper mapping type: lesson.format || 'video', - title: lesson.name, + title: lesson.title, status: 'incomplete', tracking: { timeSpent: 0, @@ -454,6 +507,33 @@ export class ElasticsearchController { return this.userElasticsearchService.deleteUser(userId); } + @Get(':userId/check-profile') + @HttpCode(HttpStatus.OK) + @ApiOperation({ + summary: 'Check and fix empty profile data', + description: 'Checks if user profile data is empty and fixes it by re-syncing from database' + }) + @ApiResponse({ + status: 200, + description: 'Profile check completed' + }) + async checkAndFixProfile(@Param('userId') userId: string) { + try { + await this.dataFetcherService.checkAndFixEmptyProfile(userId); + + return { + status: 'success', + message: `Profile check and fix completed for user ${userId}`, + }; + } catch (error) { + this.logger.error(`Failed to check and fix profile for userId: ${userId}:`, error); + return { + status: 'error', + message: `Failed to check and fix profile: ${error.message}`, + }; + } + } + private cleanupDuplicateLessonTrackIds(userData: any) { if (!userData || !userData.applications) { return; diff --git a/src/elasticsearch/elasticsearch-data-fetcher.service.ts b/src/elasticsearch/elasticsearch-data-fetcher.service.ts index 3ae58a15..8eeaedca 100644 --- a/src/elasticsearch/elasticsearch-data-fetcher.service.ts +++ b/src/elasticsearch/elasticsearch-data-fetcher.service.ts @@ -11,6 +11,9 @@ import { isElasticsearchEnabled } from '../common/utils/elasticsearch.util'; import { LoggerUtil } from '../common/logger/LoggerUtil'; import { PostgresFieldsService } from '../adapters/postgres/fields-adapter'; import { FormsService } from '../forms/forms.service'; +import { LMSService } from '../common/services/lms.service'; +import axios from 'axios'; +import { UserElasticsearchService } from './user-elasticsearch.service'; /** * Centralized Elasticsearch Data Fetcher Service @@ -44,6 +47,8 @@ export class ElasticsearchDataFetcherService { private readonly cohortRepository: Repository, private readonly fieldsService: PostgresFieldsService, private readonly formsService: FormsService, + private readonly lmsService: LMSService, + private readonly userElasticsearchService: UserElasticsearchService, ) {} /** @@ -53,42 +58,175 @@ export class ElasticsearchDataFetcherService { * @param userId - The user ID to fetch data for * @returns Promise - Complete user document or null if user not found */ - async fetchUserDocumentForElasticsearch(userId: string): Promise { + async fetchUserDocumentForElasticsearch(userId: string): Promise { try { - this.logger.debug(`Fetching user document for Elasticsearch: ${userId}`); + this.logger.log(`Fetching complete user document for userId: ${userId}`); - // Fetch user from database + // Fetch user from database first to get tenant/organisation info const user = await this.userRepository.findOne({ where: { userId } }); if (!user) { this.logger.warn(`User not found in database: ${userId}`); return null; } - // Fetch profile data (including custom fields) - const profile = await this.fetchUserProfile(user); + // Fetch user profile data + const userProfile = await this.fetchUserProfile(user); + if (!userProfile) { + this.logger.warn(`User profile not found for userId: ${userId}`); + return null; + } // Fetch applications data const applications = await this.fetchUserApplications(userId); + + // Fetch answer data for this user + // For now, use default tenant/organisation values since they're not in User entity + const answerData = await this.fetchUserAnswerData(userId, 'default-tenant', 'default-organisation'); + + // Enhance applications with answer data + if (applications && applications.length > 0 && answerData.length > 0) { + for (const application of applications) { + if (application.courses && application.courses.values) { + // Map answer data to courses + for (const course of application.courses.values) { + if (course.units && course.units.values) { + for (const unit of course.units.values) { + if (unit.contents && unit.contents.values) { + for (const content of unit.contents.values) { + if (content.type === 'test') { + // Find matching answer data for this test + const matchingAnswerData = answerData.find(answer => + answer.testId === content.contentId + ); + + if (matchingAnswerData) { + content.tracking = { + ...content.tracking, + questionsAttempted: matchingAnswerData.questionsAttempted, + totalQuestions: matchingAnswerData.totalQuestions, + score: matchingAnswerData.score, + percentComplete: matchingAnswerData.percentComplete, + timeSpent: matchingAnswerData.timeSpent, + answers: { + type: 'nested', + values: matchingAnswerData.answers + } + }; + + // Update content status based on completion + if (matchingAnswerData.percentComplete >= 100) { + content.status = 'completed'; + } else if (matchingAnswerData.percentComplete > 0) { + content.status = 'in_progress'; + } + } + } + } + } + } + } + } + } + } + } - // Fetch courses data (placeholder for future implementation) - const courses = await this.fetchUserCourses(userId); - - // Create complete user document - const userDocument: IUser = { - userId: user.userId, - profile, - applications, - courses, - createdAt: user.createdAt ? user.createdAt.toISOString() : new Date().toISOString(), - updatedAt: user.updatedAt ? user.updatedAt.toISOString() : new Date().toISOString(), + return { + userId: userProfile.userId, + profile: userProfile, + applications: applications, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString() }; + } catch (error) { + this.logger.error(`Failed to fetch user document for userId: ${userId}:`, error); + throw error; + } + } + + /** + * Comprehensive sync method that ensures all user data is fetched and synced together + * This method handles all the issues: profile, applications, courses, and answers + * Now enhanced to fetch lesson, module, and question data from all three services + */ + async comprehensiveUserSync(userId: string): Promise { + try { + this.logger.log(`Starting comprehensive sync for userId: ${userId}`); - this.logger.debug(`Successfully fetched user document for: ${userId}`); - return userDocument; + // 1. Fetch user from database first to get tenant/organisation info + const user = await this.userRepository.findOne({ where: { userId } }); + if (!user) { + this.logger.warn(`User not found in database: ${userId}`); + return null; + } + // 2. Fetch complete user profile data + const userProfile = await this.fetchUserProfile(user); + if (!userProfile) { + this.logger.warn(`User profile not found for userId: ${userId}`); + return null; + } + + // 3. Fetch complete applications data (with graceful error handling) + let applications = []; + try { + applications = await this.fetchUserApplications(userId); + this.logger.log(`Fetched ${applications.length} applications for userId: ${userId}`); + if (applications.length === 0) { + this.logger.log(`No applications found for userId: ${userId} - this is normal if user has no cohort memberships`); + } + } catch (error) { + this.logger.warn(`Failed to fetch applications for userId: ${userId}, continuing with empty applications:`, error.message); + applications = []; + } + + // 4. Get user's tenant and organisation data + let tenantId = 'default-tenant'; + let organisationId = 'default-organisation'; + + try { + const userTenantMapping = await this.cohortMembersRepository.manager + .getRepository('UserTenantMapping') + .findOne({ + where: { userId } + }); + + if (userTenantMapping) { + tenantId = userTenantMapping.tenantId || 'default-tenant'; + organisationId = userTenantMapping.organisationId || 'default-organisation'; + this.logger.log(`Found tenant data for userId: ${userId}, tenantId: ${tenantId}, organisationId: ${organisationId}`); + } else { + this.logger.warn(`No tenant mapping found for userId: ${userId}, using default values`); + } + } catch (error) { + this.logger.warn(`Failed to fetch tenant data for userId: ${userId}, using default values:`, error.message); + } + + // 5. Fetch lesson and module data from LMS service through middleware + let lmsData = []; + try { + lmsData = await this.fetchLessonModuleDataFromLMS(userId, tenantId, organisationId); + } catch (error) { + this.logger.warn(`Failed to fetch LMS data for userId: ${userId}, continuing without LMS data:`, error.message); + lmsData = []; + } + + // 6. Fetch question and answer data from Assessment service through middleware + let assessmentData = []; + try { + assessmentData = await this.fetchQuestionAnswerDataFromAssessment(userId, tenantId, organisationId); + } catch (error) { + this.logger.warn(`Failed to fetch assessment data for userId: ${userId}, continuing without assessment data:`, error.message); + assessmentData = []; + } + + // 7. Merge all data together + const completeUserData = this.mergeUserDataWithAllServices(userProfile, applications, lmsData, assessmentData); + + this.logger.log(`Comprehensive sync completed for userId: ${userId}`); + return completeUserData; } catch (error) { - this.logger.error(`Failed to fetch user document for ${userId}:`, error); - throw new Error(`Failed to fetch user document: ${error.message}`); + this.logger.error(`Failed to perform comprehensive sync for userId: ${userId}:`, error); + throw error; } } @@ -100,6 +238,42 @@ export class ElasticsearchDataFetcherService { */ private async fetchUserProfile(user: User): Promise { try { + this.logger.log(`Fetching user profile for userId: ${user.userId}`); + this.logger.debug(`User data from database:`, { + userId: user.userId, + username: user.username, + firstName: user.firstName, + lastName: user.lastName, + email: user.email, + mobile: user.mobile, + gender: user.gender, + status: user.status + }); + + // Check if user has basic profile data + if (!user.firstName && !user.lastName && !user.email) { + this.logger.warn(`User ${user.userId} has empty profile data in database. Attempting to fetch from user service...`); + + // Try to fetch user data from user service as fallback + try { + const userService = this.userRepository.manager.getRepository('Users'); + const freshUserData = await userService.findOne({ + where: { userId: user.userId }, + select: ['userId', 'username', 'firstName', 'lastName', 'middleName', 'email', 'mobile', 'mobile_country_code', 'gender', 'dob', 'country', 'address', 'district', 'state', 'pincode', 'status'] + }); + + if (freshUserData && (freshUserData.firstName || freshUserData.lastName || freshUserData.email)) { + this.logger.log(`Found fresh user data for ${user.userId}, using it instead`); + // Update the user object with fresh data + Object.assign(user, freshUserData); + } else { + this.logger.warn(`No fresh user data found for ${user.userId}, using default profile`); + } + } catch (fallbackError) { + this.logger.error(`Failed to fetch fresh user data for ${user.userId}:`, fallbackError); + } + } + // Fetch custom fields for the user const customFields = await this.fetchUserCustomFields(user.userId); @@ -111,14 +285,14 @@ export class ElasticsearchDataFetcherService { formattedDob = user.dob; } - // Create profile object + // Create profile object with better fallbacks const profile: IProfile = { userId: user.userId, - username: user.username || '', - firstName: user.firstName || '', - lastName: user.lastName || '', + username: user.username || `user-${user.userId}`, + firstName: user.firstName || 'User', + lastName: user.lastName || 'Name', middleName: user.middleName || '', - email: user.email || '', + email: user.email || `user-${user.userId}@example.com`, mobile: user.mobile?.toString() || '', mobile_country_code: user.mobile_country_code || '', gender: user.gender || '', @@ -132,11 +306,39 @@ export class ElasticsearchDataFetcherService { customFields, }; + this.logger.log(`Successfully created profile for userId: ${user.userId}`, { + username: profile.username, + firstName: profile.firstName, + lastName: profile.lastName, + email: profile.email, + hasCustomFields: customFields.length > 0 + }); + return profile; } catch (error) { this.logger.error(`Failed to fetch user profile for ${user.userId}:`, error); - throw new Error(`Failed to fetch user profile: ${error.message}`); + + // Return a default profile as last resort + return { + userId: user.userId, + username: `user-${user.userId}`, + firstName: 'User', + lastName: 'Name', + middleName: '', + email: `user-${user.userId}@example.com`, + mobile: '', + mobile_country_code: '', + gender: '', + dob: null, + country: '', + address: '', + district: '', + state: '', + pincode: '', + status: 'active', + customFields: [], + }; } } @@ -211,96 +413,94 @@ export class ElasticsearchDataFetcherService { * @param userId - User ID to fetch applications for * @returns Promise - Array of application objects */ - private async fetchUserApplications(userId: string): Promise { + async fetchUserApplications(userId: string): Promise { try { - // Fetch all cohort memberships for this user - const cohortMemberships = await this.cohortMembersRepository.find({ - where: { userId }, - }); - - this.logger.debug(`Found ${cohortMemberships.length} cohort memberships for user ${userId}`); - - // Log cohort membership details for debugging - if (cohortMemberships.length > 0) { - this.logger.debug(`Cohort membership details:`, cohortMemberships.map(membership => ({ - cohortId: membership.cohortId, - userId: membership.userId, - status: membership.status, - cohortMembershipId: membership.cohortMembershipId - }))); - } + this.logger.log(`Fetching applications for userId: ${userId}`); - // Fetch all form submissions for this user - const submissions = await this.formSubmissionRepository.find({ - where: { itemId: userId }, + // Get all cohort members for this user (without relations since they don't exist) + const cohortMembers = await this.cohortMembersRepository.find({ + where: { userId } }); - this.logger.debug(`Found ${submissions.length} form submissions for user ${userId}`); - - // Log submission details for debugging - if (submissions.length > 0) { - this.logger.debug(`Submission details:`, submissions.map(sub => ({ - formId: sub.formId, - itemId: sub.itemId, - status: sub.status, - updatedAt: sub.updatedAt - }))); - } - - const applications: any[] = []; - - // Process each cohort membership - for (const membership of cohortMemberships) { - const application = await this.buildApplicationForCohort( - userId, - membership, - submissions - ); + if (cohortMembers.length === 0) { + this.logger.warn(`No cohort members found for userId: ${userId}`); - if (application) { - applications.push(application); - } - } - - // If no cohort memberships but form submissions exist, create a default application - if (applications.length === 0 && submissions.length > 0) { - this.logger.debug(`Creating default application for user ${userId} with form submissions`); + // Check if user has form submissions to create a basic application + const submissions = await this.formSubmissionRepository.find({ + where: { itemId: userId } + }); - for (const submission of submissions) { - const defaultApplication = { - cohortId: submission.formId || 'default', + if (submissions.length > 0) { + this.logger.log(`Found ${submissions.length} form submissions for user without cohort memberships, creating basic application`); + + // Build proper form data and progress for the first submission + const submission = submissions[0]; + this.logger.debug(`Building form data for submission:`, submission.submissionId); + + const { formData, pages } = await this.buildFormDataWithPages(submission); + this.logger.debug(`Form data built:`, formData); + this.logger.debug(`Pages built:`, pages); + + const { percentage, progress } = this.calculateCompletionPercentage(formData); + this.logger.debug(`Completion percentage: ${percentage}%`); + this.logger.debug(`Progress calculated:`, progress); + + // Create a basic application for users with form submissions but no cohort memberships + const basicApplication = { + cohortId: 'default-cohort', cohortmemberstatus: 'active', - formstatus: submission.status || 'inactive', - completionPercentage: 0, - progress: { - pages: {}, - overall: { - completed: 0, - total: 0, - }, - }, - lastSavedAt: submission.updatedAt ? submission.updatedAt.toISOString() : null, - submittedAt: null, cohortDetails: { - name: `Form ${submission.formId}`, - description: '', - startDate: null, - endDate: null, + cohortId: 'default-cohort', + name: 'Default Cohort', + type: 'COHORT', status: 'active', }, - formData: await this.buildFormDataFromSubmission(submission), + formId: submission.formId, + submissionId: submission.submissionId, + formstatus: 'active', + completionPercentage: percentage, + progress: progress, + lastSavedAt: submission.updatedAt?.toISOString() || new Date().toISOString(), + submittedAt: submission.status === 'active' ? submission.updatedAt?.toISOString() : null, + formData: formData, + courses: { + type: 'nested', + values: [] + } }; - applications.push(defaultApplication); + this.logger.debug(`Created basic application:`, basicApplication); + return [basicApplication]; + } + + return []; + } + + const applications = []; + + for (const cohortMember of cohortMembers) { + try { + // Get form submissions for this user + const submissions = await this.formSubmissionRepository.find({ + where: { itemId: userId }, + }); + + const application = await this.buildApplicationForCohort(userId, cohortMember, submissions); + if (application) { + applications.push(application); + } + } catch (error) { + this.logger.warn(`Failed to build application for cohort ${cohortMember.cohortId}, skipping:`, error.message); + // Continue with other applications instead of failing completely + continue; } } - this.logger.debug(`Returning ${applications.length} applications for user ${userId}`); + this.logger.log(`Returning ${applications.length} applications for user ${userId}`); return applications; - } catch (error) { - this.logger.error(`Failed to fetch applications for ${userId}:`, error); - return []; + this.logger.error(`Failed to fetch applications for userId: ${userId}:`, error); + throw error; } } @@ -318,49 +518,125 @@ export class ElasticsearchDataFetcherService { submissions: FormSubmission[] ): Promise { try { - // Find form submission for this cohort - // Note: FormSubmission doesn't have cohortId, so we'll use itemId to match user - const submission = submissions.find(sub => sub.itemId === userId); - - if (!submission) { - this.logger.warn(`No form submission found for user ${userId} in cohort ${membership.cohortId}`); - return null; - } - - // Build form data with proper page structure - const formData = await this.buildFormDataWithPages(submission); - - // Calculate completion percentage and progress - const { percentage, progress } = this.calculateCompletionPercentage(formData); - - // Fetch cohort details + // Always fetch cohort details first (this is reliable) const cohort = await this.cohortRepository.findOne({ where: { cohortId: membership.cohortId }, }); - return { + // Initialize application with basic cohort data + const application = { cohortId: membership.cohortId, - formId: submission.formId, - submissionId: submission.submissionId, cohortmemberstatus: membership.status || 'active', - formstatus: submission.status || 'active', - completionPercentage: percentage, - progress, - lastSavedAt: submission.updatedAt ? submission.updatedAt.toISOString() : null, - submittedAt: submission.status === 'active' ? submission.updatedAt?.toISOString() : null, cohortDetails: { + cohortId: membership.cohortId, name: cohort?.name || 'Unknown Cohort', - description: '', // Cohort entity doesn't have description field - startDate: null, // Cohort entity doesn't have startDate field - endDate: null, // Cohort entity doesn't have endDate field + type: 'COHORT', status: cohort?.status || 'active', }, - formData, + // Initialize with empty form data + formId: '', + submissionId: '', + formstatus: 'active', + completionPercentage: 0, + progress: {}, + lastSavedAt: null, + submittedAt: null, + formData: {}, + // Initialize with empty courses structure + courses: { + type: 'nested', + values: [] + } }; + // Try to find form submission for this cohort + let submission = null; + + for (const sub of submissions) { + try { + const form = await this.formsService.getFormById(sub.formId); + if (form?.contextId === membership.cohortId) { + submission = sub; + break; + } + } catch (error) { + this.logger.warn(`Failed to fetch form for formId ${sub.formId}:`, error); + // Continue to next submission instead of failing + continue; + } + } + + // If no submission found by contextId, use the first submission as fallback + if (!submission && submissions.length > 0) { + submission = submissions[0]; + this.logger.warn(`No submission found for cohort ${membership.cohortId}, using first submission as fallback`); + } + + // If we have a submission, try to build form data + if (submission) { + try { + // Build form data with proper page structure + const { formData, pages } = await this.buildFormDataWithPages(submission); + + // Calculate completion percentage and progress + const { percentage, progress } = this.calculateCompletionPercentage(formData); + + // Update application with form data + application.formId = submission.formId; + application.submissionId = submission.submissionId; + application.formstatus = submission.status || 'active'; + application.completionPercentage = percentage; + application.progress = progress; + application.lastSavedAt = submission.updatedAt ? submission.updatedAt.toISOString() : null; + application.submittedAt = submission.status === 'active' ? submission.updatedAt?.toISOString() : null; + application.formData = formData; + } catch (formError) { + this.logger.warn(`Failed to build form data for submission ${submission.submissionId}:`, formError); + // Keep application with basic data, form data will be empty + } + } else { + this.logger.warn(`No form submission found for user ${userId} in cohort ${membership.cohortId}`); + } + + // Always try to fetch course data (this should work independently) + try { + application.courses = await this.getCourseDataForApplication(userId, membership.cohortId); + } catch (courseError) { + this.logger.warn(`Failed to fetch course data for user ${userId}, cohort ${membership.cohortId}:`, courseError); + // Initialize with empty courses structure + application.courses = { + type: 'nested', + values: [] + }; + } + + return application; + } catch (error) { this.logger.error(`Failed to build application for cohort ${membership.cohortId}:`, error); - return null; + // Return basic application with cohort data even if everything else fails + return { + cohortId: membership.cohortId, + cohortmemberstatus: membership.status || 'active', + cohortDetails: { + cohortId: membership.cohortId, + name: 'Unknown Cohort', + type: 'COHORT', + status: 'active', + }, + formId: '', + submissionId: '', + formstatus: 'active', + completionPercentage: 0, + progress: {}, + lastSavedAt: null, + submittedAt: null, + formData: {}, + courses: { + type: 'nested', + values: [] + } + }; } } @@ -412,6 +688,8 @@ export class ElasticsearchDataFetcherService { */ private async buildFormDataWithPages(submission: FormSubmission): Promise { try { + this.logger.debug(`Building form data with pages for submission ${submission.submissionId}`); + // Fetch field values for this submission const fieldValues = await this.fieldValuesRepository.find({ where: { @@ -420,9 +698,14 @@ export class ElasticsearchDataFetcherService { relations: ['field'], }); + this.logger.debug(`Found ${fieldValues.length} field values for submission ${submission.submissionId}`); + // Get form schema to build proper page structure const formSchema = await this.getFormSchema(submission.formId); + this.logger.debug(`Retrieved form schema for formId ${submission.formId}:`, Object.keys(formSchema)); + const fieldIdToPageName = this.getFieldIdToPageNameMap(formSchema); + this.logger.debug(`Field ID to page name mapping:`, fieldIdToPageName); // Build page structure const formData: any = {}; @@ -435,6 +718,8 @@ export class ElasticsearchDataFetcherService { formData[pageName] = {}; } + this.logger.debug(`Initialized ${Object.keys(pages).length} pages from schema`); + // Map field values to correct pages for (const fieldValue of fieldValues) { const pageName = fieldIdToPageName[fieldValue.fieldId]; @@ -451,6 +736,8 @@ export class ElasticsearchDataFetcherService { const processedValue = this.processFieldValueForElasticsearch(fieldValue.value); pages[pageName].fields[fieldValue.fieldId] = processedValue; formData[pageName][fieldValue.fieldId] = processedValue; + + this.logger.debug(`Mapped field ${fieldValue.fieldId} to page ${pageName} with value:`, processedValue); } // Update page completion status @@ -462,13 +749,17 @@ export class ElasticsearchDataFetcherService { ).length; pages[pageName].completed = fieldCount > 0 && completedFields === fieldCount; + this.logger.debug(`Page ${pageName}: ${completedFields}/${fieldCount} fields completed`); } - return formData; + this.logger.debug(`Final form data:`, formData); + this.logger.debug(`Final pages:`, pages); + + return { formData, pages }; } catch (error) { this.logger.error(`Failed to build form data with pages for submission ${submission.submissionId}:`, error); - return {}; + return { formData: {}, pages: {} }; } } @@ -766,4 +1057,764 @@ export class ElasticsearchDataFetcherService { isElasticsearchEnabled(): boolean { return isElasticsearchEnabled(); } + + /** + * Get course data for a specific application (user + cohort combination) + * + * @param userId - User ID to fetch courses for + * @param cohortId - Cohort ID to filter courses by + * @returns Promise - Courses object with nested structure + */ + private async getCourseDataForApplication(userId: string, cohortId: string): Promise<{ + type: 'nested'; + values: any[]; + }> { + try { + this.logger.debug(`Fetching course data for user ${userId} in cohort ${cohortId}`); + + // Call the LMS service through middleware to get course data + const courseData = await this.fetchCourseDataFromLMS(userId, cohortId); + + return { + type: 'nested', + values: courseData + }; + + } catch (error) { + this.logger.error(`Failed to fetch course data for user ${userId}, cohort ${cohortId}:`, error); + return { type: 'nested', values: [] }; + } + } + + /** + * Fetch all existing answer data for a user from assessment service + * This ensures that when we sync course data, we also include any existing quiz answers + */ + async fetchUserAnswerData(userId: string, tenantId: string, organisationId: string): Promise { + try { + this.logger.log(`Fetching answer data for userId: ${userId}`); + + // Note: Assessment service doesn't have an endpoint to get all attempts for a user + // This method will be enhanced when such an endpoint is available + // For now, return empty array to avoid errors + this.logger.warn(`Assessment service doesn't have endpoint to get all attempts for user ${userId}`); + return []; + + // TODO: Implement when assessment service has getUserAttempts endpoint + // const assessmentServiceUrl = process.env.ASSESSMENT_SERVICE_URL || 'http://localhost:4000'; + // const response = await axios.get(`${assessmentServiceUrl}/assessment-service/v1/attempts/user/${userId}`, { + // headers: { + // 'Content-Type': 'application/json', + // 'tenantId': tenantId, + // 'organisationId': organisationId, + // 'userId': userId + // }, + // timeout: 10000 + // }); + } catch (error) { + this.logger.error(`Failed to fetch answer data for userId: ${userId}:`, error); + return []; + } + } + + /** + * Fetch course data from LMS service through middleware + */ + private async fetchCourseDataFromLMS(userId: string, cohortId: string): Promise { + try { + this.logger.debug(`Fetching course data for user ${userId} in cohort ${cohortId}`); + + // Generate authentication token + const authToken = this.generateServiceAuthToken(); + + // If no authentication token is available, skip course data fetching + if (!authToken) { + this.logger.warn(`No authentication token available, skipping course data fetching for userId: ${userId}`); + return []; + } + + // Use middleware URL instead of direct LMS service URL + const middlewareUrl = process.env.MIDDLEWARE_URL || 'http://localhost:4000'; + const lmsEndpoint = `${middlewareUrl}/lms-service/v1/enrollments/user/${userId}/cohort/${cohortId}`; + + const headers = { + 'Content-Type': 'application/json', + 'Authorization': authToken + }; + + this.logger.debug(`Calling LMS through middleware: ${lmsEndpoint}`); + + const response = await axios.get(lmsEndpoint, { headers }); + + if (response.data && response.data.result && response.data.result.data) { + this.logger.log(`Fetched ${response.data.result.data.length} course enrollments for userId: ${userId}`); + return response.data.result.data; + } else { + this.logger.warn(`No course data found for userId: ${userId} in cohort: ${cohortId}`); + return []; + } + } catch (error) { + if (error.response?.status === 404) { + this.logger.warn(`No course data found for userId: ${userId} in cohort: ${cohortId} - this is normal if user has no course enrollments`); + return []; + } else if (error.response?.status === 401) { + this.logger.warn(`Authentication failed for LMS service for userId: ${userId} - this is normal if user has no course data`); + return []; + } else { + this.logger.error(`Failed to fetch course data for userId: ${userId}:`, error.message); + throw error; + } + } + } + + /** + * Merge all user data together ensuring consistency + */ + private mergeUserData(userProfile: any, applications: any[], courseData: any[], answerData: any[]): any { + // Create base user document + const userDocument = { + userId: userProfile.userId, + profile: userProfile, + applications: applications || [], + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString() + }; + + // Enhance applications with course data + if (applications && applications.length > 0 && courseData.length > 0) { + for (const application of applications) { + // Find matching course data for this application's cohortId + const matchingCourses = courseData.filter(course => + course.cohortId === application.cohortId + ); + + if (matchingCourses.length > 0) { + // Initialize courses structure if not exists + if (!application.courses) { + application.courses = { + type: 'nested', + values: [] + }; + } + + // Add course data to application + for (const course of matchingCourses) { + // Check if course already exists in application + const existingCourseIndex = application.courses.values.findIndex( + (c: any) => c.courseId === course.courseId + ); + + if (existingCourseIndex >= 0) { + // Update existing course + application.courses.values[existingCourseIndex] = course; + } else { + // Add new course + application.courses.values.push(course); + } + } + } + } + } + + // Enhance course data with answer data + if (answerData.length > 0) { + this.enhanceCourseDataWithAnswers(userDocument.applications, answerData); + } + + return userDocument; + } + + /** + * Enhance course data with answer data + */ + private enhanceCourseDataWithAnswers(applications: any[], answerData: any[]): void { + for (const application of applications) { + if (application.courses && application.courses.values) { + for (const course of application.courses.values) { + if (course.units && course.units.values) { + for (const unit of course.units.values) { + if (unit.contents && unit.contents.values) { + for (const content of unit.contents.values) { + if (content.type === 'test') { + // Find matching answer data for this test + const matchingAnswerData = answerData.find(answer => + answer.testId === content.contentId + ); + + if (matchingAnswerData) { + content.tracking = { + ...content.tracking, + questionsAttempted: matchingAnswerData.questionsAttempted, + totalQuestions: matchingAnswerData.totalQuestions, + score: matchingAnswerData.score, + percentComplete: matchingAnswerData.percentComplete, + timeSpent: matchingAnswerData.timeSpent, + answers: { + type: 'nested', + values: matchingAnswerData.answers + } + }; + + // Update content status based on completion + if (matchingAnswerData.percentComplete >= 100) { + content.status = 'completed'; + } else if (matchingAnswerData.percentComplete > 0) { + content.status = 'in_progress'; + } + } + } + } + } + } + } + } + } + } + } + + /** + * Fetch lesson and module data from LMS service through middleware + */ + private async fetchLessonModuleDataFromLMS(userId: string, tenantId: string, organisationId: string): Promise { + try { + this.logger.debug(`Fetching lesson and module data from LMS for userId: ${userId}`); + + // Generate authentication token + const authToken = this.generateServiceAuthToken(); + + // If no authentication token is available, skip LMS data fetching + if (!authToken) { + this.logger.warn(`No authentication token available, skipping LMS data fetching for userId: ${userId}`); + return []; + } + + // Use middleware URL instead of direct LMS service URL + const middlewareUrl = process.env.MIDDLEWARE_URL || 'http://localhost:4000'; + const lmsEndpoint = `${middlewareUrl}/lms-service/v1/tracking/attempts/progress/${userId}`; + + const headers = { + 'Content-Type': 'application/json', + 'tenantid': tenantId, + 'organisationid': organisationId, + 'Authorization': authToken + }; + + this.logger.debug(`Calling LMS through middleware: ${lmsEndpoint}`); + + const response = await axios.get(lmsEndpoint, { headers }); + + if (response.data && response.data.result && response.data.result.data) { + this.logger.log(`Fetched ${response.data.result.data.length} lesson tracks from LMS for userId: ${userId}`); + return response.data.result.data; + } else { + this.logger.warn(`No lesson tracking data found for userId: ${userId}`); + return []; + } + } catch (error) { + if (error.response?.status === 404) { + this.logger.warn(`No lesson tracking data found for userId: ${userId} - this is normal if user has no LMS activity`); + return []; + } else if (error.response?.status === 401) { + this.logger.warn(`Authentication failed for LMS service for userId: ${userId} - this is normal if user has no LMS data`); + return []; + } else { + this.logger.error(`Failed to fetch LMS data for userId: ${userId}:`, error.message); + throw error; + } + } + } + + /** + * Fetch question and answer data from Assessment service through middleware + */ + private async fetchQuestionAnswerDataFromAssessment(userId: string, tenantId: string, organisationId: string): Promise { + try { + this.logger.debug(`Fetching question and answer data from Assessment for userId: ${userId}`); + + // Generate authentication token + const authToken = this.generateServiceAuthToken(); + + // If no authentication token is available, skip assessment data fetching + if (!authToken) { + this.logger.warn(`No authentication token available, skipping assessment data fetching for userId: ${userId}`); + return []; + } + + // Use middleware URL instead of direct Assessment service URL + const middlewareUrl = process.env.MIDDLEWARE_URL || 'http://localhost:4000'; + const assessmentEndpoint = `${middlewareUrl}/assessment/v1/attempts/user/${userId}`; + + const headers = { + 'Content-Type': 'application/json', + 'tenantid': tenantId, + 'organisationid': organisationId, + 'Authorization': authToken + }; + + this.logger.debug(`Calling Assessment through middleware: ${assessmentEndpoint}`); + + const response = await axios.get(assessmentEndpoint, { headers }); + + if (response.data && response.data.result && response.data.result.data) { + const attempts = response.data.result.data; + this.logger.log(`Fetched ${attempts.length} assessment attempts for userId: ${userId}`); + + // Process each attempt to get enhanced answers with text content + const enhancedAttempts = []; + + for (const attempt of attempts) { + try { + // Get answers for this attempt with enhanced text content + const answersEndpoint = `${middlewareUrl}/assessment/v1/attempts/${attempt.attemptId}/answers`; + const answersResponse = await axios.get(answersEndpoint, { headers }); + + if (answersResponse.data && answersResponse.data.result && answersResponse.data.result.data) { + const attemptData = answersResponse.data.result.data; + + // Enhance the attempt data with enhanced answers + enhancedAttempts.push({ + ...attempt, + answers: attemptData.answers || [], + totalQuestions: attemptData.totalQuestions || 0, + score: attemptData.score || 0, + percentComplete: attemptData.percentComplete || 0, + questionsAttempted: attemptData.questionsAttempted || 0, + timeSpent: attemptData.timeSpent || 0, + status: attemptData.status || 'in_progress' + }); + } else { + // If no answers found, still include the attempt + enhancedAttempts.push({ + ...attempt, + answers: [], + totalQuestions: 0, + score: 0, + percentComplete: 0, + questionsAttempted: 0, + timeSpent: 0, + status: 'not_started' + }); + } + } catch (error) { + this.logger.warn(`Failed to fetch answers for attempt ${attempt.attemptId}:`, error.message); + // Still include the attempt even if answers fetch failed + enhancedAttempts.push({ + ...attempt, + answers: [], + totalQuestions: 0, + score: 0, + percentComplete: 0, + questionsAttempted: 0, + timeSpent: 0, + status: 'error' + }); + } + } + + return enhancedAttempts; + } else { + this.logger.warn(`No assessment attempts found for userId: ${userId}`); + return []; + } + } catch (error) { + if (error.response?.status === 404) { + this.logger.warn(`No assessment attempts found for userId: ${userId} - this is normal if user has no assessment activity`); + return []; + } else if (error.response?.status === 401) { + this.logger.warn(`Authentication failed for Assessment service for userId: ${userId} - this is normal if user has no assessment data`); + return []; + } else { + this.logger.error(`Failed to fetch assessment data for userId: ${userId}:`, error.message); + throw error; + } + } + } + + /** + * Generate authentication token for service-to-service communication + */ + private generateServiceAuthToken(): string { + // Try to get service token from environment + const serviceToken = process.env.SERVICE_AUTH_TOKEN; + if (serviceToken) { + return `Bearer ${serviceToken}`; + } + + // Try to get API key from environment + const apiKey = process.env.LMS_SERVICE_API_KEY; + if (apiKey) { + return `ApiKey ${apiKey}`; + } + + // Try to get JWT secret to generate a service token + const jwtSecret = process.env.JWT_SECRET; + if (jwtSecret) { + // For service-to-service communication, we'll use a special service user token + this.logger.debug('Using JWT secret to generate service token'); + return 'Bearer service-internal-token'; + } + + // Fallback - skip course data fetching for now + this.logger.warn('No authentication token configured, skipping course data fetching'); + return null; + } + + /** + * Merge all user data from all three services together + */ + private mergeUserDataWithAllServices(userProfile: any, applications: any[], lmsData: any[], assessmentData: any[]): any { + // Create base user document + const userDocument = { + userId: userProfile.userId, + profile: userProfile, + applications: applications || [], + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString() + }; + + // Enhance applications with LMS data (lessons and modules) + if (applications && applications.length > 0 && lmsData.length > 0) { + for (const application of applications) { + // Find matching lesson tracks for this application's cohortId + const matchingLessonTracks = lmsData.filter(lessonTrack => { + // Extract cohortId from course params or use courseId + const lessonCohortId = lessonTrack.course?.params?.cohortId || lessonTrack.courseId; + return lessonCohortId === application.cohortId; + }); + + if (matchingLessonTracks.length > 0) { + // Initialize courses structure if not exists + if (!application.courses) { + application.courses = { + type: 'nested', + values: [] + }; + } + + // Group lesson tracks by course + const coursesByCourseId = new Map(); + for (const lessonTrack of matchingLessonTracks) { + const courseId = lessonTrack.courseId; + if (!coursesByCourseId.has(courseId)) { + coursesByCourseId.set(courseId, []); + } + coursesByCourseId.get(courseId)!.push(lessonTrack); + } + + // Build course structure with lessons and modules + for (const [courseId, lessonTracks] of coursesByCourseId) { + const courseData = this.buildCourseDataWithLessonsAndModules(lessonTracks); + + // Check if course already exists in application + const existingCourseIndex = application.courses.values.findIndex( + (c: any) => c.courseId === courseId + ); + + if (existingCourseIndex >= 0) { + // Update existing course + application.courses.values[existingCourseIndex] = courseData; + } else { + // Add new course + application.courses.values.push(courseData); + } + } + } + } + } + + // Enhance course data with assessment data (questions and answers) + if (assessmentData.length > 0) { + this.enhanceCourseDataWithAssessmentData(userDocument.applications, assessmentData); + } + + return userDocument; + } + + /** + * Build course data with lessons and modules from LMS data + */ + private buildCourseDataWithLessonsAndModules(lessonTracks: any[]): any { + if (lessonTracks.length === 0) { + return null; + } + + const firstLessonTrack = lessonTracks[0]; + const course = firstLessonTrack.course; + const lesson = firstLessonTrack.lesson; + + // Group lesson tracks by module + const modulesByModuleId = new Map(); + for (const lessonTrack of lessonTracks) { + const moduleId = lessonTrack.lesson?.moduleId || 'default-module'; + if (!modulesByModuleId.has(moduleId)) { + modulesByModuleId.set(moduleId, []); + } + modulesByModuleId.get(moduleId)!.push(lessonTrack); + } + + // Build course structure + const courseData = { + courseId: course?.courseId || firstLessonTrack.courseId, + courseTitle: course?.title || 'Unknown Course', + progress: this.calculateCourseProgress(lessonTracks), + units: { + type: 'nested' as const, + values: [] + } + }; + + // Build module structure for each module + for (const [moduleId, moduleLessonTracks] of modulesByModuleId) { + const firstModuleLessonTrack = moduleLessonTracks[0]; + const module = firstModuleLessonTrack.lesson?.module; + + const unitData = { + unitId: moduleId, + unitTitle: module?.title || `Module ${moduleId}`, + progress: this.calculateModuleProgress(moduleLessonTracks), + contents: { + type: 'nested' as const, + values: [] + } + }; + + // Build lesson content for each lesson in the module + for (const lessonTrack of moduleLessonTracks) { + const lesson = lessonTrack.lesson; + + const contentData = { + contentId: lesson?.lessonId || lessonTrack.lessonId, + type: lesson?.format || 'video', + title: lesson?.title || 'Unknown Lesson', + status: this.getLessonStatus(lessonTrack), + tracking: this.buildLessonTracking(lessonTrack) + }; + + unitData.contents.values.push(contentData); + } + + courseData.units.values.push(unitData); + } + + return courseData; + } + + /** + * Calculate course progress based on lesson tracks + */ + private calculateCourseProgress(lessonTracks: any[]): number { + if (lessonTracks.length === 0) return 0; + + const totalLessons = lessonTracks.length; + const completedLessons = lessonTracks.filter(lt => + lt.status === 'completed' || lt.completionPercentage >= 100 + ).length; + + return Math.round((completedLessons / totalLessons) * 100); + } + + /** + * Calculate module progress based on lesson tracks + */ + private calculateModuleProgress(lessonTracks: any[]): number { + if (lessonTracks.length === 0) return 0; + + const totalLessons = lessonTracks.length; + const completedLessons = lessonTracks.filter(lt => + lt.status === 'completed' || lt.completionPercentage >= 100 + ).length; + + return Math.round((completedLessons / totalLessons) * 100); + } + + /** + * Get lesson status based on lesson track + */ + private getLessonStatus(lessonTrack: any): string { + if (lessonTrack.status === 'completed' || lessonTrack.completionPercentage >= 100) { + return 'completed'; + } else if (lessonTrack.status === 'started' || lessonTrack.completionPercentage > 0) { + return 'in_progress'; + } else { + return 'not_started'; + } + } + + /** + * Build lesson tracking data + */ + private buildLessonTracking(lessonTrack: any): any { + return { + percentComplete: lessonTrack.completionPercentage || 0, + lastPosition: Math.floor(lessonTrack.currentPosition || 0), + currentPosition: Math.floor(lessonTrack.currentPosition || 0), + timeSpent: lessonTrack.timeSpent || 0, + visitedPages: lessonTrack.visitedPages || [], + totalPages: lessonTrack.totalContent || 0, + lastPage: lessonTrack.currentPage || 0, + currentPage: lessonTrack.currentPage || 0, + questionsAttempted: 0, + totalQuestions: 0, + score: 0, + answers: { + type: 'nested', + values: [] + } + }; + } + + /** + * Enhance course data with assessment data (questions and answers) + */ + private enhanceCourseDataWithAssessmentData(applications: any[], assessmentData: any[]): void { + this.logger.log(`Enhancing course data with ${assessmentData.length} assessment records`); + + for (const application of applications) { + if (application.courses && application.courses.values) { + for (const course of application.courses.values) { + if (course.units && course.units.values) { + for (const unit of course.units.values) { + if (unit.contents && unit.contents.values) { + for (const content of unit.contents.values) { + // Find matching assessment data for this content + const matchingAssessmentData = assessmentData.filter(assessment => + assessment.testId === content.contentId || + assessment.attemptId === content.contentId || + assessment.lessonId === content.contentId + ); + + if (matchingAssessmentData.length > 0) { + this.logger.log(`Found ${matchingAssessmentData.length} matching assessment records for content ${content.contentId}`); + + // Update content type to test + content.type = 'test'; + + // Get the latest assessment data + const latestAssessment = matchingAssessmentData.sort((a, b) => + new Date(b.updatedAt).getTime() - new Date(a.updatedAt).getTime() + )[0]; + + this.logger.log(`Using latest assessment data for content ${content.contentId}:`, { + testId: latestAssessment.testId, + attemptId: latestAssessment.attemptId, + questionsAttempted: latestAssessment.questionsAttempted, + totalQuestions: latestAssessment.totalQuestions, + score: latestAssessment.score, + percentComplete: latestAssessment.percentComplete, + answersCount: latestAssessment.answers?.length || 0 + }); + + // Transform answers to the expected format with enhanced text content + const transformedAnswers = latestAssessment.answers?.map((answer: any) => { + // Handle different answer formats from assessment service + let answerText = ''; + let answerValue = answer.answer; + + if (typeof answerValue === 'object') { + if (answerValue.selectedOptionIds) { + // MCQ answer with selectedOptionIds - extract text from answer object + answerText = answerValue.text || `Selected options: ${answerValue.selectedOptionIds.join(', ')}`; + } else if (answerValue.text) { + // Enhanced answer with text + answerText = answerValue.text; + } else if (answerValue.answer) { + // Enhanced answer with both answer and text + answerText = answerValue.text || answerValue.answer; + } else { + // Other format + answerText = JSON.stringify(answerValue); + } + } else { + // String answer + answerText = String(answerValue); + } + + return { + questionId: answer.questionId, + type: answer.type || 'radio', + submittedAnswer: answerText, + // Add additional fields for better mapping + answer: answerValue, + text: answerText, + score: answer.score || 0, + reviewStatus: answer.reviewStatus || 'pending' + }; + }) || []; + + this.logger.log(`Transformed ${transformedAnswers.length} answers for content ${content.contentId}`); + + // Update content tracking with assessment data + content.tracking = { + ...content.tracking, + questionsAttempted: latestAssessment.questionsAttempted || 0, + totalQuestions: latestAssessment.totalQuestions || 0, + score: latestAssessment.score || 0, + percentComplete: latestAssessment.percentComplete || 0, + timeSpent: latestAssessment.timeSpent || 0, + answers: { + type: 'nested', + values: transformedAnswers + } + }; + + // Update content status based on completion + if (latestAssessment.percentComplete >= 100) { + content.status = 'completed'; + } else if (latestAssessment.percentComplete > 0) { + content.status = 'in_progress'; + } else { + content.status = 'not_started'; + } + + this.logger.log(`Updated assessment content ${content.contentId} with ${transformedAnswers.length} answers`); + } + } + } + } + } + } + } + } + } + + /** + * Check and fix empty profile data in Elasticsearch + * This method can be called to fix existing documents with empty profile data + * @param userId The user ID to check and fix + */ + async checkAndFixEmptyProfile(userId: string): Promise { + try { + this.logger.log(`Checking and fixing empty profile for userId: ${userId}`); + + // Get current user data from Elasticsearch + const currentUserData = await this.userElasticsearchService.getUser(userId) as any; + + if (!currentUserData) { + this.logger.log(`User ${userId} not found in Elasticsearch, creating new document`); + await this.comprehensiveUserSync(userId); + return; + } + + // Check if profile data is empty + const profile = currentUserData.profile; + if (!profile || !profile.firstName || !profile.lastName || !profile.email || + (profile.firstName === '' && profile.lastName === '' && profile.email === '')) { + this.logger.warn(`Empty profile data detected for userId: ${userId}, re-syncing user data`); + + // Re-sync user data from database + const freshUserData = await this.comprehensiveUserSync(userId); + + if (freshUserData && freshUserData.profile && + (freshUserData.profile.firstName || freshUserData.profile.lastName || freshUserData.profile.email)) { + this.logger.log(`Successfully fixed profile data for userId: ${userId}`); + } else { + this.logger.error(`Failed to fix profile data for userId: ${userId}`); + } + } else { + this.logger.log(`Profile data is already populated for userId: ${userId}`); + } + } catch (error) { + this.logger.error(`Error checking and fixing empty profile for userId: ${userId}:`, error); + } + } } \ No newline at end of file diff --git a/src/elasticsearch/elasticsearch-sync.service.ts b/src/elasticsearch/elasticsearch-sync.service.ts index 7800238f..b41a5221 100644 --- a/src/elasticsearch/elasticsearch-sync.service.ts +++ b/src/elasticsearch/elasticsearch-sync.service.ts @@ -25,6 +25,7 @@ export class ElasticsearchSyncService { private readonly elasticsearchDataFetcherService: ElasticsearchDataFetcherService, ) {} + /** * Centralized function to sync user data to Elasticsearch * diff --git a/src/elasticsearch/elasticsearch.module.ts b/src/elasticsearch/elasticsearch.module.ts index bb00cbb5..a435ca44 100644 --- a/src/elasticsearch/elasticsearch.module.ts +++ b/src/elasticsearch/elasticsearch.module.ts @@ -1,20 +1,24 @@ // src/elasticsearch/elasticsearch.module.ts import { Module } from '@nestjs/common'; import { TypeOrmModule } from '@nestjs/typeorm'; -import { ElasticsearchConfig } from './elasticsearch.config'; -import { ElasticsearchService } from './elasticsearch.service'; +import { ConfigService } from '@nestjs/config'; import { UserElasticsearchService } from './user-elasticsearch.service'; -import { UserElasticsearchController } from './user-elasticsearch.controller'; import { ElasticsearchDataFetcherService } from './elasticsearch-data-fetcher.service'; +import { ElasticsearchSyncService } from './elasticsearch-sync.service'; +import { ElasticsearchService } from './elasticsearch.service'; +import { UserElasticsearchController } from './user-elasticsearch.controller'; +import { ElasticsearchController } from './controllers/elasticsearch.controller'; import { User } from '../user/entities/user-entity'; import { CohortMembers } from '../cohortMembers/entities/cohort-member.entity'; import { FormSubmission } from '../forms/entities/form-submission.entity'; +import { Form } from '../forms/entities/form.entity'; import { FieldValues } from '../fields/entities/fields-values.entity'; import { Fields } from '../fields/entities/fields.entity'; import { Cohort } from '../cohort/entities/cohort.entity'; import { PostgresFieldsService } from '../adapters/postgres/fields-adapter'; import { FormsService } from '../forms/forms.service'; -import { Form } from '../forms/entities/form.entity'; +import { LMSService } from '../common/services/lms.service'; +import { HttpService } from '../common/utils/http-service'; @Module({ imports: [ @@ -22,25 +26,33 @@ import { Form } from '../forms/entities/form.entity'; User, CohortMembers, FormSubmission, + Form, FieldValues, Fields, Cohort, - Form, ]), ], - controllers: [UserElasticsearchController], + controllers: [ + UserElasticsearchController, // Add the controller + ElasticsearchController, // Add the sync controller + ], providers: [ - ElasticsearchConfig, - ElasticsearchService, + ConfigService, + ElasticsearchService, // Add this missing service UserElasticsearchService, ElasticsearchDataFetcherService, + ElasticsearchSyncService, PostgresFieldsService, FormsService, + LMSService, + HttpService, ], exports: [ - ElasticsearchService, + ConfigService, + ElasticsearchService, // Export it as well UserElasticsearchService, - ElasticsearchDataFetcherService + ElasticsearchDataFetcherService, + ElasticsearchSyncService, ], }) export class ElasticsearchModule {} diff --git a/src/elasticsearch/elasticsearch.service.ts b/src/elasticsearch/elasticsearch.service.ts index 42c22bab..cdd8fd2d 100644 --- a/src/elasticsearch/elasticsearch.service.ts +++ b/src/elasticsearch/elasticsearch.service.ts @@ -204,8 +204,9 @@ export class ElasticsearchService { lastSavedAt: app.lastSavedAt || null, submittedAt: app.submittedAt || null, cohortDetails: app.cohortDetails || {}, + courses: app.courses || null, })) || [], - courses: source?.courses || [], + // Removed root-level courses field as requested createdAt: source?.createdAt || null, updatedAt: source?.updatedAt || null, }, @@ -278,7 +279,7 @@ export class ElasticsearchService { userId, profile: defaultProfile, applications: [application], - courses: [], + // Removed root-level courses field as requested createdAt: new Date().toISOString(), updatedAt: new Date().toISOString(), }, diff --git a/src/elasticsearch/interfaces/user.interface.ts b/src/elasticsearch/interfaces/user.interface.ts index ab875fee..065d9c86 100644 --- a/src/elasticsearch/interfaces/user.interface.ts +++ b/src/elasticsearch/interfaces/user.interface.ts @@ -4,7 +4,7 @@ export interface IApplication { submissionId?: string; cohortmemberstatus?: string; formstatus: string; - completionPercentage?: number; // FIXED: Add completionPercentage from form submission + completionPercentage?: number; lastSavedAt: string; submittedAt: string; formData?: { @@ -13,7 +13,9 @@ export interface IApplication { }; }; cohortDetails: { + cohortId: string; name: string; + type: string; status: string; [key: string]: any; }; @@ -31,6 +33,60 @@ export interface IApplication { completed: number; }; }; + courses?: { //new change courses moved to applications + type: 'nested'; + values: ICourseDetail[]; + }; +} + +export interface ICourseDetail { + courseId: string; + courseTitle: string; + progress: number; + units: { + type: 'nested'; + values: IUnitDetail[]; + }; +} + +export interface IUnitDetail { + unitId: string; + unitTitle: string; + progress: number; + contents: { + type: 'nested'; + values: IContentDetail[]; + }; +} + +export interface IContentDetail { + contentId: string; + type: string; + title: string; + status: string; + tracking: { + percentComplete?: number; + lastPosition?: number; + currentPosition?: number; + timeSpent?: number; + visitedPages?: number[]; + totalPages?: number; + lastPage?: number; + currentPage?: number; + questionsAttempted?: number; + totalQuestions?: number; + score?: number; + answers?: { + type: 'nested'; + values: IAnswerDetail[]; + }; + }; +} + +export interface IAnswerDetail { + questionId: string; + type: string; + submittedAnswer: string | string[]; } export interface ICourse { @@ -82,7 +138,7 @@ export interface IUser { userId: string; profile: IProfile; applications: IApplication[]; - courses: ICourse[]; + courses?: ICourse[]; // Made optional since we're removing root-level courses createdAt: string; updatedAt: string; } diff --git a/src/elasticsearch/user-elasticsearch.controller.ts b/src/elasticsearch/user-elasticsearch.controller.ts index 6ff5e7a9..11c26354 100644 --- a/src/elasticsearch/user-elasticsearch.controller.ts +++ b/src/elasticsearch/user-elasticsearch.controller.ts @@ -41,11 +41,10 @@ export class UserElasticsearchController { lastSavedAt: application.lastSavedAt || new Date().toISOString(), submittedAt: application.submittedAt || new Date().toISOString(), cohortDetails: { + cohortId: application.cohortDetails?.cohortId || cohortId, name: application.cohortDetails?.name || '', - description: application.cohortDetails?.description ?? '', - startDate: application.cohortDetails?.startDate ?? '', - endDate: application.cohortDetails?.endDate ?? '', - status: application.cohortDetails?.status || '', + type: application.cohortDetails?.type || 'COHORT', + status: application.cohortDetails?.status || 'active', }, progress: application.progress || { pages: {}, diff --git a/src/elasticsearch/user-elasticsearch.service.ts b/src/elasticsearch/user-elasticsearch.service.ts index 0362cc79..2dad93b5 100644 --- a/src/elasticsearch/user-elasticsearch.service.ts +++ b/src/elasticsearch/user-elasticsearch.service.ts @@ -84,9 +84,11 @@ export class UserElasticsearchService implements OnModuleInit { type: 'nested', properties: { cohortId: { type: 'keyword' }, + formId: { type: 'keyword' }, + submissionId: { type: 'keyword' }, cohortmemberstatus: { type: 'keyword' }, formstatus: { type: 'keyword' }, - completionPercentage: { type: 'float' }, // FIXED: Add completionPercentage field + completionPercentage: { type: 'float' }, progress: { properties: { pages: { @@ -108,32 +110,72 @@ export class UserElasticsearchService implements OnModuleInit { submittedAt: { type: 'date', null_value: null }, cohortDetails: { properties: { + cohortId: { type: 'keyword' }, name: { type: 'text' }, - description: { type: 'text' }, - startDate: { type: 'date', null_value: null }, - endDate: { type: 'date', null_value: null }, + type: { type: 'keyword' }, status: { type: 'keyword' }, }, }, - }, - }, - courses: { - type: 'nested', - properties: { - courseId: { type: 'keyword' }, - progress: { type: 'float' }, - lessonsCompleted: { type: 'keyword' }, - lastLessonAt: { type: 'date', null_value: null }, - courseDetails: { + courses: { + type: 'nested', properties: { - name: { type: 'text' }, - description: { type: 'text' }, - duration: { type: 'integer' }, - status: { type: 'keyword' }, + courseId: { type: 'keyword' }, + courseTitle: { type: 'text' }, + progress: { type: 'float' }, + units: { + type: 'nested', + properties: { + unitId: { type: 'keyword' }, + unitTitle: { type: 'text' }, + progress: { type: 'float' }, + contents: { + type: 'nested', + properties: { + contentId: { type: 'keyword' }, + type: { type: 'keyword' }, + title: { type: 'text' }, + status: { type: 'keyword' }, + tracking: { + properties: { + percentComplete: { type: 'float' }, + lastPosition: { type: 'float' }, + currentPosition: { type: 'float' }, + timeSpent: { type: 'integer' }, + visitedPages: { type: 'integer' }, + totalPages: { type: 'integer' }, + lastPage: { type: 'integer' }, + currentPage: { type: 'integer' }, + questionsAttempted: { type: 'integer' }, + totalQuestions: { type: 'integer' }, + score: { type: 'float' }, + answers: { + type: 'nested', + properties: { + questionId: { type: 'keyword' }, + type: { type: 'keyword' }, + submittedAnswer: { type: 'text' }, + answer: { + properties: { + answer: { type: 'keyword' }, + text: { type: 'text' }, + }, + }, + text: { type: 'text' }, + score: { type: 'float' }, + reviewStatus: { type: 'keyword' }, + }, + }, + }, + }, + }, + }, + }, + }, }, }, }, }, + // Removed root-level courses mapping as requested - courses now only in applications createdAt: { type: 'date', null_value: null }, updatedAt: { type: 'date', null_value: null }, }, @@ -188,7 +230,7 @@ export class UserElasticsearchService implements OnModuleInit { customFields: user.profile.customFields || {}, }, applications: user.applications || [], - courses: user.courses || [], + // Removed root-level courses field as requested createdAt: user.createdAt, updatedAt: user.updatedAt, }; @@ -352,6 +394,16 @@ export class UserElasticsearchService implements OnModuleInit { filter: [], }, }; + + // Add userId filter if provided + if (query.userId) { + searchQuery.bool.filter.push({ + term: { + userId: query.userId, + }, + }); + } + // Add text search with partial matching support if (query.q) { if (typeof query.q !== 'string') { @@ -491,10 +543,21 @@ export class UserElasticsearchService implements OnModuleInit { } } if (query.filters && typeof query.filters === 'object') { - // Special handling for cohortId and cohortmemberstatus in applications + // Special handling for cohortId, cohortmemberstatus, completionPercentage, and courses in applications const appFilters: any = {}; + const courseFilters: any = {}; Object.entries(query.filters).forEach(([field, value]) => { if (value !== undefined && value !== null && value !== '') { + // Handle userId filter + if (field === 'userId') { + searchQuery.bool.filter.push({ + term: { + userId: value, + }, + }); + return; + } + // Handle cohortId, cohortmemberstatus, and completionPercentage as nested application filters if ( field === 'cohortId' || @@ -505,6 +568,13 @@ export class UserElasticsearchService implements OnModuleInit { return; } + // Handle courses fields as nested course filters + if (field.startsWith('courses.')) { + const courseField = field.replace('courses.', ''); + courseFilters[courseField] = value; + return; + } + // Handle custom fields filtering if (field.startsWith('customFields.')) { const customFieldName = field.replace('customFields.', ''); @@ -717,6 +787,323 @@ export class UserElasticsearchService implements OnModuleInit { }, }); } + + // Handle course filters if present + if (Object.keys(courseFilters).length > 0) { + const courseMust: any[] = []; + + Object.entries(courseFilters).forEach(([field, value]) => { + if (value !== undefined && value !== null && value !== '') { + // Handle different course field types + if (field === 'courseId') { + courseMust.push({ + term: { + 'applications.courses.courseId': value, + }, + }); + } else if (field === 'courseTitle') { + courseMust.push({ + wildcard: { + 'applications.courses.courseTitle': `*${String( + value + ).toLowerCase()}*`, + }, + }); + } else if (field === 'progress') { + const progressValue = Number(value); + if (!isNaN(progressValue)) { + courseMust.push({ + range: { + 'applications.courses.progress': { + gte: progressValue, + lte: progressValue, + }, + }, + }); + } + } else if (field.startsWith('units.')) { + // Handle nested unit fields + const unitField = field.replace('units.', ''); + if (unitField === 'unitId') { + courseMust.push({ + nested: { + path: 'applications.courses.units', + query: { + term: { + 'applications.courses.units.unitId': value, + }, + }, + }, + }); + } else if (unitField === 'unitTitle') { + courseMust.push({ + nested: { + path: 'applications.courses.units', + query: { + wildcard: { + 'applications.courses.units.unitTitle': `*${String( + value + ).toLowerCase()}*`, + }, + }, + }, + }); + } else if (unitField === 'progress') { + const unitProgressValue = Number(value); + if (!isNaN(unitProgressValue)) { + courseMust.push({ + nested: { + path: 'applications.courses.units', + query: { + range: { + 'applications.courses.units.progress': { + gte: unitProgressValue, + lte: unitProgressValue, + }, + }, + }, + }, + }); + } + } else if (unitField.startsWith('contents.')) { + // Handle nested content fields within units + const contentField = unitField.replace('contents.', ''); + if (contentField === 'contentId') { + courseMust.push({ + nested: { + path: 'applications.courses.units.contents', + query: { + term: { + 'applications.courses.units.contents.contentId': + value, + }, + }, + }, + }); + } else if (contentField === 'lessonId') { + courseMust.push({ + nested: { + path: 'applications.courses.units.contents', + query: { + term: { + 'applications.courses.units.contents.lessonId': + value, + }, + }, + }, + }); + } else if (contentField === 'title') { + courseMust.push({ + nested: { + path: 'applications.courses.units.contents', + query: { + wildcard: { + 'applications.courses.units.contents.title': `*${String( + value + ).toLowerCase()}*`, + }, + }, + }, + }); + } else if (contentField === 'type') { + courseMust.push({ + nested: { + path: 'applications.courses.units.contents', + query: { + term: { + 'applications.courses.units.contents.type': value, + }, + }, + }, + }); + } else if (contentField === 'status') { + courseMust.push({ + nested: { + path: 'applications.courses.units.contents', + query: { + term: { + 'applications.courses.units.contents.status': value, + }, + }, + }, + }); + } else if (contentField.startsWith('tracking.')) { + // Handle nested tracking fields within contents + const trackingField = contentField.replace('tracking.', ''); + if (trackingField === 'percentComplete') { + const trackingValue = Number(value); + if (!isNaN(trackingValue)) { + courseMust.push({ + nested: { + path: 'applications.courses.units.contents', + query: { + range: { + 'applications.courses.units.contents.tracking.percentComplete': + { + gte: trackingValue, + lte: trackingValue, + }, + }, + }, + }, + }); + } + } else if (trackingField === 'score') { + const trackingValue = Number(value); + if (!isNaN(trackingValue)) { + courseMust.push({ + nested: { + path: 'applications.courses.units.contents', + query: { + range: { + 'applications.courses.units.contents.tracking.score': + { + gte: trackingValue, + lte: trackingValue, + }, + }, + }, + }, + }); + } + } else if (trackingField === 'timeSpent') { + const trackingValue = Number(value); + if (!isNaN(trackingValue)) { + courseMust.push({ + nested: { + path: 'applications.courses.units.contents', + query: { + range: { + 'applications.courses.units.contents.tracking.timeSpent': + { + gte: trackingValue, + lte: trackingValue, + }, + }, + }, + }, + }); + } + } else if (trackingField === 'questionsAttempted') { + const trackingValue = Number(value); + if (!isNaN(trackingValue)) { + courseMust.push({ + nested: { + path: 'applications.courses.units.contents', + query: { + range: { + 'applications.courses.units.contents.tracking.questionsAttempted': + { + gte: trackingValue, + lte: trackingValue, + }, + }, + }, + }, + }); + } + } else if (trackingField === 'totalQuestions') { + const trackingValue = Number(value); + if (!isNaN(trackingValue)) { + courseMust.push({ + nested: { + path: 'applications.courses.units.contents', + query: { + range: { + 'applications.courses.units.contents.tracking.totalQuestions': + { + gte: trackingValue, + lte: trackingValue, + }, + }, + }, + }, + }); + } + } else if (trackingField.startsWith('answers.')) { + // Handle nested answers fields within tracking + const answerField = trackingField.replace('answers.', ''); + if (answerField === 'questionId') { + courseMust.push({ + nested: { + path: 'applications.courses.units.contents.tracking.answers', + query: { + term: { + 'applications.courses.units.contents.tracking.answers.questionId': + value, + }, + }, + }, + }); + } else if (answerField === 'answer') { + courseMust.push({ + nested: { + path: 'applications.courses.units.contents.tracking.answers', + query: { + wildcard: { + 'applications.courses.units.contents.tracking.answers.answer': `*${String( + value + ).toLowerCase()}*`, + }, + }, + }, + }); + } else if (answerField === 'score') { + const answerScoreValue = Number(value); + if (!isNaN(answerScoreValue)) { + courseMust.push({ + nested: { + path: 'applications.courses.units.contents.tracking.answers', + query: { + range: { + 'applications.courses.units.contents.tracking.answers.score': + { + gte: answerScoreValue, + lte: answerScoreValue, + }, + }, + }, + }, + }); + } + } else if (answerField === 'reviewStatus') { + courseMust.push({ + nested: { + path: 'applications.courses.units.contents.tracking.answers', + query: { + term: { + 'applications.courses.units.contents.tracking.answers.reviewStatus': + value, + }, + }, + }, + }); + } + } + } + } + } else { + // Default to wildcard for other course fields + courseMust.push({ + wildcard: { + [`applications.courses.${field}`]: `*${String( + value + ).toLowerCase()}*`, + }, + }); + } + } + }); + + if (courseMust.length > 0) { + searchQuery.bool.filter.push({ + nested: { + path: 'applications.courses', + query: { bool: { must: courseMust } }, + }, + }); + } + } } if (query.cohortId && typeof query.cohortId === 'string') { searchQuery.bool.filter.push({ @@ -853,10 +1240,17 @@ export class UserElasticsearchService implements OnModuleInit { lastSavedAt: application.lastSavedAt || new Date().toISOString(), submittedAt: application.submittedAt || new Date().toISOString(), cohortDetails: application.cohortDetails || { + cohortId: '', name: '', + type: 'COHORT', status: 'active', }, formData: {}, + // Preserve existing courses data or initialize empty courses structure + courses: application.courses || { + type: 'nested', + values: [], + }, }; // If application has formData, map it to pages structure @@ -992,37 +1386,14 @@ export class UserElasticsearchService implements OnModuleInit { course: Partial ): Promise { try { - const script = { - source: ` - if (ctx._source.courses == null) { - ctx._source.courses = []; - } - boolean found = false; - for (int i = 0; i < ctx._source.courses.length; i++) { - if (ctx._source.courses[i].courseId == params.courseId) { - ctx._source.courses[i] = params.course; - found = true; - break; - } - } - if (!found) { - ctx._source.courses.add(params.course); - } - `, - lang: 'painless', - params: { - courseId, - course, - }, - }; - - const result = await this.elasticsearchService.update( - this.indexName, - userId, - { script }, - { retry_on_conflict: 3 } + // For now, courses will be null as per requirements + // This method is kept for backward compatibility + this.logger.log( + `Course update requested for user ${userId}, course ${courseId} - currently disabled as courses are null` ); - return result; + + // Return success without actually updating anything + return { acknowledged: true }; } catch (error) { this.logger.error('Error updating course in Elasticsearch:', error); throw new Error( @@ -1031,6 +1402,39 @@ export class UserElasticsearchService implements OnModuleInit { } } + async updateApplicationCourse( + userId: string, + cohortId: string, + courseData: { + courseId: string; + courseTitle: string; + progress: number; + units?: { + type: 'nested'; + values: any[]; + }; + } + ): Promise { + try { + // For now, courses will be null as per requirements + // This method is prepared for future implementation + this.logger.log( + `Application course update requested for user ${userId}, cohort ${cohortId}, course ${courseData.courseId} - currently disabled as courses are null` + ); + + // Return success without actually updating anything + return { acknowledged: true }; + } catch (error) { + this.logger.error( + 'Error updating application course in Elasticsearch:', + error + ); + throw new Error( + `Failed to update application course in Elasticsearch: ${error.message}` + ); + } + } + async updateApplicationPage( userId: string, cohortId: string, diff --git a/src/forms/services/form-submission.service.ts b/src/forms/services/form-submission.service.ts index c8ca4008..a075c74f 100644 --- a/src/forms/services/form-submission.service.ts +++ b/src/forms/services/form-submission.service.ts @@ -37,6 +37,10 @@ import { LoggerUtil } from 'src/common/logger/LoggerUtil'; import { isElasticsearchEnabled } from 'src/common/utils/elasticsearch.util'; import { CohortMembers } from 'src/cohortMembers/entities/cohort-member.entity'; import { Cohort } from 'src/cohort/entities/cohort.entity'; +import { + ElasticsearchSyncService, + SyncSection, +} from '../../elasticsearch/elasticsearch-sync.service'; interface DateRange { start: string; @@ -78,7 +82,7 @@ interface FieldSearchResponse { @Injectable() export class FormSubmissionService { private readonly logger = new Logger(FormSubmissionService.name); - + constructor( @InjectRepository(FormSubmission) private formSubmissionRepository: Repository, @@ -94,6 +98,7 @@ export class FormSubmissionService { private readonly userElasticsearchService: UserElasticsearchService, private readonly elasticsearchDataFetcherService: ElasticsearchDataFetcherService, private readonly formsService: FormsService, + private readonly elasticsearchSyncService: ElasticsearchSyncService, @Inject(forwardRef(() => PostgresCohortService)) private readonly postgresCohortService: PostgresCohortService ) {} @@ -199,14 +204,11 @@ export class FormSubmissionService { createFormSubmissionDto.formSubmission.formId // Pass formId for checkbox processing ); - // Update Elasticsearch with complete field values from database (includes dependencies) - // Only update if Elasticsearch is enabled + // Update Elasticsearch using centralized service if (isElasticsearchEnabled()) { - await this.updateApplicationInElasticsearch( - userId, - savedSubmission, - customFields // Fixed: now passes all fields from DB - ); + await this.elasticsearchSyncService.syncUserToElasticsearch(userId, { + section: SyncSection.APPLICATIONS, + }); } // Create response object @@ -361,7 +363,7 @@ export class FormSubmissionService { Array.isArray(completionPercentage) && completionPercentage.length > 0 ) { - // Multiple ranges - use OR conditions (union) with proper casting + // Multiple ranges - use OR conditions (union) with proper numeric casting const rangeConditions = completionPercentage.map((range, index) => { const [min, max] = range.split('-').map(Number); return `(CAST(fs.completionPercentage AS DECIMAL(5,2)) >= :min${index} AND CAST(fs.completionPercentage AS DECIMAL(5,2)) <= :max${index})`; @@ -1004,14 +1006,11 @@ export class FormSubmissionService { updatedSubmission.formId // Pass formId for checkbox processing ); - // Update Elasticsearch after successful form submission update - // Only update if Elasticsearch is enabled + // Update Elasticsearch using centralized service if (isElasticsearchEnabled()) { - await this.updateApplicationInElasticsearch( - userId, - updatedSubmission, - completeFieldValues // Fixed: now passes all fields from DB - ); + await this.elasticsearchSyncService.syncUserToElasticsearch(userId, { + section: SyncSection.APPLICATIONS, + }); } const successResponse = { id: 'api.form.submission.update', @@ -1163,10 +1162,9 @@ export class FormSubmissionService { } /** - * Update the user's applications array in Elasticsearch after a form submission update. - * This will upsert (update or create) the user document in Elasticsearch if missing. - * If the document is missing, it will fetch the user from the database and create it. - * Uses centralized data fetcher for consistent data structure. + * Update application in Elasticsearch using centralized service + * This method is now deprecated in favor of the centralized service. + * @deprecated Use elasticsearchSyncService.syncUserToElasticsearch instead */ private async updateApplicationInElasticsearch( userId: string, @@ -1174,346 +1172,12 @@ export class FormSubmissionService { updatedFieldValues: any[] ): Promise { try { - // Use centralized service to get complete user document - const userDocument = await this.elasticsearchDataFetcherService.fetchUserDocumentForElasticsearch(userId); - - if (!userDocument) { - this.logger.warn(`User document not found for ${userId}, skipping Elasticsearch update`); - return; - } - - // Get the existing user document from Elasticsearch - const userDoc = await this.userElasticsearchService.getUser(userId); - - // Prepare the applications array (existing or new) - let applications: any[] = []; - if (userDoc && userDoc._source) { - const userSource = userDoc._source as IUser; - applications = userSource.applications || []; - } - - // Use the actual submissionId and formId from the updatedSubmission - const formIdToMatch = updatedSubmission.formId; - const submissionIdToMatch = updatedSubmission.submissionId; - - // --- NEW LOGIC: Use the robust getFieldIdToPageNameMap utility function --- - let fieldIdToPageName: Record = {}; - let fieldIdToFieldName: Record = {}; - let formFieldsOnly: any[] = []; - try { - const form = await this.formsService.getFormById(formIdToMatch); - const fieldsObj = form && form.fields ? (form.fields as any) : null; - - // Handle different schema structures - let schema: any = {}; - if (fieldsObj) { - // Try different possible schema structures - if ( - Array.isArray(fieldsObj?.result) && - fieldsObj.result[0]?.schema?.properties - ) { - // Structure: { result: [{ schema: { properties: {...} } }] } - schema = fieldsObj.result[0].schema.properties; - } else if (fieldsObj?.schema?.properties) { - // Structure: { schema: { properties: {...} } } - schema = fieldsObj.schema.properties; - } else if (fieldsObj?.properties) { - // Structure: { properties: {...} } - schema = fieldsObj.properties; - } else if (typeof fieldsObj === 'object' && fieldsObj !== null) { - // Try to find schema in nested structure - const findSchema = (obj: any): any => { - if (obj?.schema?.properties) return obj.schema.properties; - if (obj?.properties) return obj.properties; - if (Array.isArray(obj)) { - for (const item of obj) { - const found = findSchema(item); - if (found) return found; - } - } else if (typeof obj === 'object') { - for (const key in obj) { - const found = findSchema(obj[key]); - if (found) return found; - } - } - return null; - }; - schema = findSchema(fieldsObj) || {}; - } - } - - // Use the robust utility function to get field mappings - fieldIdToPageName = getFieldIdToPageNameMap(schema); - - // Also build fieldIdToFieldName mapping for logging - for (const [pageKey, pageSchema] of Object.entries(schema)) { - const pageName = pageKey === 'default' ? 'eligibilityCheck' : pageKey; - const fieldProps = (pageSchema as any).properties || {}; - - const extractFieldNames = (properties: any) => { - for (const [fieldKey, fieldSchema] of Object.entries(properties)) { - const fieldId = (fieldSchema as any).fieldId; - const fieldTitle = (fieldSchema as any).title || fieldKey; - if (fieldId) { - fieldIdToFieldName[fieldId] = fieldTitle; - } - - // Handle dependencies - if ((fieldSchema as any).dependencies) { - const dependencies = (fieldSchema as any).dependencies; - for (const depSchema of Object.values(dependencies)) { - if (!depSchema || typeof depSchema !== 'object') continue; - const dep = depSchema as any; - if (dep.oneOf) - dep.oneOf.forEach( - (item: any) => - item?.properties && extractFieldNames(item.properties) - ); - if (dep.allOf) - dep.allOf.forEach( - (item: any) => - item?.properties && extractFieldNames(item.properties) - ); - if (dep.anyOf) - dep.anyOf.forEach( - (item: any) => - item?.properties && extractFieldNames(item.properties) - ); - if (dep.properties) extractFieldNames(dep.properties); - } - } - } - }; - - extractFieldNames(fieldProps); - - // Handle page-level dependencies - const pageDependencies = (pageSchema as any).dependencies || {}; - for (const depSchema of Object.values(pageDependencies)) { - if (!depSchema || typeof depSchema !== 'object') continue; - const dep = depSchema as any; - if (dep.oneOf) - dep.oneOf.forEach( - (item: any) => - item?.properties && extractFieldNames(item.properties) - ); - if (dep.allOf) - dep.allOf.forEach( - (item: any) => - item?.properties && extractFieldNames(item.properties) - ); - if (dep.anyOf) - dep.anyOf.forEach( - (item: any) => - item?.properties && extractFieldNames(item.properties) - ); - if (dep.properties) extractFieldNames(dep.properties); - } - } - - for (const [fieldId, pageName] of Object.entries(fieldIdToPageName)) { - const fieldName = fieldIdToFieldName[fieldId] || fieldId; - } - - // Filter updatedFieldValues to only include form fields (not custom fields) - const formFieldIds = new Set(); - for (const [pageKey, pageSchema] of Object.entries(schema)) { - const fieldProps = (pageSchema as any).properties || {}; - - const extractFormFieldIds = (properties: any) => { - for (const [fieldKey, fieldSchema] of Object.entries(properties)) { - const fieldId = (fieldSchema as any).fieldId; - if (fieldId) { - formFieldIds.add(fieldId); - } - - // Handle nested dependencies structure - ensure dependency fields are included - if ((fieldSchema as any).dependencies) { - const dependencies = (fieldSchema as any).dependencies; - // Check if dependencies is an object - for (const [depKey, depSchema] of Object.entries( - dependencies - )) { - if ((depSchema as any).oneOf) { - // Handle oneOf dependencies - for (const oneOfItem of (depSchema as any).oneOf) { - if (oneOfItem.properties) { - extractFormFieldIds(oneOfItem.properties); - } - } - } else if ((depSchema as any).properties) { - // Handle direct properties dependencies - extractFormFieldIds((depSchema as any).properties); - } - } - } - } - }; - - extractFormFieldIds(fieldProps); - } - // Filter updatedFieldValues to only include form fields - // TEMPORARY FIX: If a field is not in formFieldIds but exists in fieldIdToPageName, include it - formFieldsOnly = updatedFieldValues.filter((field) => { - const isInFormFieldIds = formFieldIds.has(field.fieldId); - const isInFieldMapping = fieldIdToPageName[field.fieldId]; - - return isInFormFieldIds || isInFieldMapping; - }); - } catch (err) { - const logger = new Logger('FormSubmissionService'); - logger.error('Schema fetch failed, cannot proceed', err); - // If schema fetch fails, fallback to empty map (all fields go to 'default') - fieldIdToPageName = {}; - fieldIdToFieldName = {}; - // If schema fetch fails, don't filter fields (use all updatedFieldValues) - formFieldsOnly = updatedFieldValues; - } - // --- END NEW LOGIC --- - - // Always fetch cohortId from the related Form entity - // Fetch cohortId from the related Form entity with proper error handling - let cohortId = ''; - try { - const form = await this.formsService.getFormById(formIdToMatch); - cohortId = form?.contextId || ''; - } catch (error) { - LoggerUtil.warn( - `Failed to fetch cohortId for formId ${formIdToMatch}:`, - error - ); - cohortId = ''; - } - let existingAppIndex = -1; - if (cohortId) { - existingAppIndex = applications.findIndex( - (app) => app.cohortId === cohortId - ); - // If not found by cohortId, don't fallback to avoid inconsistencies - // Log this scenario for investigation - if (existingAppIndex === -1) { - } - } else { - // Use formId/submissionId only when cohortId is not available - existingAppIndex = applications.findIndex( - (app) => - app.formId === formIdToMatch && - app.submissionId === submissionIdToMatch - ); - } - - // Prepare the updated fields data using the robust mapping - const updatedFields = {}; - formFieldsOnly.forEach((field) => { - const pageKey = fieldIdToPageName[field.fieldId]; - if (!pageKey) { - // Log warning for unmapped fields instead of fallback - LoggerUtil.warn( - `FieldId ${field.fieldId} not found in schema mapping! Skipping field.` - ); - return; // Skip this field instead of assigning to wrong page - } - updatedFields[pageKey] ??= { - completed: true, - fields: {}, - }; - // Process field value for Elasticsearch - convert arrays to comma-separated strings - updatedFields[pageKey].fields[field.fieldId] = this.processFieldValueForElasticsearch(field.value); - }); - - - - if (existingAppIndex !== -1) { - // COMPLETELY REPLACE old pages structure instead of merging - const mergedPages = {}; - - // Only use the new schema-based mapping, don't merge with old data - for (const [pageKey, pageValue] of Object.entries(updatedFields)) { - const newPage = pageValue as { - completed: boolean; - fields: { [key: string]: any }; - }; - mergedPages[pageKey] = newPage; - } - - // Merge overall progress - const mergedOverall = applications[existingAppIndex]?.progress?.overall - ? { ...applications[existingAppIndex].progress.overall } - : { - completed: updatedFieldValues.length, - total: updatedFieldValues.length, - }; - - // Use completionPercentage from the updatedSubmission (payload value) instead of calculating - const completionPercentage = - updatedSubmission.completionPercentage ?? 0; - - // --- Update cohortmemberstatus and cohortDetails logic --- - // cohortmemberstatus is not a property of FormSubmission; set as empty string or fetch from CohortMembers if needed - applications[existingAppIndex].cohortmemberstatus = - applications[existingAppIndex].cohortmemberstatus ?? ''; // preserve if already set - // Ensure cohortDetails is populated; if missing or empty, fetch from DB - if ( - !applications[existingAppIndex].cohortDetails || - Object.keys(applications[existingAppIndex].cohortDetails).length === 0 - ) { - applications[existingAppIndex].cohortDetails = - await this.fetchCohortDetailsFromDB(updatedSubmission); - } - // --- End cohortmemberstatus and cohortDetails logic --- - - applications[existingAppIndex] = { - ...applications[existingAppIndex], - formId: formIdToMatch, - submissionId: submissionIdToMatch, - formstatus: - updatedSubmission.status ?? - applications[existingAppIndex].formstatus, - // Use completionPercentage from payload (updatedSubmission) - completionPercentage: completionPercentage, - progress: { - pages: mergedPages, - overall: mergedOverall, - }, - lastSavedAt: new Date().toISOString(), - submittedAt: new Date().toISOString(), - }; - - // Also update the FormSubmission entity in the database - try { - const submissionToUpdate = - await this.formSubmissionRepository.findOne({ - where: { submissionId: submissionIdToMatch }, - }); - if (submissionToUpdate) { - submissionToUpdate.completionPercentage = completionPercentage; - await this.formSubmissionRepository.save(submissionToUpdate); - } - } catch (error) { - LoggerUtil.warn( - 'Failed to update FormSubmission completionPercentage:', - error - ); - } - } else { - // If no existing application found, build from DB with correct cohortDetails - const newApp = await this.buildApplicationFromDB(updatedSubmission); - applications.push(newApp); - } - - // Upsert (update or create) the user document in Elasticsearch if (isElasticsearchEnabled()) { - await this.userElasticsearchService.updateUser( - userId, - { doc: { applications: applications } }, - async (userId: string) => { - // Build the full user document for Elasticsearch, including profile and all applications - return await this.buildUserDocumentForElasticsearch(userId); - } - ); + await this.elasticsearchSyncService.syncUserToElasticsearch(userId, { + section: SyncSection.APPLICATIONS, + }); } } catch (elasticError) { - // Log Elasticsearch error but don't fail the request LoggerUtil.warn('Failed to update Elasticsearch:', elasticError); } } @@ -1583,8 +1247,20 @@ export class FormSubmissionService { const form = await this.formsService.getFormById(submission.formId); const fieldsObj = form && form.fields ? (form.fields as any) : null; - // Get cohortId from form context - cohortId = form?.contextId || ''; + // Get cohortId from form's contextId + try { + // Get form to find contextId (which is the cohortId) + cohortId = form?.contextId || ''; + LoggerUtil.warn( + `Using form contextId as cohortId: ${cohortId} for formId: ${submission.formId}` + ); + } catch (error) { + LoggerUtil.warn( + `Failed to fetch form for formId ${submission.formId}:`, + error + ); + cohortId = ''; + } // Handle different schema structures if (fieldsObj) { @@ -1891,8 +1567,10 @@ export class FormSubmissionService { } // Process field value for Elasticsearch - convert arrays to comma-separated strings - pages[pageName].fields[field.fieldId] = this.processFieldValueForElasticsearch(field.value); - formData[pageName][field.fieldId] = this.processFieldValueForElasticsearch(field.value); + pages[pageName].fields[field.fieldId] = + this.processFieldValueForElasticsearch(field.value); + formData[pageName][field.fieldId] = + this.processFieldValueForElasticsearch(field.value); } if (Object.keys(pages).length === 0) { @@ -1900,7 +1578,9 @@ export class FormSubmissionService { completed: true, fields: formFieldsOnly.reduce((acc, field) => { // Process field value for Elasticsearch - convert arrays to comma-separated strings - acc[field.fieldId] = this.processFieldValueForElasticsearch(field.value); + acc[field.fieldId] = this.processFieldValueForElasticsearch( + field.value + ); return acc; }, {}), }; @@ -1961,7 +1641,7 @@ export class FormSubmissionService { parentId: cohortDetails.parentId, type: cohortDetails.type, status: cohortDetails.status, - // Removed customFields from cohortDetails as per requirement + customFields: cohortCustomFields, }; } } catch (error) { @@ -1996,7 +1676,7 @@ export class FormSubmissionService { /** * Build user document for Elasticsearch using centralized data fetcher. * This method provides a centralized way to fetch user data for Elasticsearch. - * + * * @param userId - User ID to fetch data for * @returns Promise - Complete user document or null if user not found */ @@ -2004,7 +1684,9 @@ export class FormSubmissionService { userId: string ): Promise { // Use centralized data fetcher service for consistent data structure - return await this.elasticsearchDataFetcherService.fetchUserDocumentForElasticsearch(userId); + return await this.elasticsearchDataFetcherService.fetchUserDocumentForElasticsearch( + userId + ); } /** @@ -2628,15 +2310,19 @@ export class FormSubmissionService { } // Process field value for Elasticsearch - convert arrays to comma-separated strings - pages[pageName].fields[field.fieldId] = this.processFieldValueForElasticsearch(field.value); - formData[pageName][field.fieldId] = this.processFieldValueForElasticsearch(field.value); + pages[pageName].fields[field.fieldId] = + this.processFieldValueForElasticsearch(field.value); + formData[pageName][field.fieldId] = + this.processFieldValueForElasticsearch(field.value); } if (Object.keys(pages).length === 0) { pages['eligibilityCheck'] = { completed: true, fields: formFieldsOnly.reduce((acc, field) => { // Process field value for Elasticsearch - convert arrays to comma-separated strings - acc[field.fieldId] = this.processFieldValueForElasticsearch(field.value); + acc[field.fieldId] = this.processFieldValueForElasticsearch( + field.value + ); return acc; }, {}), }; @@ -3048,7 +2734,7 @@ export class FormSubmissionService { customFields: profileCustomFields, // Only user profile custom fields }, applications, - courses: [], + // Removed root-level courses field as requested createdAt: user.createdAt ? user.createdAt.toISOString() : new Date().toISOString(), @@ -3072,6 +2758,60 @@ export class FormSubmissionService { // Return value as-is for non-array values return value; } + + /** + * Helper function to build fieldId to fieldName mapping from form schema + * @param schema - Form schema object + * @returns Record - Mapping of fieldId to fieldName + */ + private getFieldIdToFieldNameMap(schema: any): Record { + const fieldIdToFieldName: Record = {}; + + function extract(properties: any, currentPage: string) { + for (const [fieldKey, fieldSchema] of Object.entries(properties)) { + const fieldId = (fieldSchema as any).fieldId; + const fieldTitle = (fieldSchema as any).title || fieldKey; + if (fieldId) { + fieldIdToFieldName[fieldId] = fieldTitle; + } + + // Handle dependencies + if ((fieldSchema as any).dependencies) { + const dependencies = (fieldSchema as any).dependencies; + for (const depSchema of Object.values(dependencies)) { + if (!depSchema || typeof depSchema !== 'object') continue; + const dep = depSchema as any; + if (dep.oneOf) + dep.oneOf.forEach( + (item: any) => + item?.properties && extract(item.properties, currentPage) + ); + if (dep.allOf) + dep.allOf.forEach( + (item: any) => + item?.properties && extract(item.properties, currentPage) + ); + if (dep.anyOf) + dep.anyOf.forEach( + (item: any) => + item?.properties && extract(item.properties, currentPage) + ); + if (dep.properties) extract(dep.properties, currentPage); + } + } + } + } + + // Handle different schema structures + if (schema?.properties) { + for (const [pageKey, pageSchema] of Object.entries(schema.properties)) { + const fieldProps = (pageSchema as any).properties || {}; + extract(fieldProps, pageKey); + } + } + + return fieldIdToFieldName; + } } /**