From c8febecfe5de1f2fbf5eb1706bbfab43f108a56b Mon Sep 17 00:00:00 2001 From: chris Date: Thu, 8 Jun 2023 12:35:50 +0300 Subject: [PATCH 01/17] feat(database): add index support for all dialects & uniqueness testing wip --- libraries/grpc-sdk/src/interfaces/Model.ts | 93 ++++++++++++------- .../src/adapters/mongoose-adapter/utils.ts | 4 +- .../src/adapters/sequelize-adapter/index.ts | 42 +++++---- .../adapters/sequelize-adapter/utils/index.ts | 23 ++++- .../utils/database-transform-utils.ts | 86 +++++++++-------- 5 files changed, 157 insertions(+), 91 deletions(-) diff --git a/libraries/grpc-sdk/src/interfaces/Model.ts b/libraries/grpc-sdk/src/interfaces/Model.ts index d1b1fd83b..57afa866a 100644 --- a/libraries/grpc-sdk/src/interfaces/Model.ts +++ b/libraries/grpc-sdk/src/interfaces/Model.ts @@ -1,3 +1,5 @@ +import { Document } from 'bson'; +import { IndexesOptions } from 'sequelize'; export enum TYPE { String = 'String', Number = 'Number', @@ -22,25 +24,6 @@ export enum SQLDataType { TIMESTAMP = 'TIMESTAMP', } -export enum MongoIndexType { - Ascending = 1, - Descending = -1, - GeoSpatial2d = '2d', - GeoSpatial2dSphere = '2dsphere', - GeoHaystack = 'geoHaystack', - Hashed = 'hashed', - Text = 'text', -} - -export enum PostgresIndexType { - BTREE = 'BTREE', - HASH = 'HASH', - GIST = 'GIST', - SPGIST = 'SPGIST', - GIN = 'GIN', - BRIN = 'BRIN', -} - export type Array = any[]; type BaseConduitModelField = { @@ -161,21 +144,58 @@ export interface ConduitSchemaOptions { indexes?: ModelOptionsIndexes[]; } +// Index types export interface SchemaFieldIndex { - type?: MongoIndexType | PostgresIndexType; - options?: MongoIndexOptions | PostgresIndexOptions; - - [field: string]: any; + type?: MongoIndexType | SequelizeIndexType; + options?: MongoIndexOptions | SequelizeIndexOptions; + //[field: string]: any; } export interface ModelOptionsIndexes { fields: string[]; - types?: MongoIndexType[] | PostgresIndexType; - options?: MongoIndexOptions | PostgresIndexOptions; - + types?: MongoIndexType[] | SequelizeIndexType; + options?: MongoIndexOptions | SequelizeIndexOptions; [field: string]: any; } +export enum MongoIndexType { + Ascending = 1, + Descending = -1, + GeoSpatial2d = '2d', + GeoSpatial2dSphere = '2dsphere', + GeoHaystack = 'geoHaystack', + Hashed = 'hashed', + Text = 'text', +} + +export enum SQLIndexType { + BTREE = 'BTREE', + HASH = 'HASH', +} + +export enum PostgresIndexType { + GIST = 'GIST', + SPGIST = 'SPGIST', + GIN = 'GIN', + BRIN = 'BRIN', +} + +export enum MySQLMariaDBIndexType { + UNIQUE = 'UNIQUE', + FULLTEXT = 'FULLTEXT', + SPATIAL = 'SPATIAL', +} + +export enum SQLiteIndexType { + BTREE = 'BTREE', +} + +export type SequelizeIndexType = + | SQLIndexType + | PostgresIndexType + | MySQLMariaDBIndexType + | SQLiteIndexType; + export interface MongoIndexOptions { background?: boolean; unique?: boolean; @@ -199,15 +219,24 @@ export interface MongoIndexOptions { hidden?: boolean; } -export interface PostgresIndexOptions { - concurrently?: boolean; +export interface SequelizeIndexOptions { name?: string; - operator?: string; - parser?: null | string; - prefix?: string; + parser?: string | null; unique?: boolean; - using?: PostgresIndexType; + concurrently?: boolean; // Postgres only + // used instead of ModelOptionsIndexes fields for more complex index definitions + fields?: { + name: string; + length?: number; + order?: 'ASC' | 'DESC'; + collate?: string; + operator?: string; + }[]; + operator?: string; // Postgres only where?: { [opt: string]: any; }; + prefix?: string; + using?: SQLIndexType | PostgresIndexType | SQLiteIndexType; + type?: MySQLMariaDBIndexType; } diff --git a/modules/database/src/adapters/mongoose-adapter/utils.ts b/modules/database/src/adapters/mongoose-adapter/utils.ts index bc01a72ed..ba46817e2 100644 --- a/modules/database/src/adapters/mongoose-adapter/utils.ts +++ b/modules/database/src/adapters/mongoose-adapter/utils.ts @@ -2,7 +2,7 @@ import { ConduitModel, Indexable, MongoIndexOptions, - PostgresIndexOptions, + SequelizeIndexOptions, } from '@conduitplatform/grpc-sdk'; import { MongooseAdapter } from './index'; import _, { isArray, isObject } from 'lodash'; @@ -79,7 +79,7 @@ async function _createWithPopulations( } } -export function checkIfMongoOptions(options: MongoIndexOptions | PostgresIndexOptions) { +export function checkIfMongoOptions(options: MongoIndexOptions | SequelizeIndexOptions) { const mongoOptions = [ 'background', 'unique', diff --git a/modules/database/src/adapters/sequelize-adapter/index.ts b/modules/database/src/adapters/sequelize-adapter/index.ts index 748931feb..ed461cdba 100644 --- a/modules/database/src/adapters/sequelize-adapter/index.ts +++ b/modules/database/src/adapters/sequelize-adapter/index.ts @@ -5,10 +5,13 @@ import ConduitGrpcSdk, { GrpcError, Indexable, ModelOptionsIndexes, - PostgresIndexOptions, + MySQLMariaDBIndexType, PostgresIndexType, RawSQLQuery, + SequelizeIndexOptions, sleep, + SQLIndexType, + SQLiteIndexType, UntypedArray, } from '@conduitplatform/grpc-sdk'; import { status } from '@grpc/grpc-js'; @@ -16,7 +19,7 @@ import { SequelizeAuto } from 'sequelize-auto'; import { DatabaseAdapter } from '../DatabaseAdapter'; import { SequelizeSchema } from './SequelizeSchema'; import { - checkIfPostgresOptions, + checkIfSequelizeIndexOptions, compileSchema, resolveRelatedSchemas, tableFetch, @@ -28,7 +31,7 @@ import { } from '../../interfaces'; import { sqlSchemaConverter } from './sql-adapter/SqlSchemaConverter'; import { pgSchemaConverter } from './postgres-adapter/PgSchemaConverter'; -import { isNil, merge } from 'lodash'; +import { isArray, isNil, merge } from 'lodash'; const sqlSchemaName = process.env.SQL_SCHEMA ?? 'public'; @@ -484,21 +487,8 @@ export abstract class SequelizeAdapter extends DatabaseAdapter ) { for (const index of indexes) { if (!index.types && !index.options) continue; - if (index.types) { - if ( - Array.isArray(index.types) || - !Object.values(PostgresIndexType).includes(index.types) - ) { - throw new GrpcError( - status.INVALID_ARGUMENT, - 'Invalid index type for PostgreSQL', - ); - } - (index.options as PostgresIndexOptions).using = index.types; - delete index.types; - } if (index.options) { - if (!checkIfPostgresOptions(index.options)) { + if (!checkIfSequelizeIndexOptions(index.options)) { throw new GrpcError( status.INVALID_ARGUMENT, 'Invalid index options for PostgreSQL', @@ -514,6 +504,24 @@ export abstract class SequelizeAdapter extends DatabaseAdapter ); } } + if (index.types) { + if (isArray(index.types) || !(index.types in PostgresIndexType)) { + throw new GrpcError( + status.INVALID_ARGUMENT, + 'Invalid index type for PostgreSQL', + ); + } + if (index.types in MySQLMariaDBIndexType) { + (index.options as SequelizeIndexOptions).type = + index.types as MySQLMariaDBIndexType; + } else { + (index.options as SequelizeIndexOptions).using = index.types as + | SQLIndexType + | PostgresIndexType + | SQLiteIndexType; + } + delete index.types; + } } return indexes; } diff --git a/modules/database/src/adapters/sequelize-adapter/utils/index.ts b/modules/database/src/adapters/sequelize-adapter/utils/index.ts index 557fef447..8c8a5d06a 100644 --- a/modules/database/src/adapters/sequelize-adapter/utils/index.ts +++ b/modules/database/src/adapters/sequelize-adapter/utils/index.ts @@ -1,10 +1,17 @@ -import { MongoIndexOptions, PostgresIndexOptions } from '@conduitplatform/grpc-sdk'; +import { + MongoIndexOptions, + MySQLMariaDBIndexType, + PostgresIndexType, + SequelizeIndexOptions, + SQLIndexType, + SQLiteIndexType, +} from '@conduitplatform/grpc-sdk'; export * from './schema'; export * from './collectionUtils'; -export function checkIfPostgresOptions( - options: MongoIndexOptions | PostgresIndexOptions, +export function checkIfSequelizeIndexOptions( + options: MongoIndexOptions | SequelizeIndexOptions, ) { const postgresOptions = [ 'concurrently', @@ -15,7 +22,17 @@ export function checkIfPostgresOptions( 'unique', 'using', 'where', + 'fields', ]; const result = Object.keys(options).some(option => !postgresOptions.includes(option)); return !result; } + +export function checkIfSequelizeIndexType(type: any) { + return ( + Object.values(SQLIndexType).includes(type as SQLIndexType) || + Object.values(PostgresIndexType).includes(type as PostgresIndexType) || + Object.values(MySQLMariaDBIndexType).includes(type as MySQLMariaDBIndexType) || + Object.values(SQLiteIndexType).includes(type as SQLiteIndexType) + ); +} diff --git a/modules/database/src/adapters/utils/database-transform-utils.ts b/modules/database/src/adapters/utils/database-transform-utils.ts index 7d0e5d831..02beb8b9a 100644 --- a/modules/database/src/adapters/utils/database-transform-utils.ts +++ b/modules/database/src/adapters/utils/database-transform-utils.ts @@ -3,10 +3,16 @@ import { ConduitModelField, ConduitSchema, Indexable, - PostgresIndexOptions, + MySQLMariaDBIndexType, PostgresIndexType, + SequelizeIndexType, + SQLIndexType, + SQLiteIndexType, } from '@conduitplatform/grpc-sdk'; -import { checkIfPostgresOptions } from '../sequelize-adapter/utils'; +import { + checkIfSequelizeIndexType, + checkIfSequelizeIndexOptions, +} from '../sequelize-adapter/utils'; export function checkDefaultValue(type: string, value: string) { switch (type) { @@ -29,24 +35,27 @@ export function checkDefaultValue(type: string, value: string) { export function convertModelOptionsIndexes(copy: ConduitSchema) { for (const index of copy.modelOptions.indexes!) { - if (index.types) { - if ( - isArray(index.types) || - !Object.values(PostgresIndexType).includes(index.types as PostgresIndexType) - ) { - throw new Error('Incorrect index type for PostgreSQL'); + if (index.options) { + if (!checkIfSequelizeIndexOptions(index.options)) { + throw new Error('Incorrect index options for sequelize'); } - index.using = index.types as PostgresIndexType; - delete index.types; + // if ((index.options as SequelizeIndexOptions).fields) { // TODO: integrate this logic somehow + // delete index.fields; + // } + Object.assign(index, index.options); + delete index.options; } - if (index.options) { - if (!checkIfPostgresOptions(index.options)) { - throw new Error('Incorrect index options for PostgreSQL'); + if (index.types) { + // TODO: put null to check + if (isArray(index.types) || !checkIfSequelizeIndexType(index.types)) { + throw new Error('Invalid index type'); } - for (const [option, value] of Object.entries(index.options)) { - index[option as keyof PostgresIndexOptions] = value; + if (index.types in MySQLMariaDBIndexType) { + index.type = index.types as MySQLMariaDBIndexType; + } else { + index.using = index.types as SQLIndexType | PostgresIndexType | SQLiteIndexType; } - delete index.options; + delete index.types; } } return copy; @@ -54,35 +63,38 @@ export function convertModelOptionsIndexes(copy: ConduitSchema) { export function convertSchemaFieldIndexes(copy: ConduitSchema) { const indexes = []; - for (const field of Object.entries(copy.fields)) { - const fieldName = field[0]; - const index = (copy.fields[fieldName] as ConduitModelField).index; + for (const [fieldName, fieldValue] of Object.entries(copy.fields)) { + const field = fieldValue as ConduitModelField; + // Move unique field indexes to modelOptions workaround + // if (field.unique && fieldName !== '_id') { + // field.index = { + // options: { unique: true } + // }; + // delete (field as ConduitModelField).unique; + // } + const index = field.index; if (!index) continue; - const newIndex: any = { - fields: [fieldName], - }; + const newIndex: any = { fields: [fieldName] }; // TODO: remove this any if (index.type) { - if (!Object.values(PostgresIndexType).includes(index.type as PostgresIndexType)) { - throw new Error('Invalid index type for PostgreSQL'); + if (isArray(index.type) || !checkIfSequelizeIndexType(index.type)) { + throw new Error('Invalid index type'); } - newIndex.using = index.type; - } - if (index.options) { - if (!checkIfPostgresOptions(index.options)) { - throw new Error('Invalid index options for PostgreSQL'); - } - for (const [option, value] of Object.entries(index.options)) { - newIndex[option] = value; + if (!(index.type in MySQLMariaDBIndexType)) { + newIndex.using = index.type as SQLIndexType | PostgresIndexType | SQLiteIndexType; + } else { + newIndex.type = index.type as MySQLMariaDBIndexType; } } + if (index.options && !checkIfSequelizeIndexOptions(index.options)) { + throw new Error('Invalid index options for sequelize'); + } + Object.assign(newIndex, index.options); indexes.push(newIndex); delete copy.fields[fieldName]; } - if (copy.modelOptions.indexes) { - copy.modelOptions.indexes = [...copy.modelOptions.indexes, ...indexes]; - } else { - copy.modelOptions.indexes = indexes; - } + copy.modelOptions.indexes = copy.modelOptions.indexes + ? [...copy.modelOptions.indexes, ...indexes] + : indexes; return copy; } From 886878e7689e999efa6555a0b350828164825335 Mon Sep 17 00:00:00 2001 From: chris Date: Tue, 13 Jun 2023 12:52:50 +0300 Subject: [PATCH 02/17] fix(database): wip index types and options per dialect checks --- .../src/adapters/sequelize-adapter/index.ts | 1 + .../src/adapters/sequelize-adapter/utils/index.ts | 15 +++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/modules/database/src/adapters/sequelize-adapter/index.ts b/modules/database/src/adapters/sequelize-adapter/index.ts index ed461cdba..e68b6c200 100644 --- a/modules/database/src/adapters/sequelize-adapter/index.ts +++ b/modules/database/src/adapters/sequelize-adapter/index.ts @@ -481,6 +481,7 @@ export abstract class SequelizeAdapter extends DatabaseAdapter protected abstract hasLegacyCollections(): Promise; private checkAndConvertIndexes( + //TODO: check types & options depending on dialect schemaName: string, indexes: ModelOptionsIndexes[], callerModule: string, diff --git a/modules/database/src/adapters/sequelize-adapter/utils/index.ts b/modules/database/src/adapters/sequelize-adapter/utils/index.ts index 8c8a5d06a..01ea8afbb 100644 --- a/modules/database/src/adapters/sequelize-adapter/utils/index.ts +++ b/modules/database/src/adapters/sequelize-adapter/utils/index.ts @@ -36,3 +36,18 @@ export function checkIfSequelizeIndexType(type: any) { Object.values(SQLiteIndexType).includes(type as SQLiteIndexType) ); } + +export function checkSequelizeIndexType(type: any, dialect?: string) { + switch (dialect) { + case 'postgres': + return Object.values(PostgresIndexType).includes(type as PostgresIndexType); + case 'mysql': + return Object.values(MySQLMariaDBIndexType).includes(type as MySQLMariaDBIndexType); + case 'mariadb': + return Object.values(MySQLMariaDBIndexType).includes(type as MySQLMariaDBIndexType); + case 'sqlite': + return Object.values(SQLiteIndexType).includes(type as SQLiteIndexType); + default: + return; + } +} From d3b3cbcfb9deaf1815073d4ac8681493778e74a9 Mon Sep 17 00:00:00 2001 From: chris Date: Thu, 15 Jun 2023 16:53:30 +0300 Subject: [PATCH 03/17] feat(database): add types & checks for sequelize dialects --- libraries/grpc-sdk/src/interfaces/Model.ts | 16 ++++-- .../src/adapters/sequelize-adapter/index.ts | 34 ++++++++---- .../postgres-adapter/PgSchemaConverter.ts | 10 ++-- .../sql-adapter/SqlSchemaConverter.ts | 10 ++-- .../adapters/sequelize-adapter/utils/index.ts | 52 +------------------ .../sequelize-adapter/utils/indexChecks.ts | 46 ++++++++++++++++ .../utils/database-transform-utils.ts | 38 +++++++------- 7 files changed, 115 insertions(+), 91 deletions(-) create mode 100644 modules/database/src/adapters/sequelize-adapter/utils/indexChecks.ts diff --git a/libraries/grpc-sdk/src/interfaces/Model.ts b/libraries/grpc-sdk/src/interfaces/Model.ts index 57afa866a..80c50d04f 100644 --- a/libraries/grpc-sdk/src/interfaces/Model.ts +++ b/libraries/grpc-sdk/src/interfaces/Model.ts @@ -173,7 +173,7 @@ export enum SQLIndexType { HASH = 'HASH', } -export enum PostgresIndexType { +export enum PgIndexType { GIST = 'GIST', SPGIST = 'SPGIST', GIN = 'GIN', @@ -192,7 +192,7 @@ export enum SQLiteIndexType { export type SequelizeIndexType = | SQLIndexType - | PostgresIndexType + | PgIndexType | MySQLMariaDBIndexType | SQLiteIndexType; @@ -223,7 +223,6 @@ export interface SequelizeIndexOptions { name?: string; parser?: string | null; unique?: boolean; - concurrently?: boolean; // Postgres only // used instead of ModelOptionsIndexes fields for more complex index definitions fields?: { name: string; @@ -232,11 +231,18 @@ export interface SequelizeIndexOptions { collate?: string; operator?: string; }[]; - operator?: string; // Postgres only where?: { [opt: string]: any; }; prefix?: string; - using?: SQLIndexType | PostgresIndexType | SQLiteIndexType; + using?: SQLIndexType | PgIndexType | SQLiteIndexType; +} + +export interface PgIndexOptions extends SequelizeIndexOptions { + concurrently?: boolean; + operator?: string; +} + +export interface MySQLMariaDBIndexOptions extends SequelizeIndexOptions { type?: MySQLMariaDBIndexType; } diff --git a/modules/database/src/adapters/sequelize-adapter/index.ts b/modules/database/src/adapters/sequelize-adapter/index.ts index e68b6c200..3d1d0e2ba 100644 --- a/modules/database/src/adapters/sequelize-adapter/index.ts +++ b/modules/database/src/adapters/sequelize-adapter/index.ts @@ -5,8 +5,9 @@ import ConduitGrpcSdk, { GrpcError, Indexable, ModelOptionsIndexes, + MySQLMariaDBIndexOptions, MySQLMariaDBIndexType, - PostgresIndexType, + PgIndexType, RawSQLQuery, SequelizeIndexOptions, sleep, @@ -19,7 +20,8 @@ import { SequelizeAuto } from 'sequelize-auto'; import { DatabaseAdapter } from '../DatabaseAdapter'; import { SequelizeSchema } from './SequelizeSchema'; import { - checkIfSequelizeIndexOptions, + checkSequelizeIndexOptions, + checkSequelizeIndexType, compileSchema, resolveRelatedSchemas, tableFetch, @@ -273,8 +275,8 @@ export abstract class SequelizeAdapter extends DatabaseAdapter const dialect = this.sequelize.getDialect(); const [newSchema, objectPaths, extractedRelations] = dialect === 'postgres' - ? pgSchemaConverter(compiledSchema) - : sqlSchemaConverter(compiledSchema); + ? pgSchemaConverter(compiledSchema, dialect) + : sqlSchemaConverter(compiledSchema, dialect); this.registeredSchemas.set( schema.name, Object.freeze(JSON.parse(JSON.stringify(schema))), @@ -481,18 +483,28 @@ export abstract class SequelizeAdapter extends DatabaseAdapter protected abstract hasLegacyCollections(): Promise; private checkAndConvertIndexes( - //TODO: check types & options depending on dialect schemaName: string, indexes: ModelOptionsIndexes[], callerModule: string, ) { + const dialect = this.sequelize.getDialect(); for (const index of indexes) { + if ( + index.fields.some( + field => + !Object.keys(this.models[schemaName].originalSchema.compiledFields).includes( + field, + ), + ) + ) { + throw new Error(`Invalid fields for index creation`); + } if (!index.types && !index.options) continue; if (index.options) { - if (!checkIfSequelizeIndexOptions(index.options)) { + if (!checkSequelizeIndexOptions(index.options, dialect)) { throw new GrpcError( status.INVALID_ARGUMENT, - 'Invalid index options for PostgreSQL', + `Invalid index options for ${dialect}`, ); } if ( @@ -506,19 +518,19 @@ export abstract class SequelizeAdapter extends DatabaseAdapter } } if (index.types) { - if (isArray(index.types) || !(index.types in PostgresIndexType)) { + if (isArray(index.types) || !checkSequelizeIndexType(index.types, dialect)) { throw new GrpcError( status.INVALID_ARGUMENT, - 'Invalid index type for PostgreSQL', + `Invalid index type for ${dialect}`, ); } if (index.types in MySQLMariaDBIndexType) { - (index.options as SequelizeIndexOptions).type = + (index.options as MySQLMariaDBIndexOptions).type = index.types as MySQLMariaDBIndexType; } else { (index.options as SequelizeIndexOptions).using = index.types as | SQLIndexType - | PostgresIndexType + | PgIndexType | SQLiteIndexType; } delete index.types; diff --git a/modules/database/src/adapters/sequelize-adapter/postgres-adapter/PgSchemaConverter.ts b/modules/database/src/adapters/sequelize-adapter/postgres-adapter/PgSchemaConverter.ts index 24ba800bc..73c51487a 100644 --- a/modules/database/src/adapters/sequelize-adapter/postgres-adapter/PgSchemaConverter.ts +++ b/modules/database/src/adapters/sequelize-adapter/postgres-adapter/PgSchemaConverter.ts @@ -18,12 +18,16 @@ import { extractRelations, RelationType, } from '../utils/extractors'; +import { ConduitDatabaseSchema } from '../../../interfaces'; /** * This function should take as an input a JSON schema and convert it to the sequelize equivalent * @param jsonSchema */ -export function pgSchemaConverter(jsonSchema: ConduitSchema): [ +export function pgSchemaConverter( + jsonSchema: ConduitDatabaseSchema, + dialect: string, +): [ ConduitSchema, { [key: string]: { parentKey: string; childKey: string }; @@ -37,13 +41,13 @@ export function pgSchemaConverter(jsonSchema: ConduitSchema): [ delete copy.fields['_id']; } if (copy.modelOptions.indexes) { - copy = convertModelOptionsIndexes(copy); + copy = convertModelOptionsIndexes(copy, dialect); } const objectPaths: any = {}; convertObjectToDotNotation(jsonSchema.fields, copy.fields, objectPaths); const secondaryCopy = cloneDeep(copy.fields); const extractedRelations = extractRelations(secondaryCopy, copy.fields); - copy = convertSchemaFieldIndexes(copy); + copy = convertSchemaFieldIndexes(copy, dialect); iterDeep(secondaryCopy, copy.fields); return [copy, objectPaths, extractedRelations]; } diff --git a/modules/database/src/adapters/sequelize-adapter/sql-adapter/SqlSchemaConverter.ts b/modules/database/src/adapters/sequelize-adapter/sql-adapter/SqlSchemaConverter.ts index fc4c0f53d..be3b683fa 100644 --- a/modules/database/src/adapters/sequelize-adapter/sql-adapter/SqlSchemaConverter.ts +++ b/modules/database/src/adapters/sequelize-adapter/sql-adapter/SqlSchemaConverter.ts @@ -18,12 +18,16 @@ import { extractRelations, RelationType, } from '../utils/extractors'; +import { ConduitDatabaseSchema } from '../../../interfaces'; /** * This function should take as an input a JSON schema and convert it to the sequelize equivalent * @param jsonSchema */ -export function sqlSchemaConverter(jsonSchema: ConduitSchema): [ +export function sqlSchemaConverter( + jsonSchema: ConduitDatabaseSchema, + dialect: string, +): [ ConduitSchema, { [key: string]: { parentKey: string; childKey: string }; @@ -35,13 +39,13 @@ export function sqlSchemaConverter(jsonSchema: ConduitSchema): [ delete copy.fields['_id']; } if (copy.modelOptions.indexes) { - copy = convertModelOptionsIndexes(copy); + copy = convertModelOptionsIndexes(copy, dialect); } const objectPaths: any = {}; convertObjectToDotNotation(jsonSchema.fields, copy.fields, objectPaths); const secondaryCopy = cloneDeep(copy.fields); const extractedRelations = extractRelations(secondaryCopy, copy.fields); - copy = convertSchemaFieldIndexes(copy); + copy = convertSchemaFieldIndexes(copy, dialect); iterDeep(secondaryCopy, copy.fields); return [copy, objectPaths, extractedRelations]; } diff --git a/modules/database/src/adapters/sequelize-adapter/utils/index.ts b/modules/database/src/adapters/sequelize-adapter/utils/index.ts index 01ea8afbb..768dcf149 100644 --- a/modules/database/src/adapters/sequelize-adapter/utils/index.ts +++ b/modules/database/src/adapters/sequelize-adapter/utils/index.ts @@ -1,53 +1,3 @@ -import { - MongoIndexOptions, - MySQLMariaDBIndexType, - PostgresIndexType, - SequelizeIndexOptions, - SQLIndexType, - SQLiteIndexType, -} from '@conduitplatform/grpc-sdk'; - export * from './schema'; export * from './collectionUtils'; - -export function checkIfSequelizeIndexOptions( - options: MongoIndexOptions | SequelizeIndexOptions, -) { - const postgresOptions = [ - 'concurrently', - 'name', - 'operator', - 'parser', - 'prefix', - 'unique', - 'using', - 'where', - 'fields', - ]; - const result = Object.keys(options).some(option => !postgresOptions.includes(option)); - return !result; -} - -export function checkIfSequelizeIndexType(type: any) { - return ( - Object.values(SQLIndexType).includes(type as SQLIndexType) || - Object.values(PostgresIndexType).includes(type as PostgresIndexType) || - Object.values(MySQLMariaDBIndexType).includes(type as MySQLMariaDBIndexType) || - Object.values(SQLiteIndexType).includes(type as SQLiteIndexType) - ); -} - -export function checkSequelizeIndexType(type: any, dialect?: string) { - switch (dialect) { - case 'postgres': - return Object.values(PostgresIndexType).includes(type as PostgresIndexType); - case 'mysql': - return Object.values(MySQLMariaDBIndexType).includes(type as MySQLMariaDBIndexType); - case 'mariadb': - return Object.values(MySQLMariaDBIndexType).includes(type as MySQLMariaDBIndexType); - case 'sqlite': - return Object.values(SQLiteIndexType).includes(type as SQLiteIndexType); - default: - return; - } -} +export * from './indexChecks'; diff --git a/modules/database/src/adapters/sequelize-adapter/utils/indexChecks.ts b/modules/database/src/adapters/sequelize-adapter/utils/indexChecks.ts new file mode 100644 index 000000000..292d06234 --- /dev/null +++ b/modules/database/src/adapters/sequelize-adapter/utils/indexChecks.ts @@ -0,0 +1,46 @@ +import { + MySQLMariaDBIndexType, + PgIndexType, + SQLIndexType, + SQLiteIndexType, +} from '@conduitplatform/grpc-sdk'; + +export function checkSequelizeIndexType(type: any, dialect?: string) { + switch (dialect) { + case 'postgres': + return Object.values(PgIndexType).includes(type as PgIndexType); + case 'mysql' || 'mariadb': + return Object.values(MySQLMariaDBIndexType).includes(type as MySQLMariaDBIndexType); + case 'sqlite': + return Object.values(SQLiteIndexType).includes(type as SQLiteIndexType); + default: + return ( + Object.values(SQLIndexType).includes(type as SQLIndexType) || + Object.values(PgIndexType).includes(type as PgIndexType) || + Object.values(MySQLMariaDBIndexType).includes(type as MySQLMariaDBIndexType) || + Object.values(SQLiteIndexType).includes(type as SQLiteIndexType) + ); + } +} + +export function checkSequelizeIndexOptions(options: any, dialect?: string) { + const sequelizeOptions = [ + 'name', + 'parser', + 'unique', + 'fields', + 'where', + 'prefix', + 'using', + ]; + const pgOptions = sequelizeOptions.concat(['concurrently', 'operator']); + const mySqlMariaDbOptions = sequelizeOptions.concat(['type']); + switch (dialect) { + case 'postgres': + return !Object.keys(options).some(option => !pgOptions.includes(option)); + case 'mysql' || 'mariadb': + return !Object.keys(options).some(option => !mySqlMariaDbOptions.includes(option)); + default: + return !Object.keys(options).some(option => !sequelizeOptions.includes(option)); + } +} diff --git a/modules/database/src/adapters/utils/database-transform-utils.ts b/modules/database/src/adapters/utils/database-transform-utils.ts index 02beb8b9a..3f671a262 100644 --- a/modules/database/src/adapters/utils/database-transform-utils.ts +++ b/modules/database/src/adapters/utils/database-transform-utils.ts @@ -1,18 +1,17 @@ import { isArray, isBoolean, isNumber, isString } from 'lodash'; import { ConduitModelField, - ConduitSchema, Indexable, MySQLMariaDBIndexType, - PostgresIndexType, - SequelizeIndexType, + PgIndexType, SQLIndexType, SQLiteIndexType, } from '@conduitplatform/grpc-sdk'; import { - checkIfSequelizeIndexType, - checkIfSequelizeIndexOptions, + checkSequelizeIndexOptions, + checkSequelizeIndexType, } from '../sequelize-adapter/utils'; +import { ConduitDatabaseSchema } from '../../interfaces'; export function checkDefaultValue(type: string, value: string) { switch (type) { @@ -33,11 +32,14 @@ export function checkDefaultValue(type: string, value: string) { } } -export function convertModelOptionsIndexes(copy: ConduitSchema) { +export function convertModelOptionsIndexes(copy: ConduitDatabaseSchema, dialect: string) { for (const index of copy.modelOptions.indexes!) { + if (index.fields.some(field => !Object.keys(copy.compiledFields).includes(field))) { + throw new Error(`Invalid fields for index creation`); + } if (index.options) { - if (!checkIfSequelizeIndexOptions(index.options)) { - throw new Error('Incorrect index options for sequelize'); + if (!checkSequelizeIndexOptions(index.options, dialect)) { + throw new Error(`Invalid index options for ${dialect}`); } // if ((index.options as SequelizeIndexOptions).fields) { // TODO: integrate this logic somehow // delete index.fields; @@ -47,13 +49,13 @@ export function convertModelOptionsIndexes(copy: ConduitSchema) { } if (index.types) { // TODO: put null to check - if (isArray(index.types) || !checkIfSequelizeIndexType(index.types)) { - throw new Error('Invalid index type'); + if (isArray(index.types) || !checkSequelizeIndexType(index.types, dialect)) { + throw new Error(`Invalid index type for ${dialect}`); } if (index.types in MySQLMariaDBIndexType) { index.type = index.types as MySQLMariaDBIndexType; } else { - index.using = index.types as SQLIndexType | PostgresIndexType | SQLiteIndexType; + index.using = index.types as SQLIndexType | PgIndexType | SQLiteIndexType; } delete index.types; } @@ -61,11 +63,11 @@ export function convertModelOptionsIndexes(copy: ConduitSchema) { return copy; } -export function convertSchemaFieldIndexes(copy: ConduitSchema) { +export function convertSchemaFieldIndexes(copy: ConduitDatabaseSchema, dialect: string) { const indexes = []; for (const [fieldName, fieldValue] of Object.entries(copy.fields)) { const field = fieldValue as ConduitModelField; - // Move unique field indexes to modelOptions workaround + // Move unique field constraints to modelOptions workaround // if (field.unique && fieldName !== '_id') { // field.index = { // options: { unique: true } @@ -76,17 +78,17 @@ export function convertSchemaFieldIndexes(copy: ConduitSchema) { if (!index) continue; const newIndex: any = { fields: [fieldName] }; // TODO: remove this any if (index.type) { - if (isArray(index.type) || !checkIfSequelizeIndexType(index.type)) { - throw new Error('Invalid index type'); + if (isArray(index.type) || !checkSequelizeIndexType(index.type, dialect)) { + throw new Error(`Invalid index type for ${dialect}`); } if (!(index.type in MySQLMariaDBIndexType)) { - newIndex.using = index.type as SQLIndexType | PostgresIndexType | SQLiteIndexType; + newIndex.using = index.type as SQLIndexType | PgIndexType | SQLiteIndexType; } else { newIndex.type = index.type as MySQLMariaDBIndexType; } } - if (index.options && !checkIfSequelizeIndexOptions(index.options)) { - throw new Error('Invalid index options for sequelize'); + if (index.options && !checkSequelizeIndexOptions(index.options, dialect)) { + throw new Error(`Invalid index options for ${dialect}`); } Object.assign(newIndex, index.options); indexes.push(newIndex); From f55bdbed8180cc4475d0f35b4c77da0f9ead16f2 Mon Sep 17 00:00:00 2001 From: chris Date: Mon, 19 Jun 2023 16:18:08 +0300 Subject: [PATCH 04/17] refactor(database,grpc-sdk): refactoring for mongo & sequelize indexes & add fields option --- libraries/grpc-sdk/src/interfaces/Model.ts | 12 +++- .../mongoose-adapter/SchemaConverter.ts | 45 ++++++------ .../src/adapters/mongoose-adapter/index.ts | 17 +++-- .../src/adapters/mongoose-adapter/utils.ts | 2 +- .../src/adapters/sequelize-adapter/index.ts | 23 +++--- .../utils/database-transform-utils.ts | 70 +++++++++++-------- 6 files changed, 102 insertions(+), 67 deletions(-) diff --git a/libraries/grpc-sdk/src/interfaces/Model.ts b/libraries/grpc-sdk/src/interfaces/Model.ts index 80c50d04f..64e63a090 100644 --- a/libraries/grpc-sdk/src/interfaces/Model.ts +++ b/libraries/grpc-sdk/src/interfaces/Model.ts @@ -1,5 +1,4 @@ import { Document } from 'bson'; -import { IndexesOptions } from 'sequelize'; export enum TYPE { String = 'String', Number = 'Number', @@ -196,6 +195,15 @@ export type SequelizeIndexType = | MySQLMariaDBIndexType | SQLiteIndexType; +// Used for more complex index definitions +export interface SequelizeObjectIndexType { + name: string; + length?: number; + order?: 'ASC' | 'DESC'; + collate?: string; + operator?: string; // pg only +} + export interface MongoIndexOptions { background?: boolean; unique?: boolean; @@ -223,7 +231,7 @@ export interface SequelizeIndexOptions { name?: string; parser?: string | null; unique?: boolean; - // used instead of ModelOptionsIndexes fields for more complex index definitions + // Used instead of ModelOptionsIndexes fields for more complex index definitions fields?: { name: string; length?: number; diff --git a/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts b/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts index 8b4cbfcb5..a5630b118 100644 --- a/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts +++ b/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts @@ -6,7 +6,8 @@ import { SchemaFieldIndex, } from '@conduitplatform/grpc-sdk'; import { cloneDeep, isArray, isNil, isObject } from 'lodash'; -import { checkIfMongoOptions } from './utils'; +import { checkMongoOptions } from './utils'; +import { ConduitDatabaseSchema } from '../../interfaces'; const deepdash = require('deepdash/standalone'); @@ -71,7 +72,7 @@ function convert(value: any, key: any, parentValue: any) { _id: false, timestamps: false, }); - parentValue[key] = schemaConverter(typeSchema).fields; + parentValue[key] = schemaConverter(typeSchema as ConduitDatabaseSchema).fields; return true; } @@ -100,13 +101,12 @@ function convertSchemaFieldIndexes(copy: ConduitSchema) { for (const field of Object.entries(copy.fields)) { const index = (field[1] as ConduitModelField).index; if (!index) continue; - const type = index.type; - const options = index.options; - if (type && !Object.values(MongoIndexType).includes(type)) { + const { type, options } = index; + if (type && !(type in MongoIndexType)) { throw new Error('Incorrect index type for MongoDB'); } if (options) { - if (!checkIfMongoOptions(options)) { + if (!checkMongoOptions(options)) { throw new Error('Incorrect index options for MongoDB'); } for (const [option, optionValue] of Object.entries(options)) { @@ -120,31 +120,34 @@ function convertSchemaFieldIndexes(copy: ConduitSchema) { function convertModelOptionsIndexes(copy: ConduitSchema) { for (const index of copy.modelOptions.indexes!) { - // compound indexes are maintained in modelOptions in order to be created after schema creation - // single field index => add it to specified schema field - if (index.fields.length !== 1) continue; - const modelField = copy.fields[index.fields[0]] as ConduitModelField; - if (!modelField) { - throw new Error(`Field ${modelField} in index definition doesn't exist`); + // Compound indexes are maintained in modelOptions in order to be created after schema creation + // Single field index => add it to specified schema field + const { fields, types, options } = index; + if (fields.length === 0) { + throw new Error('Undefined fields for index creation'); + } + if (fields.some(field => !Object.keys(copy.fields).includes(field))) { + throw new Error(`Invalid fields for index creation`); } - if (index.types) { + if (fields.length !== 1) continue; + const modelField = copy.fields[index.fields[0]] as ConduitModelField; + if (types) { if ( - !isArray(index.types) || - !Object.values(MongoIndexType).includes(index.types[0]) || - index.fields.length !== index.types.length + !isArray(types) || + !(types[0] in MongoIndexType) || + fields.length !== types.length ) { throw new Error('Invalid index type for MongoDB'); } - const type = index.types[0] as MongoIndexType; modelField.index = { - type: type, + type: types[0] as MongoIndexType, }; } - if (index.options) { - if (!checkIfMongoOptions(index.options)) { + if (options) { + if (!checkMongoOptions(options)) { throw new Error('Incorrect index options for MongoDB'); } - for (const [option, optionValue] of Object.entries(index.options)) { + for (const [option, optionValue] of Object.entries(options)) { modelField.index![option as keyof SchemaFieldIndex] = optionValue; } } diff --git a/modules/database/src/adapters/mongoose-adapter/index.ts b/modules/database/src/adapters/mongoose-adapter/index.ts index 5305c93bb..dd9c4e78c 100644 --- a/modules/database/src/adapters/mongoose-adapter/index.ts +++ b/modules/database/src/adapters/mongoose-adapter/index.ts @@ -14,7 +14,7 @@ import { validateFieldChanges, validateFieldConstraints } from '../utils'; import pluralize from '../../utils/pluralize'; import { mongoSchemaConverter } from '../../introspection/mongoose/utils'; import { status } from '@grpc/grpc-js'; -import { checkIfMongoOptions } from './utils'; +import { checkMongoOptions } from './utils'; import { ConduitDatabaseSchema, introspectedSchemaCmsOptionsDefaults, @@ -431,11 +431,20 @@ export class MongooseAdapter extends DatabaseAdapter { callerModule: string, ) { for (const index of indexes) { - const options = index.options; - const types = index.types; + const { fields, types, options } = index; + if ( + fields.some( + field => + !Object.keys(this.models[schemaName].originalSchema.compiledFields).includes( + field, + ), + ) + ) { + throw new Error(`Invalid fields for index creation`); + } if (!options && !types) continue; if (options) { - if (!checkIfMongoOptions(options)) { + if (!checkMongoOptions(options)) { throw new GrpcError(status.INTERNAL, 'Invalid index options for mongoDB'); } if ( diff --git a/modules/database/src/adapters/mongoose-adapter/utils.ts b/modules/database/src/adapters/mongoose-adapter/utils.ts index ba46817e2..eba06f028 100644 --- a/modules/database/src/adapters/mongoose-adapter/utils.ts +++ b/modules/database/src/adapters/mongoose-adapter/utils.ts @@ -79,7 +79,7 @@ async function _createWithPopulations( } } -export function checkIfMongoOptions(options: MongoIndexOptions | SequelizeIndexOptions) { +export function checkMongoOptions(options: MongoIndexOptions | SequelizeIndexOptions) { const mongoOptions = [ 'background', 'unique', diff --git a/modules/database/src/adapters/sequelize-adapter/index.ts b/modules/database/src/adapters/sequelize-adapter/index.ts index 3d1d0e2ba..51fd15898 100644 --- a/modules/database/src/adapters/sequelize-adapter/index.ts +++ b/modules/database/src/adapters/sequelize-adapter/index.ts @@ -403,7 +403,7 @@ export abstract class SequelizeAdapter extends DatabaseAdapter fieldNames.push(field.attribute); } index.fields = fieldNames; - // extract index type from index definition + // Extract index type from index definition let tmp = index.definition.split('USING '); tmp = tmp[1].split(' '); index.types = tmp[0]; @@ -489,8 +489,9 @@ export abstract class SequelizeAdapter extends DatabaseAdapter ) { const dialect = this.sequelize.getDialect(); for (const index of indexes) { + const { fields, types, options } = index; if ( - index.fields.some( + fields.some( field => !Object.keys(this.models[schemaName].originalSchema.compiledFields).includes( field, @@ -499,16 +500,16 @@ export abstract class SequelizeAdapter extends DatabaseAdapter ) { throw new Error(`Invalid fields for index creation`); } - if (!index.types && !index.options) continue; - if (index.options) { - if (!checkSequelizeIndexOptions(index.options, dialect)) { + if (!types && !options) continue; + if (options) { + if (!checkSequelizeIndexOptions(options, dialect)) { throw new GrpcError( status.INVALID_ARGUMENT, `Invalid index options for ${dialect}`, ); } if ( - Object.keys(index.options).includes('unique') && + Object.keys(options).includes('unique') && this.models[schemaName].originalSchema.ownerModule !== callerModule ) { throw new GrpcError( @@ -517,18 +518,18 @@ export abstract class SequelizeAdapter extends DatabaseAdapter ); } } - if (index.types) { - if (isArray(index.types) || !checkSequelizeIndexType(index.types, dialect)) { + if (types) { + if (isArray(types) || !checkSequelizeIndexType(types, dialect)) { throw new GrpcError( status.INVALID_ARGUMENT, `Invalid index type for ${dialect}`, ); } - if (index.types in MySQLMariaDBIndexType) { + if (types in MySQLMariaDBIndexType) { (index.options as MySQLMariaDBIndexOptions).type = - index.types as MySQLMariaDBIndexType; + types as MySQLMariaDBIndexType; } else { - (index.options as SequelizeIndexOptions).using = index.types as + (index.options as SequelizeIndexOptions).using = types as | SQLIndexType | PgIndexType | SQLiteIndexType; diff --git a/modules/database/src/adapters/utils/database-transform-utils.ts b/modules/database/src/adapters/utils/database-transform-utils.ts index 3f671a262..0e4d939df 100644 --- a/modules/database/src/adapters/utils/database-transform-utils.ts +++ b/modules/database/src/adapters/utils/database-transform-utils.ts @@ -1,9 +1,10 @@ -import { isArray, isBoolean, isNumber, isString } from 'lodash'; +import { isArray, isBoolean, isNil, isNumber, isString } from 'lodash'; import { ConduitModelField, Indexable, MySQLMariaDBIndexType, PgIndexType, + SequelizeIndexOptions, SQLIndexType, SQLiteIndexType, } from '@conduitplatform/grpc-sdk'; @@ -34,28 +35,39 @@ export function checkDefaultValue(type: string, value: string) { export function convertModelOptionsIndexes(copy: ConduitDatabaseSchema, dialect: string) { for (const index of copy.modelOptions.indexes!) { - if (index.fields.some(field => !Object.keys(copy.compiledFields).includes(field))) { + const { fields, types, options } = index; + const compiledFields = Object.keys(copy.compiledFields); + if (fields.length === 0) { + throw new Error('Undefined fields for index creation'); + } + if (fields.some(field => !compiledFields.includes(field))) { throw new Error(`Invalid fields for index creation`); } - if (index.options) { - if (!checkSequelizeIndexOptions(index.options, dialect)) { + // Convert conduit indexes to sequelize indexes + if (options) { + if (!checkSequelizeIndexOptions(options, dialect)) { throw new Error(`Invalid index options for ${dialect}`); } - // if ((index.options as SequelizeIndexOptions).fields) { // TODO: integrate this logic somehow - // delete index.fields; - // } - Object.assign(index, index.options); + // Used instead of ModelOptionsIndexes fields for more complex index definitions + const seqOptions = options as SequelizeIndexOptions; + if ( + !isNil(seqOptions.fields) && + seqOptions.fields.every(f => compiledFields.includes(f.name)) + ) { + (index.fields as any) = seqOptions.fields; + delete (index.options as SequelizeIndexOptions).fields; + } + Object.assign(index, options); delete index.options; } - if (index.types) { - // TODO: put null to check - if (isArray(index.types) || !checkSequelizeIndexType(index.types, dialect)) { + if (types) { + if (isArray(types) || !checkSequelizeIndexType(types, dialect)) { throw new Error(`Invalid index type for ${dialect}`); } - if (index.types in MySQLMariaDBIndexType) { - index.type = index.types as MySQLMariaDBIndexType; + if (types in MySQLMariaDBIndexType) { + index.type = types as MySQLMariaDBIndexType; } else { - index.using = index.types as SQLIndexType | PgIndexType | SQLiteIndexType; + index.using = types as SQLIndexType | PgIndexType | SQLiteIndexType; } delete index.types; } @@ -68,29 +80,31 @@ export function convertSchemaFieldIndexes(copy: ConduitDatabaseSchema, dialect: for (const [fieldName, fieldValue] of Object.entries(copy.fields)) { const field = fieldValue as ConduitModelField; // Move unique field constraints to modelOptions workaround - // if (field.unique && fieldName !== '_id') { - // field.index = { - // options: { unique: true } - // }; - // delete (field as ConduitModelField).unique; - // } + if (field.unique) { + field.index = { + options: { unique: true }, + }; + delete (field as ConduitModelField).unique; + } const index = field.index; if (!index) continue; - const newIndex: any = { fields: [fieldName] }; // TODO: remove this any - if (index.type) { - if (isArray(index.type) || !checkSequelizeIndexType(index.type, dialect)) { + // Convert conduit indexes to sequelize indexes + const { type, options } = index; + const newIndex: any = { fields: [fieldName] }; + if (type) { + if (isArray(type) || !checkSequelizeIndexType(type, dialect)) { throw new Error(`Invalid index type for ${dialect}`); } - if (!(index.type in MySQLMariaDBIndexType)) { - newIndex.using = index.type as SQLIndexType | PgIndexType | SQLiteIndexType; + if (!(type in MySQLMariaDBIndexType)) { + newIndex.using = type as SQLIndexType | PgIndexType | SQLiteIndexType; } else { - newIndex.type = index.type as MySQLMariaDBIndexType; + newIndex.type = type as MySQLMariaDBIndexType; } } - if (index.options && !checkSequelizeIndexOptions(index.options, dialect)) { + if (options && !checkSequelizeIndexOptions(options, dialect)) { throw new Error(`Invalid index options for ${dialect}`); } - Object.assign(newIndex, index.options); + Object.assign(newIndex, options); indexes.push(newIndex); delete copy.fields[fieldName]; } From 9beb866f3b46d1bb36f24a742d38090a82c1cf66 Mon Sep 17 00:00:00 2001 From: chris Date: Mon, 19 Jun 2023 18:18:41 +0300 Subject: [PATCH 05/17] fix(database): some sequelize index type fixes --- libraries/grpc-sdk/src/interfaces/Model.ts | 15 +++++---------- .../src/adapters/sequelize-adapter/index.ts | 10 ++++------ .../sequelize-adapter/utils/indexChecks.ts | 16 ++++++---------- .../utils/database-transform-utils.ts | 19 +++++++++---------- modules/database/src/admin/index.ts | 2 +- 5 files changed, 25 insertions(+), 37 deletions(-) diff --git a/libraries/grpc-sdk/src/interfaces/Model.ts b/libraries/grpc-sdk/src/interfaces/Model.ts index 64e63a090..c55ceb486 100644 --- a/libraries/grpc-sdk/src/interfaces/Model.ts +++ b/libraries/grpc-sdk/src/interfaces/Model.ts @@ -167,12 +167,9 @@ export enum MongoIndexType { Text = 'text', } -export enum SQLIndexType { +export enum PgIndexType { BTREE = 'BTREE', HASH = 'HASH', -} - -export enum PgIndexType { GIST = 'GIST', SPGIST = 'SPGIST', GIN = 'GIN', @@ -180,6 +177,8 @@ export enum PgIndexType { } export enum MySQLMariaDBIndexType { + BTREE = 'BTREE', + HASH = 'HASH', UNIQUE = 'UNIQUE', FULLTEXT = 'FULLTEXT', SPATIAL = 'SPATIAL', @@ -189,11 +188,7 @@ export enum SQLiteIndexType { BTREE = 'BTREE', } -export type SequelizeIndexType = - | SQLIndexType - | PgIndexType - | MySQLMariaDBIndexType - | SQLiteIndexType; +export type SequelizeIndexType = PgIndexType | MySQLMariaDBIndexType | SQLiteIndexType; // Used for more complex index definitions export interface SequelizeObjectIndexType { @@ -243,7 +238,7 @@ export interface SequelizeIndexOptions { [opt: string]: any; }; prefix?: string; - using?: SQLIndexType | PgIndexType | SQLiteIndexType; + using?: PgIndexType | SQLiteIndexType; } export interface PgIndexOptions extends SequelizeIndexOptions { diff --git a/modules/database/src/adapters/sequelize-adapter/index.ts b/modules/database/src/adapters/sequelize-adapter/index.ts index 51fd15898..a3a63cc90 100644 --- a/modules/database/src/adapters/sequelize-adapter/index.ts +++ b/modules/database/src/adapters/sequelize-adapter/index.ts @@ -11,7 +11,6 @@ import ConduitGrpcSdk, { RawSQLQuery, SequelizeIndexOptions, sleep, - SQLIndexType, SQLiteIndexType, UntypedArray, } from '@conduitplatform/grpc-sdk'; @@ -20,8 +19,8 @@ import { SequelizeAuto } from 'sequelize-auto'; import { DatabaseAdapter } from '../DatabaseAdapter'; import { SequelizeSchema } from './SequelizeSchema'; import { - checkSequelizeIndexOptions, - checkSequelizeIndexType, + checkIfSequelizeIndexOptions, + checkIfSequelizeIndexType, compileSchema, resolveRelatedSchemas, tableFetch, @@ -502,7 +501,7 @@ export abstract class SequelizeAdapter extends DatabaseAdapter } if (!types && !options) continue; if (options) { - if (!checkSequelizeIndexOptions(options, dialect)) { + if (!checkIfSequelizeIndexOptions(options, dialect)) { throw new GrpcError( status.INVALID_ARGUMENT, `Invalid index options for ${dialect}`, @@ -519,7 +518,7 @@ export abstract class SequelizeAdapter extends DatabaseAdapter } } if (types) { - if (isArray(types) || !checkSequelizeIndexType(types, dialect)) { + if (isArray(types) || !checkIfSequelizeIndexType(types, dialect)) { throw new GrpcError( status.INVALID_ARGUMENT, `Invalid index type for ${dialect}`, @@ -530,7 +529,6 @@ export abstract class SequelizeAdapter extends DatabaseAdapter types as MySQLMariaDBIndexType; } else { (index.options as SequelizeIndexOptions).using = types as - | SQLIndexType | PgIndexType | SQLiteIndexType; } diff --git a/modules/database/src/adapters/sequelize-adapter/utils/indexChecks.ts b/modules/database/src/adapters/sequelize-adapter/utils/indexChecks.ts index 292d06234..f2aa5885e 100644 --- a/modules/database/src/adapters/sequelize-adapter/utils/indexChecks.ts +++ b/modules/database/src/adapters/sequelize-adapter/utils/indexChecks.ts @@ -1,29 +1,25 @@ import { MySQLMariaDBIndexType, PgIndexType, - SQLIndexType, SQLiteIndexType, } from '@conduitplatform/grpc-sdk'; -export function checkSequelizeIndexType(type: any, dialect?: string) { +export function checkIfSequelizeIndexType(type: any, dialect?: string) { switch (dialect) { case 'postgres': - return Object.values(PgIndexType).includes(type as PgIndexType); + return type in PgIndexType; case 'mysql' || 'mariadb': - return Object.values(MySQLMariaDBIndexType).includes(type as MySQLMariaDBIndexType); + return type in MySQLMariaDBIndexType; case 'sqlite': - return Object.values(SQLiteIndexType).includes(type as SQLiteIndexType); + return type in SQLiteIndexType; default: return ( - Object.values(SQLIndexType).includes(type as SQLIndexType) || - Object.values(PgIndexType).includes(type as PgIndexType) || - Object.values(MySQLMariaDBIndexType).includes(type as MySQLMariaDBIndexType) || - Object.values(SQLiteIndexType).includes(type as SQLiteIndexType) + type in PgIndexType || type in MySQLMariaDBIndexType || type in SQLiteIndexType ); } } -export function checkSequelizeIndexOptions(options: any, dialect?: string) { +export function checkIfSequelizeIndexOptions(options: any, dialect?: string) { const sequelizeOptions = [ 'name', 'parser', diff --git a/modules/database/src/adapters/utils/database-transform-utils.ts b/modules/database/src/adapters/utils/database-transform-utils.ts index 0e4d939df..d07234d4e 100644 --- a/modules/database/src/adapters/utils/database-transform-utils.ts +++ b/modules/database/src/adapters/utils/database-transform-utils.ts @@ -5,12 +5,11 @@ import { MySQLMariaDBIndexType, PgIndexType, SequelizeIndexOptions, - SQLIndexType, SQLiteIndexType, } from '@conduitplatform/grpc-sdk'; import { - checkSequelizeIndexOptions, - checkSequelizeIndexType, + checkIfSequelizeIndexOptions, + checkIfSequelizeIndexType, } from '../sequelize-adapter/utils'; import { ConduitDatabaseSchema } from '../../interfaces'; @@ -45,7 +44,7 @@ export function convertModelOptionsIndexes(copy: ConduitDatabaseSchema, dialect: } // Convert conduit indexes to sequelize indexes if (options) { - if (!checkSequelizeIndexOptions(options, dialect)) { + if (!checkIfSequelizeIndexOptions(options, dialect)) { throw new Error(`Invalid index options for ${dialect}`); } // Used instead of ModelOptionsIndexes fields for more complex index definitions @@ -61,13 +60,13 @@ export function convertModelOptionsIndexes(copy: ConduitDatabaseSchema, dialect: delete index.options; } if (types) { - if (isArray(types) || !checkSequelizeIndexType(types, dialect)) { + if (isArray(types) || !checkIfSequelizeIndexType(types, dialect)) { throw new Error(`Invalid index type for ${dialect}`); } if (types in MySQLMariaDBIndexType) { index.type = types as MySQLMariaDBIndexType; } else { - index.using = types as SQLIndexType | PgIndexType | SQLiteIndexType; + index.using = types as PgIndexType | SQLiteIndexType; } delete index.types; } @@ -92,21 +91,21 @@ export function convertSchemaFieldIndexes(copy: ConduitDatabaseSchema, dialect: const { type, options } = index; const newIndex: any = { fields: [fieldName] }; if (type) { - if (isArray(type) || !checkSequelizeIndexType(type, dialect)) { + if (isArray(type) || !checkIfSequelizeIndexType(type, dialect)) { throw new Error(`Invalid index type for ${dialect}`); } if (!(type in MySQLMariaDBIndexType)) { - newIndex.using = type as SQLIndexType | PgIndexType | SQLiteIndexType; + newIndex.using = type as PgIndexType | SQLiteIndexType; } else { newIndex.type = type as MySQLMariaDBIndexType; } } - if (options && !checkSequelizeIndexOptions(options, dialect)) { + if (options && !checkIfSequelizeIndexOptions(options, dialect)) { throw new Error(`Invalid index options for ${dialect}`); } Object.assign(newIndex, options); indexes.push(newIndex); - delete copy.fields[fieldName]; + delete (copy.fields[fieldName] as ConduitModelField).index; } copy.modelOptions.indexes = copy.modelOptions.indexes ? [...copy.modelOptions.indexes, ...indexes] diff --git a/modules/database/src/admin/index.ts b/modules/database/src/admin/index.ts index e2c954012..40e30919b 100644 --- a/modules/database/src/admin/index.ts +++ b/modules/database/src/admin/index.ts @@ -619,7 +619,7 @@ export class AdminHandlers { }, }, new ConduitRouteReturnDefinition('getSchemaIndexes', { - indexes: [ConduitJson.Required], + indexes: [ConduitJson.Required], // TODO: define this more clearly if possible }), this.schemaAdmin.getIndexes.bind(this.schemaAdmin), ); From 83f29902fc764221072f8e602271c10368624710 Mon Sep 17 00:00:00 2001 From: chris Date: Wed, 21 Jun 2023 12:07:30 +0300 Subject: [PATCH 06/17] feat(database)!: changes in index endpoints & wip import/export --- libraries/grpc-sdk/src/interfaces/Model.ts | 13 +- .../database/src/adapters/DatabaseAdapter.ts | 8 +- .../src/adapters/mongoose-adapter/index.ts | 109 ++++++++-------- .../src/adapters/sequelize-adapter/index.ts | 119 +++++++++--------- .../utils/database-transform-utils.ts | 28 ++--- modules/database/src/admin/index.ts | 48 ++++++- modules/database/src/admin/schema.admin.ts | 38 +++++- 7 files changed, 209 insertions(+), 154 deletions(-) diff --git a/libraries/grpc-sdk/src/interfaces/Model.ts b/libraries/grpc-sdk/src/interfaces/Model.ts index c55ceb486..2f858b733 100644 --- a/libraries/grpc-sdk/src/interfaces/Model.ts +++ b/libraries/grpc-sdk/src/interfaces/Model.ts @@ -140,19 +140,18 @@ export interface ConduitSchemaOptions { enabled: boolean; }; }; - indexes?: ModelOptionsIndexes[]; + indexes?: ModelOptionsIndex[]; } // Index types export interface SchemaFieldIndex { type?: MongoIndexType | SequelizeIndexType; options?: MongoIndexOptions | SequelizeIndexOptions; - //[field: string]: any; } -export interface ModelOptionsIndexes { +export interface ModelOptionsIndex { fields: string[]; - types?: MongoIndexType[] | SequelizeIndexType; + types?: MongoIndexType[] | SequelizeIndexType[]; options?: MongoIndexOptions | SequelizeIndexOptions; [field: string]: any; } @@ -238,14 +237,18 @@ export interface SequelizeIndexOptions { [opt: string]: any; }; prefix?: string; - using?: PgIndexType | SQLiteIndexType; } export interface PgIndexOptions extends SequelizeIndexOptions { concurrently?: boolean; operator?: string; + using?: PgIndexType; } export interface MySQLMariaDBIndexOptions extends SequelizeIndexOptions { type?: MySQLMariaDBIndexType; } + +export interface SQLiteIndexOptions extends SequelizeIndexOptions { + using?: SQLiteIndexType; +} diff --git a/modules/database/src/adapters/DatabaseAdapter.ts b/modules/database/src/adapters/DatabaseAdapter.ts index 4de579399..c64f2203f 100644 --- a/modules/database/src/adapters/DatabaseAdapter.ts +++ b/modules/database/src/adapters/DatabaseAdapter.ts @@ -2,7 +2,7 @@ import ConduitGrpcSdk, { ConduitModel, ConduitSchema, GrpcError, - ModelOptionsIndexes, + ModelOptionsIndex, RawMongoQuery, RawSQLQuery, TYPE, @@ -132,13 +132,13 @@ export abstract class DatabaseAdapter { abstract getDatabaseType(): string; - abstract createIndexes( + abstract createIndex( schemaName: string, - indexes: ModelOptionsIndexes[], + index: ModelOptionsIndex, callerModule: string, ): Promise; - abstract getIndexes(schemaName: string): Promise; + abstract getIndexes(schemaName: string): Promise; abstract createView( modelName: string, diff --git a/modules/database/src/adapters/mongoose-adapter/index.ts b/modules/database/src/adapters/mongoose-adapter/index.ts index dd9c4e78c..8c73f719e 100644 --- a/modules/database/src/adapters/mongoose-adapter/index.ts +++ b/modules/database/src/adapters/mongoose-adapter/index.ts @@ -5,7 +5,7 @@ import ConduitGrpcSdk, { ConduitSchema, GrpcError, Indexable, - ModelOptionsIndexes, + ModelOptionsIndex, MongoIndexType, RawMongoQuery, } from '@conduitplatform/grpc-sdk'; @@ -239,32 +239,30 @@ export class MongooseAdapter extends DatabaseAdapter { return 'MongoDB'; } - async createIndexes( + async createIndex( schemaName: string, - indexes: ModelOptionsIndexes[], + index: ModelOptionsIndex, callerModule: string, ): Promise { if (!this.models[schemaName]) throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); - this.checkIndexes(schemaName, indexes, callerModule); + this.checkIndex(schemaName, index, callerModule); const collection = this.mongoose.model(schemaName).collection; - for (const index of indexes) { - const indexSpecs = []; - for (let i = 0; i < index.fields.length; i++) { - const spec: any = {}; - spec[index.fields[i]] = index.types ? index.types[i] : 1; - indexSpecs.push(spec); - } - await collection - .createIndex(indexSpecs, index.options as IndexOptions) - .catch((e: Error) => { - throw new GrpcError(status.INTERNAL, e.message); - }); + const indexSpecs = []; + for (let i = 0; i < index.fields.length; i++) { + const spec: any = {}; + spec[index.fields[i]] = index.types ? index.types[i] : 1; + indexSpecs.push(spec); } - return 'Indexes created!'; + await collection + .createIndex(indexSpecs, index.options as IndexOptions) + .catch((e: Error) => { + throw new GrpcError(status.INTERNAL, e.message); + }); + return 'Index created!'; } - async getIndexes(schemaName: string): Promise { + async getIndexes(schemaName: string): Promise { if (!this.models[schemaName]) throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); const collection = this.mongoose.model(schemaName).collection; @@ -290,7 +288,7 @@ export class MongooseAdapter extends DatabaseAdapter { delete index.key; } }); - return result as ModelOptionsIndexes[]; + return result as ModelOptionsIndex[]; } async deleteIndexes(schemaName: string, indexNames: string[]): Promise { @@ -418,54 +416,47 @@ export class MongooseAdapter extends DatabaseAdapter { await this.compareAndStoreMigratedSchema(schema); await this.saveSchemaToDatabase(schema); } - if (indexes) { - await this.createIndexes(schema.name, indexes, schema.ownerModule); + for (const i of indexes) { + await this.createIndex(schema.name, i, schema.ownerModule); + } } return this.models[schema.name]; } - private checkIndexes( - schemaName: string, - indexes: ModelOptionsIndexes[], - callerModule: string, - ) { - for (const index of indexes) { - const { fields, types, options } = index; + private checkIndex(schemaName: string, index: ModelOptionsIndex, callerModule: string) { + const { fields, types, options } = index; + if ( + fields.some( + field => + !Object.keys(this.models[schemaName].originalSchema.compiledFields).includes( + field, + ), + ) + ) { + throw new Error(`Invalid fields for index creation`); + } + if (!options && !types) return; + if (options) { + if (!checkMongoOptions(options)) { + throw new GrpcError(status.INTERNAL, 'Invalid index options for mongoDB'); + } if ( - fields.some( - field => - !Object.keys(this.models[schemaName].originalSchema.compiledFields).includes( - field, - ), - ) + Object.keys(options).includes('unique') && + this.models[schemaName].originalSchema.ownerModule !== callerModule ) { - throw new Error(`Invalid fields for index creation`); - } - if (!options && !types) continue; - if (options) { - if (!checkMongoOptions(options)) { - throw new GrpcError(status.INTERNAL, 'Invalid index options for mongoDB'); - } - if ( - Object.keys(options).includes('unique') && - this.models[schemaName].originalSchema.ownerModule !== callerModule - ) { - throw new GrpcError( - status.PERMISSION_DENIED, - 'Not authorized to create unique index', - ); - } + throw new GrpcError( + status.PERMISSION_DENIED, + 'Not authorized to create unique index', + ); } - if (types) { - if (!Array.isArray(types) || types.length !== index.fields.length) { - throw new GrpcError(status.INTERNAL, 'Invalid index types format'); - } - for (const type of types) { - if (!Object.values(MongoIndexType).includes(type)) { - throw new GrpcError(status.INTERNAL, 'Invalid index type for mongoDB'); - } - } + } + if (types) { + if ( + types.length !== index.fields.length || + types.some(type => !(type in MongoIndexType)) + ) { + throw new GrpcError(status.INTERNAL, 'Invalid index types'); } } } diff --git a/modules/database/src/adapters/sequelize-adapter/index.ts b/modules/database/src/adapters/sequelize-adapter/index.ts index a3a63cc90..95bf6aefd 100644 --- a/modules/database/src/adapters/sequelize-adapter/index.ts +++ b/modules/database/src/adapters/sequelize-adapter/index.ts @@ -4,13 +4,15 @@ import ConduitGrpcSdk, { ConduitSchema, GrpcError, Indexable, - ModelOptionsIndexes, + ModelOptionsIndex, MySQLMariaDBIndexOptions, MySQLMariaDBIndexType, + PgIndexOptions, PgIndexType, RawSQLQuery, SequelizeIndexOptions, sleep, + SQLiteIndexOptions, SQLiteIndexType, UntypedArray, } from '@conduitplatform/grpc-sdk'; @@ -371,31 +373,35 @@ export abstract class SequelizeAdapter extends DatabaseAdapter return type; } - async createIndexes( + async createIndex( schemaName: string, - indexes: ModelOptionsIndexes[], + index: ModelOptionsIndex, callerModule: string, ): Promise { if (!this.models[schemaName]) throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); - indexes = this.checkAndConvertIndexes(schemaName, indexes, callerModule); + index = this.checkAndConvertIndex(schemaName, index, callerModule); const queryInterface = this.sequelize.getQueryInterface(); - for (const index of indexes) { - await queryInterface - .addIndex('cnd_' + schemaName, index.fields, index.options) - .catch(() => { - throw new GrpcError(status.INTERNAL, 'Unsuccessful index creation'); - }); - } + await queryInterface + .addIndex( + this.models[schemaName].originalSchema.collectionName, + index.fields, + index.options, + ) + .catch(() => { + throw new GrpcError(status.INTERNAL, 'Unsuccessful index creation'); + }); await this.models[schemaName].sync(); - return 'Indexes created!'; + return 'Index created!'; } - async getIndexes(schemaName: string): Promise { + async getIndexes(schemaName: string): Promise { if (!this.models[schemaName]) throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); const queryInterface = this.sequelize.getQueryInterface(); - const result = (await queryInterface.showIndex('cnd_' + schemaName)) as UntypedArray; + const result = (await queryInterface.showIndex( + this.models[schemaName].originalSchema.collectionName, + )) as UntypedArray; result.filter(index => { const fieldNames = []; for (const field of index.fields) { @@ -405,7 +411,7 @@ export abstract class SequelizeAdapter extends DatabaseAdapter // Extract index type from index definition let tmp = index.definition.split('USING '); tmp = tmp[1].split(' '); - index.types = tmp[0]; + index.types = [tmp[0]]; delete index.definition; index.options = {}; for (const indexEntry of Object.entries(index)) { @@ -481,60 +487,53 @@ export abstract class SequelizeAdapter extends DatabaseAdapter protected abstract hasLegacyCollections(): Promise; - private checkAndConvertIndexes( + private checkAndConvertIndex( schemaName: string, - indexes: ModelOptionsIndexes[], + index: ModelOptionsIndex, callerModule: string, ) { const dialect = this.sequelize.getDialect(); - for (const index of indexes) { - const { fields, types, options } = index; + const { fields, types, options } = index; + if ( + fields.some( + field => + !Object.keys(this.models[schemaName].originalSchema.compiledFields).includes( + field, + ), + ) + ) { + throw new Error(`Invalid fields for index creation`); + } + if (!types && !options) return index; + if (options) { + if (!checkIfSequelizeIndexOptions(options, dialect)) { + throw new GrpcError( + status.INVALID_ARGUMENT, + `Invalid index options for ${dialect}`, + ); + } if ( - fields.some( - field => - !Object.keys(this.models[schemaName].originalSchema.compiledFields).includes( - field, - ), - ) + Object.keys(options).includes('unique') && + this.models[schemaName].originalSchema.ownerModule !== callerModule ) { - throw new Error(`Invalid fields for index creation`); + throw new GrpcError( + status.PERMISSION_DENIED, + 'Not authorized to create unique index', + ); } - if (!types && !options) continue; - if (options) { - if (!checkIfSequelizeIndexOptions(options, dialect)) { - throw new GrpcError( - status.INVALID_ARGUMENT, - `Invalid index options for ${dialect}`, - ); - } - if ( - Object.keys(options).includes('unique') && - this.models[schemaName].originalSchema.ownerModule !== callerModule - ) { - throw new GrpcError( - status.PERMISSION_DENIED, - 'Not authorized to create unique index', - ); - } + } + if (types) { + if (types.length !== 1 || !checkIfSequelizeIndexType(types[0], dialect)) { + throw new GrpcError(status.INVALID_ARGUMENT, `Invalid index type for ${dialect}`); } - if (types) { - if (isArray(types) || !checkIfSequelizeIndexType(types, dialect)) { - throw new GrpcError( - status.INVALID_ARGUMENT, - `Invalid index type for ${dialect}`, - ); - } - if (types in MySQLMariaDBIndexType) { - (index.options as MySQLMariaDBIndexOptions).type = - types as MySQLMariaDBIndexType; - } else { - (index.options as SequelizeIndexOptions).using = types as - | PgIndexType - | SQLiteIndexType; - } - delete index.types; + if (dialect === 'mysql' || dialect === 'mariadb') { + (index.options as MySQLMariaDBIndexOptions).type = + types[0] as MySQLMariaDBIndexType; + } else if (dialect === 'postgres') { + (index.options as PgIndexOptions).using = types[0] as PgIndexType; } + delete index.types; } - return indexes; + return index; } } diff --git a/modules/database/src/adapters/utils/database-transform-utils.ts b/modules/database/src/adapters/utils/database-transform-utils.ts index d07234d4e..bcbecb171 100644 --- a/modules/database/src/adapters/utils/database-transform-utils.ts +++ b/modules/database/src/adapters/utils/database-transform-utils.ts @@ -2,7 +2,9 @@ import { isArray, isBoolean, isNil, isNumber, isString } from 'lodash'; import { ConduitModelField, Indexable, + MySQLMariaDBIndexOptions, MySQLMariaDBIndexType, + PgIndexOptions, PgIndexType, SequelizeIndexOptions, SQLiteIndexType, @@ -60,13 +62,13 @@ export function convertModelOptionsIndexes(copy: ConduitDatabaseSchema, dialect: delete index.options; } if (types) { - if (isArray(types) || !checkIfSequelizeIndexType(types, dialect)) { + if (types.length !== 1 || !checkIfSequelizeIndexType(types[0], dialect)) { throw new Error(`Invalid index type for ${dialect}`); } - if (types in MySQLMariaDBIndexType) { - index.type = types as MySQLMariaDBIndexType; - } else { - index.using = types as PgIndexType | SQLiteIndexType; + if (dialect === 'mysql' || dialect === 'mariadb') { + index.type = types[0] as MySQLMariaDBIndexType; + } else if (dialect === 'postgres') { + index.using = types[0] as PgIndexType | SQLiteIndexType; } delete index.types; } @@ -77,15 +79,7 @@ export function convertModelOptionsIndexes(copy: ConduitDatabaseSchema, dialect: export function convertSchemaFieldIndexes(copy: ConduitDatabaseSchema, dialect: string) { const indexes = []; for (const [fieldName, fieldValue] of Object.entries(copy.fields)) { - const field = fieldValue as ConduitModelField; - // Move unique field constraints to modelOptions workaround - if (field.unique) { - field.index = { - options: { unique: true }, - }; - delete (field as ConduitModelField).unique; - } - const index = field.index; + const index = (fieldValue as ConduitModelField).index; if (!index) continue; // Convert conduit indexes to sequelize indexes const { type, options } = index; @@ -94,10 +88,10 @@ export function convertSchemaFieldIndexes(copy: ConduitDatabaseSchema, dialect: if (isArray(type) || !checkIfSequelizeIndexType(type, dialect)) { throw new Error(`Invalid index type for ${dialect}`); } - if (!(type in MySQLMariaDBIndexType)) { - newIndex.using = type as PgIndexType | SQLiteIndexType; - } else { + if (dialect === 'mysql' || dialect === 'mariadb') { newIndex.type = type as MySQLMariaDBIndexType; + } else if (dialect === 'postgres') { + newIndex.using = type as PgIndexType; } } if (options && !checkIfSequelizeIndexOptions(options, dialect)) { diff --git a/modules/database/src/admin/index.ts b/modules/database/src/admin/index.ts index 40e30919b..48df6cb59 100644 --- a/modules/database/src/admin/index.ts +++ b/modules/database/src/admin/index.ts @@ -594,20 +594,52 @@ export class AdminHandlers { }), this.customEndpointsAdmin.schemaDetailsForOperation.bind(this.customEndpointsAdmin), ); + this.routingManager.route( + { + path: '/schemas/indexes/import', + action: ConduitRouteActions.POST, + description: `Imports indexes.`, + bodyParams: { + indexes: [ConduitJson.Required], + }, + }, + new ConduitRouteReturnDefinition('ImportIndexes', 'String'), + this.schemaAdmin.importIndexes.bind(this.schemaAdmin), + ); + this.routingManager.route( + { + path: '/schemas/indexes/export', + action: ConduitRouteActions.GET, + description: `Exports indexes.`, + }, + new ConduitRouteReturnDefinition('ExportIndexes', { + indexes: [ + { + schemaName: ConduitString.Required, + fields: [ConduitString.Required], + types: [ConduitString.Required], + options: ConduitJson.Optional, + }, + ], + }), + this.schemaAdmin.exportIndexes.bind(this.schemaAdmin), + ); this.routingManager.route( { path: '/schemas/:id/indexes', action: ConduitRouteActions.POST, - description: `Creates indexes for a schema.`, + description: `Creates an index for a schema.`, urlParams: { id: { type: TYPE.String, required: true }, }, bodyParams: { - indexes: [ConduitJson.Required], + fields: [ConduitString.Required], + types: [ConduitString.Required], + options: ConduitJson.Optional, }, }, - new ConduitRouteReturnDefinition('CreateSchemaIndexes', 'String'), - this.schemaAdmin.createIndexes.bind(this.schemaAdmin), + new ConduitRouteReturnDefinition('CreateSchemaIndex', 'String'), + this.schemaAdmin.createIndex.bind(this.schemaAdmin), ); this.routingManager.route( { @@ -619,7 +651,13 @@ export class AdminHandlers { }, }, new ConduitRouteReturnDefinition('getSchemaIndexes', { - indexes: [ConduitJson.Required], // TODO: define this more clearly if possible + indexes: [ + { + fields: [ConduitString.Required], + types: [ConduitString.Required], + options: ConduitJson.Optional, // TODO: check this and the other places + }, + ], }), this.schemaAdmin.getIndexes.bind(this.schemaAdmin), ); diff --git a/modules/database/src/admin/schema.admin.ts b/modules/database/src/admin/schema.admin.ts index fbdb7bc35..19324da4f 100644 --- a/modules/database/src/admin/schema.admin.ts +++ b/modules/database/src/admin/schema.admin.ts @@ -581,15 +581,44 @@ export class SchemaAdmin { return this.database.getDatabaseType(); } - async createIndexes(call: ParsedRouterRequest): Promise { - const { id, indexes } = call.request.params; + async importIndexes(call: ParsedRouterRequest): Promise { + for (const index of call.request.params.indexes) { + await this.database.createIndex(index.schemaName, index, 'database').catch(e => { + throw new GrpcError(status.INTERNAL, `${index.options.name}: ${e.message}`); + }); + } + return 'Indexes imported successfully'; + } + + async exportIndexes(): Promise { + const indexes = []; + const schemas = await this.database + .getSchemaModel('_DeclaredSchema') + .model.findMany({}); + for (const schema of schemas) { + const schemaIndexes = await this.database.getIndexes(schema.name); + if (!isNil(schemaIndexes) && !isEmpty(schemaIndexes)) { + indexes.push( + ...schemaIndexes.map(index => ({ ...index, schemaName: schema.name })), + ); + } + } + return { indexes }; + } + + async createIndex(call: ParsedRouterRequest): Promise { + const { id, fields, types, options } = call.request.params; const requestedSchema = await this.database .getSchemaModel('_DeclaredSchema') .model.findOne({ _id: id }); if (isNil(requestedSchema)) { throw new GrpcError(status.NOT_FOUND, 'Schema does not exist'); } - return await this.database.createIndexes(requestedSchema.name, indexes, 'database'); + return await this.database.createIndex( + requestedSchema.name, + { fields, types, options }, + 'database', + ); } async getIndexes(call: ParsedRouterRequest): Promise { @@ -600,7 +629,8 @@ export class SchemaAdmin { if (isNil(requestedSchema)) { throw new GrpcError(status.NOT_FOUND, 'Schema does not exist'); } - return this.database.getIndexes(requestedSchema.name); + const indexes = await this.database.getIndexes(requestedSchema.name); + return { indexes }; } async deleteIndexes(call: ParsedRouterRequest): Promise { From 7cf050a2f4ffd82fac1ec80fa0b51e7308f6be09 Mon Sep 17 00:00:00 2001 From: chris Date: Wed, 21 Jun 2023 12:16:49 +0300 Subject: [PATCH 07/17] chore: remove unused interface --- libraries/grpc-sdk/src/interfaces/Model.ts | 9 --------- 1 file changed, 9 deletions(-) diff --git a/libraries/grpc-sdk/src/interfaces/Model.ts b/libraries/grpc-sdk/src/interfaces/Model.ts index 2f858b733..ef68dce0f 100644 --- a/libraries/grpc-sdk/src/interfaces/Model.ts +++ b/libraries/grpc-sdk/src/interfaces/Model.ts @@ -189,15 +189,6 @@ export enum SQLiteIndexType { export type SequelizeIndexType = PgIndexType | MySQLMariaDBIndexType | SQLiteIndexType; -// Used for more complex index definitions -export interface SequelizeObjectIndexType { - name: string; - length?: number; - order?: 'ASC' | 'DESC'; - collate?: string; - operator?: string; // pg only -} - export interface MongoIndexOptions { background?: boolean; unique?: boolean; From db63ce7fd19c3de121a9b7b9ac91e30906f11bcb Mon Sep 17 00:00:00 2001 From: chris Date: Wed, 21 Jun 2023 12:20:49 +0300 Subject: [PATCH 08/17] chore: rename function but to original --- .../src/adapters/mongoose-adapter/SchemaConverter.ts | 6 +++--- modules/database/src/adapters/mongoose-adapter/index.ts | 4 ++-- modules/database/src/adapters/mongoose-adapter/utils.ts | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts b/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts index a5630b118..f56a41b7f 100644 --- a/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts +++ b/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts @@ -6,7 +6,7 @@ import { SchemaFieldIndex, } from '@conduitplatform/grpc-sdk'; import { cloneDeep, isArray, isNil, isObject } from 'lodash'; -import { checkMongoOptions } from './utils'; +import { checkIfMongoOptions } from './utils'; import { ConduitDatabaseSchema } from '../../interfaces'; const deepdash = require('deepdash/standalone'); @@ -106,7 +106,7 @@ function convertSchemaFieldIndexes(copy: ConduitSchema) { throw new Error('Incorrect index type for MongoDB'); } if (options) { - if (!checkMongoOptions(options)) { + if (!checkIfMongoOptions(options)) { throw new Error('Incorrect index options for MongoDB'); } for (const [option, optionValue] of Object.entries(options)) { @@ -144,7 +144,7 @@ function convertModelOptionsIndexes(copy: ConduitSchema) { }; } if (options) { - if (!checkMongoOptions(options)) { + if (!checkIfMongoOptions(options)) { throw new Error('Incorrect index options for MongoDB'); } for (const [option, optionValue] of Object.entries(options)) { diff --git a/modules/database/src/adapters/mongoose-adapter/index.ts b/modules/database/src/adapters/mongoose-adapter/index.ts index 8c73f719e..0ce8b8658 100644 --- a/modules/database/src/adapters/mongoose-adapter/index.ts +++ b/modules/database/src/adapters/mongoose-adapter/index.ts @@ -14,7 +14,7 @@ import { validateFieldChanges, validateFieldConstraints } from '../utils'; import pluralize from '../../utils/pluralize'; import { mongoSchemaConverter } from '../../introspection/mongoose/utils'; import { status } from '@grpc/grpc-js'; -import { checkMongoOptions } from './utils'; +import { checkIfMongoOptions } from './utils'; import { ConduitDatabaseSchema, introspectedSchemaCmsOptionsDefaults, @@ -438,7 +438,7 @@ export class MongooseAdapter extends DatabaseAdapter { } if (!options && !types) return; if (options) { - if (!checkMongoOptions(options)) { + if (!checkIfMongoOptions(options)) { throw new GrpcError(status.INTERNAL, 'Invalid index options for mongoDB'); } if ( diff --git a/modules/database/src/adapters/mongoose-adapter/utils.ts b/modules/database/src/adapters/mongoose-adapter/utils.ts index eba06f028..ba46817e2 100644 --- a/modules/database/src/adapters/mongoose-adapter/utils.ts +++ b/modules/database/src/adapters/mongoose-adapter/utils.ts @@ -79,7 +79,7 @@ async function _createWithPopulations( } } -export function checkMongoOptions(options: MongoIndexOptions | SequelizeIndexOptions) { +export function checkIfMongoOptions(options: MongoIndexOptions | SequelizeIndexOptions) { const mongoOptions = [ 'background', 'unique', From 82b8b6317da174488ac54937b9158c665748153e Mon Sep 17 00:00:00 2001 From: chris Date: Tue, 27 Jun 2023 15:58:03 +0300 Subject: [PATCH 09/17] fix(database,grpc-sdk): fix index functions not considering updating schema indexes --- libraries/grpc-sdk/src/interfaces/Model.ts | 4 +- .../mongoose-adapter/SchemaConverter.ts | 3 +- .../src/adapters/mongoose-adapter/index.ts | 63 +++++++----- .../src/adapters/sequelize-adapter/index.ts | 95 ++++++++----------- .../utils/database-transform-utils.ts | 36 ++++++- modules/database/src/admin/index.ts | 7 +- modules/database/src/admin/schema.admin.ts | 4 +- modules/database/src/utils/utilities.ts | 33 ++++++- 8 files changed, 151 insertions(+), 94 deletions(-) diff --git a/libraries/grpc-sdk/src/interfaces/Model.ts b/libraries/grpc-sdk/src/interfaces/Model.ts index ef68dce0f..62476f33f 100644 --- a/libraries/grpc-sdk/src/interfaces/Model.ts +++ b/libraries/grpc-sdk/src/interfaces/Model.ts @@ -145,11 +145,13 @@ export interface ConduitSchemaOptions { // Index types export interface SchemaFieldIndex { + name: string; type?: MongoIndexType | SequelizeIndexType; options?: MongoIndexOptions | SequelizeIndexOptions; } export interface ModelOptionsIndex { + name: string; fields: string[]; types?: MongoIndexType[] | SequelizeIndexType[]; options?: MongoIndexOptions | SequelizeIndexOptions; @@ -192,7 +194,6 @@ export type SequelizeIndexType = PgIndexType | MySQLMariaDBIndexType | SQLiteInd export interface MongoIndexOptions { background?: boolean; unique?: boolean; - name?: string; partialFilterExpression?: Document; sparse?: boolean; expireAfterSeconds?: number; @@ -213,7 +214,6 @@ export interface MongoIndexOptions { } export interface SequelizeIndexOptions { - name?: string; parser?: string | null; unique?: boolean; // Used instead of ModelOptionsIndexes fields for more complex index definitions diff --git a/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts b/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts index f56a41b7f..5108d21f8 100644 --- a/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts +++ b/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts @@ -122,7 +122,7 @@ function convertModelOptionsIndexes(copy: ConduitSchema) { for (const index of copy.modelOptions.indexes!) { // Compound indexes are maintained in modelOptions in order to be created after schema creation // Single field index => add it to specified schema field - const { fields, types, options } = index; + const { name, fields, types, options } = index; if (fields.length === 0) { throw new Error('Undefined fields for index creation'); } @@ -140,6 +140,7 @@ function convertModelOptionsIndexes(copy: ConduitSchema) { throw new Error('Invalid index type for MongoDB'); } modelField.index = { + name, type: types[0] as MongoIndexType, }; } diff --git a/modules/database/src/adapters/mongoose-adapter/index.ts b/modules/database/src/adapters/mongoose-adapter/index.ts index 0ce8b8658..570b26d66 100644 --- a/modules/database/src/adapters/mongoose-adapter/index.ts +++ b/modules/database/src/adapters/mongoose-adapter/index.ts @@ -2,6 +2,7 @@ import { ConnectOptions, IndexOptions, Mongoose } from 'mongoose'; import { MongooseSchema } from './MongooseSchema'; import { schemaConverter } from './SchemaConverter'; import ConduitGrpcSdk, { + ConduitModelField, ConduitSchema, GrpcError, Indexable, @@ -10,7 +11,11 @@ import ConduitGrpcSdk, { RawMongoQuery, } from '@conduitplatform/grpc-sdk'; import { DatabaseAdapter } from '../DatabaseAdapter'; -import { validateFieldChanges, validateFieldConstraints } from '../utils'; +import { + findAndRemoveIndex, + validateFieldChanges, + validateFieldConstraints, +} from '../utils'; import pluralize from '../../utils/pluralize'; import { mongoSchemaConverter } from '../../introspection/mongoose/utils'; import { status } from '@grpc/grpc-js'; @@ -250,56 +255,64 @@ export class MongooseAdapter extends DatabaseAdapter { const collection = this.mongoose.model(schemaName).collection; const indexSpecs = []; for (let i = 0; i < index.fields.length; i++) { + // TODO: make this simpler const spec: any = {}; spec[index.fields[i]] = index.types ? index.types[i] : 1; indexSpecs.push(spec); } await collection - .createIndex(indexSpecs, index.options as IndexOptions) + .createIndex(indexSpecs, { name: index.name, ...index.options } as IndexOptions) .catch((e: Error) => { throw new GrpcError(status.INTERNAL, e.message); }); + // Add index to modelOptions + const foundSchema = await this.models['_DeclaredSchema'].findOne({ + name: schemaName, + }); + await this.models['_DeclaredSchema'].findByIdAndUpdate(foundSchema!._id, { + modelOptions: foundSchema.modelOptions.push(index), + }); // TODO: check return 'Index created!'; } async getIndexes(schemaName: string): Promise { if (!this.models[schemaName]) throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); - const collection = this.mongoose.model(schemaName).collection; - const result = await collection.indexes(); - result.filter(index => { - index.options = {}; - for (const indexEntry of Object.entries(index)) { - if (indexEntry[0] === 'key' || indexEntry[0] === 'options') { - continue; - } - if (indexEntry[0] === 'v') { - delete index.v; - continue; - } - index.options[indexEntry[0]] = indexEntry[1]; - delete index[indexEntry[0]]; - } - index.fields = []; - index.types = []; - for (const keyEntry of Object.entries(index.key)) { - index.fields.push(keyEntry[0]); - index.types.push(keyEntry[1]); - delete index.key; + const indexes: ModelOptionsIndex[] = []; + // Find schema field indexes and convert them to modelOption indexes + for (const [field, value] of Object.entries( + this.models[schemaName].originalSchema.fields, + )) { + const index = (value as ConduitModelField).index; + if (index) { + indexes.push({ + name: index.name, + fields: [field], + types: index.type ? [index.type as MongoIndexType] : undefined, + options: index.options ?? undefined, + }); } - }); - return result as ModelOptionsIndex[]; + } + indexes.push(...(this.models[schemaName].originalSchema.modelOptions.indexes ?? [])); + return indexes; } async deleteIndexes(schemaName: string, indexNames: string[]): Promise { if (!this.models[schemaName]) throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); + const foundSchema = await this.models['_DeclaredSchema'].findOne({ + name: schemaName, + }); const collection = this.mongoose.model(schemaName).collection; + let newSchema; for (const name of indexNames) { collection.dropIndex(name).catch(() => { throw new GrpcError(status.INTERNAL, 'Unsuccessful index deletion'); }); + // Remove index from fields/compiledFields or modelOptions + newSchema = findAndRemoveIndex(foundSchema, name); } + await this.models['_DeclaredSchema'].findByIdAndUpdate(foundSchema!._id, newSchema); return 'Indexes deleted'; } diff --git a/modules/database/src/adapters/sequelize-adapter/index.ts b/modules/database/src/adapters/sequelize-adapter/index.ts index 95bf6aefd..aee3336de 100644 --- a/modules/database/src/adapters/sequelize-adapter/index.ts +++ b/modules/database/src/adapters/sequelize-adapter/index.ts @@ -1,6 +1,7 @@ import { Sequelize } from 'sequelize'; import ConduitGrpcSdk, { ConduitModel, + ConduitModelField, ConduitSchema, GrpcError, Indexable, @@ -10,11 +11,8 @@ import ConduitGrpcSdk, { PgIndexOptions, PgIndexType, RawSQLQuery, - SequelizeIndexOptions, + SequelizeIndexType, sleep, - SQLiteIndexOptions, - SQLiteIndexType, - UntypedArray, } from '@conduitplatform/grpc-sdk'; import { status } from '@grpc/grpc-js'; import { SequelizeAuto } from 'sequelize-auto'; @@ -34,7 +32,8 @@ import { } from '../../interfaces'; import { sqlSchemaConverter } from './sql-adapter/SqlSchemaConverter'; import { pgSchemaConverter } from './postgres-adapter/PgSchemaConverter'; -import { isArray, isNil, merge } from 'lodash'; +import { isNil, merge } from 'lodash'; +import { findAndRemoveIndex } from '../utils'; const sqlSchemaName = process.env.SQL_SCHEMA ?? 'public'; @@ -381,67 +380,56 @@ export abstract class SequelizeAdapter extends DatabaseAdapter if (!this.models[schemaName]) throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); index = this.checkAndConvertIndex(schemaName, index, callerModule); - const queryInterface = this.sequelize.getQueryInterface(); - await queryInterface - .addIndex( - this.models[schemaName].originalSchema.collectionName, - index.fields, - index.options, - ) - .catch(() => { - throw new GrpcError(status.INTERNAL, 'Unsuccessful index creation'); - }); - await this.models[schemaName].sync(); + const schema = this.models[schemaName].originalSchema; + const indexes = schema.modelOptions.indexes ?? []; + if (!indexes.map(i => i.name).includes(index.name)) { + indexes.push(index); + } + Object.assign(schema.modelOptions, { indexes }); + await this.createSchemaFromAdapter(schema); return 'Index created!'; } async getIndexes(schemaName: string): Promise { if (!this.models[schemaName]) throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); - const queryInterface = this.sequelize.getQueryInterface(); - const result = (await queryInterface.showIndex( - this.models[schemaName].originalSchema.collectionName, - )) as UntypedArray; - result.filter(index => { - const fieldNames = []; - for (const field of index.fields) { - fieldNames.push(field.attribute); - } - index.fields = fieldNames; - // Extract index type from index definition - let tmp = index.definition.split('USING '); - tmp = tmp[1].split(' '); - index.types = [tmp[0]]; - delete index.definition; - index.options = {}; - for (const indexEntry of Object.entries(index)) { - if ( - indexEntry[0] === 'options' || - indexEntry[0] === 'types' || - indexEntry[0] === 'fields' - ) { - continue; - } - if (indexEntry[0] === 'indkey') { - delete index.indkey; - continue; - } - index.options[indexEntry[0]] = indexEntry[1]; - delete index[indexEntry[0]]; + const indexes: ModelOptionsIndex[] = []; + // Find schema field indexes and convert them to modelOption indexes + for (const [field, value] of Object.entries( + this.models[schemaName].originalSchema.fields, + )) { + const index = (value as ConduitModelField).index; + if (index) { + indexes.push({ + name: index.name, + fields: [field], + types: index.type ? [index.type as SequelizeIndexType] : undefined, + options: index.options ?? undefined, + }); } - }); - return result; + } + indexes.push(...(this.models[schemaName].originalSchema.modelOptions.indexes ?? [])); + return indexes; } async deleteIndexes(schemaName: string, indexNames: string[]): Promise { if (!this.models[schemaName]) throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); + const foundSchema = await this.models['_DeclaredSchema'].findOne({ + name: schemaName, + }); const queryInterface = this.sequelize.getQueryInterface(); + let newSchema; for (const name of indexNames) { - queryInterface.removeIndex('cnd_' + schemaName, name).catch(() => { - throw new GrpcError(status.INTERNAL, 'Unsuccessful index deletion'); - }); + queryInterface + .removeIndex(this.models[schemaName].originalSchema.collectionName, name) + .catch(() => { + throw new GrpcError(status.INTERNAL, 'Unsuccessful index deletion'); + }); + // Remove index from fields/compiledFields or modelOptions + newSchema = findAndRemoveIndex(foundSchema, name); } + await this.models['_DeclaredSchema'].findByIdAndUpdate(foundSchema!._id, newSchema); return 'Indexes deleted'; } @@ -527,10 +515,9 @@ export abstract class SequelizeAdapter extends DatabaseAdapter throw new GrpcError(status.INVALID_ARGUMENT, `Invalid index type for ${dialect}`); } if (dialect === 'mysql' || dialect === 'mariadb') { - (index.options as MySQLMariaDBIndexOptions).type = - types[0] as MySQLMariaDBIndexType; + index.options = { ...index.options, type: types[0] } as MySQLMariaDBIndexOptions; } else if (dialect === 'postgres') { - (index.options as PgIndexOptions).using = types[0] as PgIndexType; + index.options = { ...index.options, using: types[0] } as PgIndexOptions; } delete index.types; } diff --git a/modules/database/src/adapters/utils/database-transform-utils.ts b/modules/database/src/adapters/utils/database-transform-utils.ts index bcbecb171..0ce7699b4 100644 --- a/modules/database/src/adapters/utils/database-transform-utils.ts +++ b/modules/database/src/adapters/utils/database-transform-utils.ts @@ -1,10 +1,18 @@ -import { isArray, isBoolean, isNil, isNumber, isString } from 'lodash'; +import { + isArray, + isBoolean, + isNil, + isNumber, + isString, + isObject, + forEach, + has, +} from 'lodash'; import { ConduitModelField, Indexable, - MySQLMariaDBIndexOptions, + ModelOptionsIndex, MySQLMariaDBIndexType, - PgIndexOptions, PgIndexType, SequelizeIndexOptions, SQLiteIndexType, @@ -82,8 +90,8 @@ export function convertSchemaFieldIndexes(copy: ConduitDatabaseSchema, dialect: const index = (fieldValue as ConduitModelField).index; if (!index) continue; // Convert conduit indexes to sequelize indexes - const { type, options } = index; - const newIndex: any = { fields: [fieldName] }; + const { name, type, options } = index; + const newIndex: any = { name, fields: [fieldName] }; if (type) { if (isArray(type) || !checkIfSequelizeIndexType(type, dialect)) { throw new Error(`Invalid index type for ${dialect}`); @@ -130,3 +138,21 @@ export function extractFieldProperties( return res; } + +export function findAndRemoveIndex(schema: any, indexName: string) { + const arrayIndex = schema.modelOptions.indexes.findIndex( + (i: ModelOptionsIndex) => i.name === indexName, + ); + if (arrayIndex !== -1) { + schema.modelOptions.indexes.splice(arrayIndex, 1); + return schema; + } + forEach(schema.fields, (value: ConduitModelField, key: string, fields: any) => { + if (isObject(value) && has(value, 'index') && value.index!.name === indexName) { + delete fields[key].index; + delete schema.compiledFields[key].index; + return schema; + } + }); + throw new Error('Index not found in schema'); +} diff --git a/modules/database/src/admin/index.ts b/modules/database/src/admin/index.ts index 48df6cb59..e04ae70e3 100644 --- a/modules/database/src/admin/index.ts +++ b/modules/database/src/admin/index.ts @@ -616,8 +616,9 @@ export class AdminHandlers { indexes: [ { schemaName: ConduitString.Required, + name: ConduitString.Required, fields: [ConduitString.Required], - types: [ConduitString.Required], + types: [ConduitString.Optional], options: ConduitJson.Optional, }, ], @@ -633,6 +634,7 @@ export class AdminHandlers { id: { type: TYPE.String, required: true }, }, bodyParams: { + name: ConduitString.Required, fields: [ConduitString.Required], types: [ConduitString.Required], options: ConduitJson.Optional, @@ -653,8 +655,9 @@ export class AdminHandlers { new ConduitRouteReturnDefinition('getSchemaIndexes', { indexes: [ { + name: ConduitString.Required, fields: [ConduitString.Required], - types: [ConduitString.Required], + types: [ConduitString.Optional], options: ConduitJson.Optional, // TODO: check this and the other places }, ], diff --git a/modules/database/src/admin/schema.admin.ts b/modules/database/src/admin/schema.admin.ts index 19324da4f..c3f24d72f 100644 --- a/modules/database/src/admin/schema.admin.ts +++ b/modules/database/src/admin/schema.admin.ts @@ -607,7 +607,7 @@ export class SchemaAdmin { } async createIndex(call: ParsedRouterRequest): Promise { - const { id, fields, types, options } = call.request.params; + const { id, name, fields, types, options } = call.request.params; const requestedSchema = await this.database .getSchemaModel('_DeclaredSchema') .model.findOne({ _id: id }); @@ -616,7 +616,7 @@ export class SchemaAdmin { } return await this.database.createIndex( requestedSchema.name, - { fields, types, options }, + { name, fields, types, options }, 'database', ); } diff --git a/modules/database/src/utils/utilities.ts b/modules/database/src/utils/utilities.ts index b7547a416..c46cedbbe 100644 --- a/modules/database/src/utils/utilities.ts +++ b/modules/database/src/utils/utilities.ts @@ -13,6 +13,7 @@ import { ConduitModelOptionsPermModifyType as ValidModifyPermValues, ConduitSchemaOptions, Indexable, + ModelOptionsIndex, TYPE, } from '@conduitplatform/grpc-sdk'; @@ -148,8 +149,8 @@ export function populateArray(pop: any) { function validateModelOptions(modelOptions: ConduitSchemaOptions) { if (!isPlainObject(modelOptions)) throw new Error('Model options must be an object'); Object.keys(modelOptions).forEach(key => { - if (key !== 'conduit' && key !== 'timestamps') - throw new Error("Only 'conduit' and 'timestamps' options allowed"); + if (key !== 'conduit' && key !== 'timestamps' && key !== 'indexes') + throw new Error("Only 'conduit', 'timestamps' and 'indexes' options allowed"); else if (key === 'timestamps' && !isBoolean(modelOptions.timestamps)) throw new Error("Option 'timestamps' must be of type Boolean"); else if (key === 'conduit') { @@ -163,7 +164,7 @@ function validateModelOptions(modelOptions: ConduitSchemaOptions) { conduitKey !== 'imported' ) throw new Error( - "Only 'cms' and 'permissions' fields allowed inside 'conduit' field", + "Only 'cms', 'permissions', 'authorization' and 'imported' fields allowed inside 'conduit' field", ); if (conduitKey === 'imported') { if (!isBoolean(modelOptions.conduit!.imported)) @@ -183,6 +184,32 @@ function validateModelOptions(modelOptions: ConduitSchemaOptions) { if (modelOptions.conduit!.permissions) { validatePermissions(modelOptions.conduit.permissions); } + } else if (key === 'indexes') { + if (!isArray(modelOptions.indexes)) + throw new Error("Option 'indexes' must be an array"); + modelOptions.indexes.forEach((index: ModelOptionsIndex) => { + if ( + Object.keys(index).some( + key => !['name', 'fields', 'types', 'options'].includes(key), + ) + ) { + throw new Error( + "Only 'name', 'fields', 'types' and 'options' fields allowed inside 'indexes' array", + ); + } + if (!isString(index.name)) { + throw new Error("Index field option 'name' must be of type String"); + } + if (!isArray(index.fields)) { + throw new Error("Index field option 'fields' must be of type Array"); + } + if (index.types && !isArray(index.types)) { + throw new Error("Index field option 'types' must be of type Array"); + } + if (index.options && !isObject(index.options)) { + throw new Error("Index field option 'options' must be of type Object"); + } + }); } }); } From 1fc7a28d15841a19bdd548838d9d0f4c965467a7 Mon Sep 17 00:00:00 2001 From: chris Date: Tue, 27 Jun 2023 17:28:28 +0300 Subject: [PATCH 10/17] fix(database): fixes in mongoose create index --- .../src/adapters/mongoose-adapter/index.ts | 22 +++++++++++-------- .../src/adapters/sequelize-adapter/index.ts | 2 -- modules/database/src/admin/index.ts | 2 +- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/modules/database/src/adapters/mongoose-adapter/index.ts b/modules/database/src/adapters/mongoose-adapter/index.ts index 570b26d66..571334d35 100644 --- a/modules/database/src/adapters/mongoose-adapter/index.ts +++ b/modules/database/src/adapters/mongoose-adapter/index.ts @@ -253,25 +253,29 @@ export class MongooseAdapter extends DatabaseAdapter { throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); this.checkIndex(schemaName, index, callerModule); const collection = this.mongoose.model(schemaName).collection; - const indexSpecs = []; - for (let i = 0; i < index.fields.length; i++) { - // TODO: make this simpler - const spec: any = {}; - spec[index.fields[i]] = index.types ? index.types[i] : 1; - indexSpecs.push(spec); - } + const indexSpecs: Indexable[] = index.fields.map((field, i) => ({ + [field]: index.types + ? MongoIndexType[index.types[i] as unknown as keyof typeof MongoIndexType] + : 1, + })); await collection .createIndex(indexSpecs, { name: index.name, ...index.options } as IndexOptions) .catch((e: Error) => { throw new GrpcError(status.INTERNAL, e.message); }); // Add index to modelOptions + const schema = this.models[schemaName].originalSchema; + const indexes = schema.modelOptions.indexes ?? []; + if (!indexes.map((i: ModelOptionsIndex) => i.name).includes(index.name)) { + indexes.push(index); + } + Object.assign(schema.modelOptions, { indexes }); const foundSchema = await this.models['_DeclaredSchema'].findOne({ name: schemaName, }); await this.models['_DeclaredSchema'].findByIdAndUpdate(foundSchema!._id, { - modelOptions: foundSchema.modelOptions.push(index), - }); // TODO: check + modelOptions: schema.modelOptions, + }); return 'Index created!'; } diff --git a/modules/database/src/adapters/sequelize-adapter/index.ts b/modules/database/src/adapters/sequelize-adapter/index.ts index aee3336de..c0bfaf9ec 100644 --- a/modules/database/src/adapters/sequelize-adapter/index.ts +++ b/modules/database/src/adapters/sequelize-adapter/index.ts @@ -7,9 +7,7 @@ import ConduitGrpcSdk, { Indexable, ModelOptionsIndex, MySQLMariaDBIndexOptions, - MySQLMariaDBIndexType, PgIndexOptions, - PgIndexType, RawSQLQuery, SequelizeIndexType, sleep, diff --git a/modules/database/src/admin/index.ts b/modules/database/src/admin/index.ts index e04ae70e3..51c04e7d7 100644 --- a/modules/database/src/admin/index.ts +++ b/modules/database/src/admin/index.ts @@ -658,7 +658,7 @@ export class AdminHandlers { name: ConduitString.Required, fields: [ConduitString.Required], types: [ConduitString.Optional], - options: ConduitJson.Optional, // TODO: check this and the other places + options: ConduitJson.Optional, }, ], }), From 27e0bd8dfbded5f347eb0e58d647ef832ff381d2 Mon Sep 17 00:00:00 2001 From: chris Date: Wed, 28 Jun 2023 13:17:27 +0300 Subject: [PATCH 11/17] fix(database): fix index type bugs in mongoose --- .../mongoose-adapter/SchemaConverter.ts | 9 ++-- .../src/adapters/mongoose-adapter/index.ts | 41 ++++++++++++++----- modules/database/src/admin/schema.admin.ts | 2 +- 3 files changed, 36 insertions(+), 16 deletions(-) diff --git a/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts b/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts index 5108d21f8..60b2c962c 100644 --- a/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts +++ b/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts @@ -130,16 +130,15 @@ function convertModelOptionsIndexes(copy: ConduitSchema) { throw new Error(`Invalid fields for index creation`); } if (fields.length !== 1) continue; - const modelField = copy.fields[index.fields[0]] as ConduitModelField; if (types) { if ( !isArray(types) || - !(types[0] in MongoIndexType) || + !Object.values(MongoIndexType).includes(types[0]) || fields.length !== types.length ) { throw new Error('Invalid index type for MongoDB'); } - modelField.index = { + (copy.fields[index.fields[0]] as ConduitModelField).index = { name, type: types[0] as MongoIndexType, }; @@ -149,7 +148,9 @@ function convertModelOptionsIndexes(copy: ConduitSchema) { throw new Error('Incorrect index options for MongoDB'); } for (const [option, optionValue] of Object.entries(options)) { - modelField.index![option as keyof SchemaFieldIndex] = optionValue; + (copy.fields[index.fields[0]] as ConduitModelField).index![ + option as keyof SchemaFieldIndex + ] = optionValue; } } copy.modelOptions.indexes!.splice(copy.modelOptions.indexes!.indexOf(index), 1); diff --git a/modules/database/src/adapters/mongoose-adapter/index.ts b/modules/database/src/adapters/mongoose-adapter/index.ts index 571334d35..ba9e4deea 100644 --- a/modules/database/src/adapters/mongoose-adapter/index.ts +++ b/modules/database/src/adapters/mongoose-adapter/index.ts @@ -251,23 +251,24 @@ export class MongooseAdapter extends DatabaseAdapter { ): Promise { if (!this.models[schemaName]) throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); - this.checkIndex(schemaName, index, callerModule); const collection = this.mongoose.model(schemaName).collection; - const indexSpecs: Indexable[] = index.fields.map((field, i) => ({ - [field]: index.types - ? MongoIndexType[index.types[i] as unknown as keyof typeof MongoIndexType] - : 1, + const convIndex = this.checkAndConvertIndex(schemaName, index, callerModule); + const indexSpecs: Indexable[] = convIndex.fields.map((field, i) => ({ + [field]: convIndex.types?.[i] ?? 1, })); await collection - .createIndex(indexSpecs, { name: index.name, ...index.options } as IndexOptions) + .createIndex(indexSpecs, { + name: convIndex.name, + ...convIndex.options, + } as IndexOptions) .catch((e: Error) => { throw new GrpcError(status.INTERNAL, e.message); }); // Add index to modelOptions const schema = this.models[schemaName].originalSchema; const indexes = schema.modelOptions.indexes ?? []; - if (!indexes.map((i: ModelOptionsIndex) => i.name).includes(index.name)) { - indexes.push(index); + if (!indexes.map((i: ModelOptionsIndex) => i.name).includes(convIndex.name)) { + indexes.push(convIndex); } Object.assign(schema.modelOptions, { indexes }); const foundSchema = await this.models['_DeclaredSchema'].findOne({ @@ -441,7 +442,11 @@ export class MongooseAdapter extends DatabaseAdapter { return this.models[schema.name]; } - private checkIndex(schemaName: string, index: ModelOptionsIndex, callerModule: string) { + private checkAndConvertIndex( + schemaName: string, + index: ModelOptionsIndex, + callerModule: string, + ) { const { fields, types, options } = index; if ( fields.some( @@ -453,7 +458,7 @@ export class MongooseAdapter extends DatabaseAdapter { ) { throw new Error(`Invalid fields for index creation`); } - if (!options && !types) return; + if (!options && !types) return index; if (options) { if (!checkIfMongoOptions(options)) { throw new GrpcError(status.INTERNAL, 'Invalid index options for mongoDB'); @@ -471,10 +476,24 @@ export class MongooseAdapter extends DatabaseAdapter { if (types) { if ( types.length !== index.fields.length || - types.some(type => !(type in MongoIndexType)) + types.some( + type => + !(type in MongoIndexType) && !Object.values(MongoIndexType).includes(type), + ) ) { throw new GrpcError(status.INTERNAL, 'Invalid index types'); } + // Convert index types (if called by endpoint index.types contains the keys of MongoIndexType enum) + index.types = types.map(type => { + if ( + Object.keys(MongoIndexType).includes( + type as unknown as keyof typeof MongoIndexType, + ) + ) + return MongoIndexType[type as unknown as keyof typeof MongoIndexType]; + return type; + }) as MongoIndexType[]; } + return index; } } diff --git a/modules/database/src/admin/schema.admin.ts b/modules/database/src/admin/schema.admin.ts index c3f24d72f..66cbd4a07 100644 --- a/modules/database/src/admin/schema.admin.ts +++ b/modules/database/src/admin/schema.admin.ts @@ -584,7 +584,7 @@ export class SchemaAdmin { async importIndexes(call: ParsedRouterRequest): Promise { for (const index of call.request.params.indexes) { await this.database.createIndex(index.schemaName, index, 'database').catch(e => { - throw new GrpcError(status.INTERNAL, `${index.options.name}: ${e.message}`); + throw new GrpcError(status.INTERNAL, `${index.name}: ${e.message}`); }); } return 'Indexes imported successfully'; From 45229a647d64c79667427057e4b3ca51ef52b06c Mon Sep 17 00:00:00 2001 From: chris Date: Fri, 30 Jun 2023 10:58:26 +0300 Subject: [PATCH 12/17] fix(database): fix mysql index type bug --- .../authentication/src/models/User.schema.ts | 20 +++++++++++++++++++ .../src/adapters/sequelize-adapter/index.ts | 7 +++++-- .../utils/database-transform-utils.ts | 16 ++++++++++----- 3 files changed, 36 insertions(+), 7 deletions(-) diff --git a/modules/authentication/src/models/User.schema.ts b/modules/authentication/src/models/User.schema.ts index eaa23152d..720408a27 100644 --- a/modules/authentication/src/models/User.schema.ts +++ b/modules/authentication/src/models/User.schema.ts @@ -2,6 +2,8 @@ import { ConduitModel, DatabaseProvider, Indexable, + ModelOptionsIndex, + SQLiteIndexType, TYPE, } from '@conduitplatform/grpc-sdk'; import { ConduitActiveSchema } from '@conduitplatform/module-tools'; @@ -34,6 +36,9 @@ const schema: ConduitModel = { email: { type: TYPE.String, required: false, + index: { + name: 'emailIndex', + }, }, hashedPassword: { type: TYPE.String, @@ -79,6 +84,20 @@ const schema: ConduitModel = { createdAt: TYPE.Date, updatedAt: TYPE.Date, }; +const indexes: ModelOptionsIndex[] = [ + { + name: 'compoundIndex', + fields: ['phoneNumber', 'hashedPassword'], + }, + { + name: 'createdAtIndex', + fields: ['createdAt'], + types: [SQLiteIndexType.BTREE], + options: { + unique: true, + }, + }, +]; const modelOptions = { timestamps: true, conduit: { @@ -89,6 +108,7 @@ const modelOptions = { canDelete: false, }, }, + indexes, } as const; const collectionName = undefined; diff --git a/modules/database/src/adapters/sequelize-adapter/index.ts b/modules/database/src/adapters/sequelize-adapter/index.ts index c0bfaf9ec..e9fb65687 100644 --- a/modules/database/src/adapters/sequelize-adapter/index.ts +++ b/modules/database/src/adapters/sequelize-adapter/index.ts @@ -512,9 +512,12 @@ export abstract class SequelizeAdapter extends DatabaseAdapter if (types.length !== 1 || !checkIfSequelizeIndexType(types[0], dialect)) { throw new GrpcError(status.INVALID_ARGUMENT, `Invalid index type for ${dialect}`); } - if (dialect === 'mysql' || dialect === 'mariadb') { + if ( + (dialect === 'mysql' || dialect === 'mariadb') && + ['UNIQUE', 'FULLTEXT', 'SPATIAL'].includes(types[0] as string) + ) { index.options = { ...index.options, type: types[0] } as MySQLMariaDBIndexOptions; - } else if (dialect === 'postgres') { + } else { index.options = { ...index.options, using: types[0] } as PgIndexOptions; } delete index.types; diff --git a/modules/database/src/adapters/utils/database-transform-utils.ts b/modules/database/src/adapters/utils/database-transform-utils.ts index 0ce7699b4..0c9d93a74 100644 --- a/modules/database/src/adapters/utils/database-transform-utils.ts +++ b/modules/database/src/adapters/utils/database-transform-utils.ts @@ -73,9 +73,12 @@ export function convertModelOptionsIndexes(copy: ConduitDatabaseSchema, dialect: if (types.length !== 1 || !checkIfSequelizeIndexType(types[0], dialect)) { throw new Error(`Invalid index type for ${dialect}`); } - if (dialect === 'mysql' || dialect === 'mariadb') { + if ( + (dialect === 'mysql' || dialect === 'mariadb') && + ['UNIQUE', 'FULLTEXT', 'SPATIAL'].includes(types[0] as string) + ) { index.type = types[0] as MySQLMariaDBIndexType; - } else if (dialect === 'postgres') { + } else { index.using = types[0] as PgIndexType | SQLiteIndexType; } delete index.types; @@ -96,10 +99,13 @@ export function convertSchemaFieldIndexes(copy: ConduitDatabaseSchema, dialect: if (isArray(type) || !checkIfSequelizeIndexType(type, dialect)) { throw new Error(`Invalid index type for ${dialect}`); } - if (dialect === 'mysql' || dialect === 'mariadb') { + if ( + (dialect === 'mysql' || dialect === 'mariadb') && + ['UNIQUE', 'FULLTEXT', 'SPATIAL'].includes(type as string) + ) { newIndex.type = type as MySQLMariaDBIndexType; - } else if (dialect === 'postgres') { - newIndex.using = type as PgIndexType; + } else { + newIndex.using = type as PgIndexType | SQLiteIndexType; } } if (options && !checkIfSequelizeIndexOptions(options, dialect)) { From 9b859bac6f77242d4cd14895d00c820f5dc4e1c9 Mon Sep 17 00:00:00 2001 From: chris Date: Fri, 30 Jun 2023 11:01:37 +0300 Subject: [PATCH 13/17] chore: remove testing code --- .../authentication/src/models/User.schema.ts | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/modules/authentication/src/models/User.schema.ts b/modules/authentication/src/models/User.schema.ts index 720408a27..eaa23152d 100644 --- a/modules/authentication/src/models/User.schema.ts +++ b/modules/authentication/src/models/User.schema.ts @@ -2,8 +2,6 @@ import { ConduitModel, DatabaseProvider, Indexable, - ModelOptionsIndex, - SQLiteIndexType, TYPE, } from '@conduitplatform/grpc-sdk'; import { ConduitActiveSchema } from '@conduitplatform/module-tools'; @@ -36,9 +34,6 @@ const schema: ConduitModel = { email: { type: TYPE.String, required: false, - index: { - name: 'emailIndex', - }, }, hashedPassword: { type: TYPE.String, @@ -84,20 +79,6 @@ const schema: ConduitModel = { createdAt: TYPE.Date, updatedAt: TYPE.Date, }; -const indexes: ModelOptionsIndex[] = [ - { - name: 'compoundIndex', - fields: ['phoneNumber', 'hashedPassword'], - }, - { - name: 'createdAtIndex', - fields: ['createdAt'], - types: [SQLiteIndexType.BTREE], - options: { - unique: true, - }, - }, -]; const modelOptions = { timestamps: true, conduit: { @@ -108,7 +89,6 @@ const modelOptions = { canDelete: false, }, }, - indexes, } as const; const collectionName = undefined; From 911111ae73b96fda57981c34dda9ad592e2fb34a Mon Sep 17 00:00:00 2001 From: chris Date: Fri, 30 Jun 2023 11:13:23 +0300 Subject: [PATCH 14/17] refactor(database,authorization)!: getDatabaseType returning dialect --- .../authorization/src/controllers/permissions.controller.ts | 2 +- modules/database/src/adapters/sequelize-adapter/index.ts | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/modules/authorization/src/controllers/permissions.controller.ts b/modules/authorization/src/controllers/permissions.controller.ts index dd5368d55..258325f2a 100644 --- a/modules/authorization/src/controllers/permissions.controller.ts +++ b/modules/authorization/src/controllers/permissions.controller.ts @@ -215,7 +215,7 @@ export class PermissionsController { }, ], sqlQuery: - dbType === 'PostgreSQL' + dbType === 'postgres' ? getPostgresAccessListQuery( objectTypeCollection, computedTuple, diff --git a/modules/database/src/adapters/sequelize-adapter/index.ts b/modules/database/src/adapters/sequelize-adapter/index.ts index e9fb65687..bf8a65e6d 100644 --- a/modules/database/src/adapters/sequelize-adapter/index.ts +++ b/modules/database/src/adapters/sequelize-adapter/index.ts @@ -363,11 +363,7 @@ export abstract class SequelizeAdapter extends DatabaseAdapter } getDatabaseType(): string { - const type = this.sequelize.getDialect(); - if (type === 'postgres') { - return 'PostgreSQL'; // TODO: clean up - } - return type; + return this.sequelize.getDialect(); } async createIndex( From d78f3a6440e3d9e573fa679383ba92a4574fd3ed Mon Sep 17 00:00:00 2001 From: chris Date: Thu, 1 Aug 2024 11:49:19 +0300 Subject: [PATCH 15/17] refactor(database, grpc-sdk): cleanup code refactor(authorization): add default index names fix(database): import/export checks & ignore indexes of other db type --- libraries/grpc-sdk/src/interfaces/Model.ts | 12 ++ .../authentication/src/models/User.schema.ts | 2 + .../src/models/ActorIndex.schema.ts | 2 + .../src/models/ObjectIndex.schema.ts | 2 + .../src/models/Permission.schema.ts | 2 + .../src/models/Relationship.schema.ts | 2 + .../database/src/adapters/DatabaseAdapter.ts | 4 +- .../mongoose-adapter/SchemaConverter.ts | 53 +++++-- .../src/adapters/mongoose-adapter/index.ts | 140 +++++++++--------- .../src/adapters/sequelize-adapter/index.ts | 82 +++++----- .../postgres-adapter/PgSchemaConverter.ts | 2 +- .../sql-adapter/SqlSchemaConverter.ts | 2 +- .../utils/database-transform-utils.ts | 28 +++- .../src/adapters/utils/indexValidations.ts | 32 ++++ modules/database/src/admin/index.ts | 38 +++-- modules/database/src/admin/schema.admin.ts | 42 ++++-- 16 files changed, 275 insertions(+), 170 deletions(-) create mode 100644 modules/database/src/adapters/utils/indexValidations.ts diff --git a/libraries/grpc-sdk/src/interfaces/Model.ts b/libraries/grpc-sdk/src/interfaces/Model.ts index 62476f33f..927698ba6 100644 --- a/libraries/grpc-sdk/src/interfaces/Model.ts +++ b/libraries/grpc-sdk/src/interfaces/Model.ts @@ -168,6 +168,18 @@ export enum MongoIndexType { Text = 'text', } +export const ReverseMongoIndexTypeMap: { + [key: string]: string; +} = { + '1': 'Ascending', + '-1': 'Descending', + '2d': 'GeoSpatial2d', + '2dsphere': 'GeoSpatial2dSphere', + geoHaystack: 'GeoHaystack', + hashed: 'Hashed', + text: 'Text', +}; + export enum PgIndexType { BTREE = 'BTREE', HASH = 'HASH', diff --git a/modules/authentication/src/models/User.schema.ts b/modules/authentication/src/models/User.schema.ts index a7a5bcc7e..e1eb6e29a 100644 --- a/modules/authentication/src/models/User.schema.ts +++ b/modules/authentication/src/models/User.schema.ts @@ -2,6 +2,8 @@ import { ConduitModel, DatabaseProvider, Indexable, + ModelOptionsIndex, + MongoIndexType, TYPE, } from '@conduitplatform/grpc-sdk'; import { ConduitActiveSchema } from '@conduitplatform/module-tools'; diff --git a/modules/authorization/src/models/ActorIndex.schema.ts b/modules/authorization/src/models/ActorIndex.schema.ts index 396ca3f24..7bbd6d669 100644 --- a/modules/authorization/src/models/ActorIndex.schema.ts +++ b/modules/authorization/src/models/ActorIndex.schema.ts @@ -22,6 +22,7 @@ const schema: ConduitModel = { type: TYPE.String, required: true, index: { + name: 'subject_1', type: MongoIndexType.Ascending, }, }, @@ -40,6 +41,7 @@ const schema: ConduitModel = { type: TYPE.String, required: true, index: { + name: 'entity_1', type: MongoIndexType.Ascending, }, }, diff --git a/modules/authorization/src/models/ObjectIndex.schema.ts b/modules/authorization/src/models/ObjectIndex.schema.ts index f3de35ad4..087235007 100644 --- a/modules/authorization/src/models/ObjectIndex.schema.ts +++ b/modules/authorization/src/models/ObjectIndex.schema.ts @@ -22,6 +22,7 @@ const schema: ConduitModel = { type: TYPE.String, required: true, index: { + name: 'subject_1', type: MongoIndexType.Ascending, }, }, @@ -46,6 +47,7 @@ const schema: ConduitModel = { type: TYPE.String, required: true, index: { + name: 'entity_1', type: MongoIndexType.Ascending, }, }, diff --git a/modules/authorization/src/models/Permission.schema.ts b/modules/authorization/src/models/Permission.schema.ts index c7cdef5f9..90622ff0f 100644 --- a/modules/authorization/src/models/Permission.schema.ts +++ b/modules/authorization/src/models/Permission.schema.ts @@ -18,6 +18,7 @@ const schema: ConduitModel = { type: TYPE.String, required: true, index: { + name: 'resource_1', type: MongoIndexType.Ascending, }, }, @@ -37,6 +38,7 @@ const schema: ConduitModel = { type: TYPE.String, required: true, index: { + name: 'subject_1', type: MongoIndexType.Ascending, }, }, diff --git a/modules/authorization/src/models/Relationship.schema.ts b/modules/authorization/src/models/Relationship.schema.ts index d378d4d48..9ee6c5673 100644 --- a/modules/authorization/src/models/Relationship.schema.ts +++ b/modules/authorization/src/models/Relationship.schema.ts @@ -17,6 +17,7 @@ const schema: ConduitModel = { type: TYPE.String, required: true, index: { + name: 'resource_1', type: MongoIndexType.Ascending, }, }, @@ -36,6 +37,7 @@ const schema: ConduitModel = { type: TYPE.String, required: true, index: { + name: 'subject_1', type: MongoIndexType.Ascending, }, }, diff --git a/modules/database/src/adapters/DatabaseAdapter.ts b/modules/database/src/adapters/DatabaseAdapter.ts index 6a1c9655e..fd1cbcc31 100644 --- a/modules/database/src/adapters/DatabaseAdapter.ts +++ b/modules/database/src/adapters/DatabaseAdapter.ts @@ -169,9 +169,9 @@ export abstract class DatabaseAdapter { abstract getDatabaseType(): string; - abstract createIndex( + abstract createIndexes( schemaName: string, - index: ModelOptionsIndex, + indexes: ModelOptionsIndex[], callerModule: string, ): Promise; diff --git a/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts b/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts index a2d7febca..e358b83f8 100644 --- a/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts +++ b/modules/database/src/adapters/mongoose-adapter/SchemaConverter.ts @@ -1,7 +1,9 @@ import { Schema } from 'mongoose'; import { + ConduitGrpcSdk, ConduitModelField, ConduitSchema, + ModelOptionsIndex, MongoIndexType, SchemaFieldIndex, } from '@conduitplatform/grpc-sdk'; @@ -15,14 +17,14 @@ import * as deepdash from 'deepdash-es/standalone'; * @param jsonSchema */ export function schemaConverter(jsonSchema: ConduitSchema) { - let copy = cloneDeep(jsonSchema); + const copy = cloneDeep(jsonSchema); if (copy.fields.hasOwnProperty('_id')) { delete copy.fields['_id']; } - copy = convertSchemaFieldIndexes(copy); + convertSchemaFieldIndexes(copy); deepdash.eachDeep(copy.fields, convert); if (copy.modelOptions.indexes) { - copy = convertModelOptionsIndexes(copy); + copy.modelOptions.indexes = convertModelOptionsIndexes(copy); } iterDeep(copy.fields); return copy; @@ -97,16 +99,24 @@ function convert(value: any, key: any, parentValue: any) { } function convertSchemaFieldIndexes(copy: ConduitSchema) { - for (const field of Object.entries(copy.fields)) { - const index = (field[1] as ConduitModelField).index; + for (const [, fieldObj] of Object.entries(copy.fields)) { + const index = (fieldObj as ConduitModelField).index; if (!index) continue; const { type, options } = index; if (type && !(type in MongoIndexType)) { - throw new Error('Incorrect index type for MongoDB'); + ConduitGrpcSdk.Logger.warn( + `Invalid index type for MongoDB found in: ${copy.name}. Index ignored`, + ); + delete (fieldObj as ConduitModelField).index; + continue; } if (options) { if (!checkIfMongoOptions(options)) { - throw new Error('Incorrect index options for MongoDB'); + ConduitGrpcSdk.Logger.warn( + `Invalid index options for MongoDB found in: ${copy.name}. Index ignored`, + ); + delete (fieldObj as ConduitModelField).index; + continue; } for (const [option, optionValue] of Object.entries(options)) { index[option as keyof SchemaFieldIndex] = optionValue; @@ -114,13 +124,12 @@ function convertSchemaFieldIndexes(copy: ConduitSchema) { delete index.options; } } - return copy; } function convertModelOptionsIndexes(copy: ConduitSchema) { + const convertedIndexes: ModelOptionsIndex[] = []; + for (const index of copy.modelOptions.indexes!) { - // Compound indexes are maintained in modelOptions in order to be created after schema creation - // Single field index => add it to specified schema field const { name, fields, types, options } = index; if (fields.length === 0) { throw new Error('Undefined fields for index creation'); @@ -128,23 +137,35 @@ function convertModelOptionsIndexes(copy: ConduitSchema) { if (fields.some(field => !Object.keys(copy.fields).includes(field))) { throw new Error(`Invalid fields for index creation`); } - if (fields.length !== 1) continue; + // Compound indexes are maintained in modelOptions in order to be created after schema creation + // Single field index are added to specified schema field + if (fields.length !== 1) { + convertedIndexes.push(index); + continue; + } if (types) { + const indexField = index.fields[0]; if ( !isArray(types) || !Object.values(MongoIndexType).includes(types[0]) || fields.length !== types.length ) { - throw new Error('Invalid index type for MongoDB'); + ConduitGrpcSdk.Logger.warn( + `Invalid index type for MongoDB found in: ${copy.name}. Index ignored`, + ); + continue; } - (copy.fields[index.fields[0]] as ConduitModelField).index = { + (copy.fields[indexField] as ConduitModelField).index = { name, type: types[0] as MongoIndexType, }; } if (options) { if (!checkIfMongoOptions(options)) { - throw new Error('Incorrect index options for MongoDB'); + ConduitGrpcSdk.Logger.warn( + `Invalid index options for MongoDB found in: ${copy.name}. Index ignored`, + ); + continue; } for (const [option, optionValue] of Object.entries(options)) { (copy.fields[index.fields[0]] as ConduitModelField).index![ @@ -152,7 +173,7 @@ function convertModelOptionsIndexes(copy: ConduitSchema) { ] = optionValue; } } - copy.modelOptions.indexes!.splice(copy.modelOptions.indexes!.indexOf(index), 1); + convertedIndexes.push(index); } - return copy; + return convertedIndexes; } diff --git a/modules/database/src/adapters/mongoose-adapter/index.ts b/modules/database/src/adapters/mongoose-adapter/index.ts index ed486c762..85568eca1 100644 --- a/modules/database/src/adapters/mongoose-adapter/index.ts +++ b/modules/database/src/adapters/mongoose-adapter/index.ts @@ -10,6 +10,7 @@ import { ModelOptionsIndex, MongoIndexType, RawMongoQuery, + ReverseMongoIndexTypeMap, } from '@conduitplatform/grpc-sdk'; import { DatabaseAdapter } from '../DatabaseAdapter.js'; import { @@ -25,10 +26,11 @@ import { ConduitDatabaseSchema, introspectedSchemaCmsOptionsDefaults, } from '../../interfaces/index.js'; -import { isNil, isEqual } from 'lodash-es'; +import { isEqual, isNil } from 'lodash-es'; // @ts-ignore import * as parseSchema from 'mongodb-schema'; +import { validateIndexFields } from '../utils/indexValidations.js'; export class MongooseAdapter extends DatabaseAdapter { connected: boolean = false; @@ -267,69 +269,87 @@ export class MongooseAdapter extends DatabaseAdapter { return 'MongoDB'; } - async createIndex( + async createIndexes( schemaName: string, - index: ModelOptionsIndex, + indexes: ModelOptionsIndex[], callerModule: string, ): Promise { if (!this.models[schemaName]) throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); const collection = this.mongoose.model(schemaName).collection; - const convIndex = this.checkAndConvertIndex(schemaName, index, callerModule); - const indexSpecs: Indexable[] = convIndex.fields.map( - (field: any, i: string | number) => ({ - [field]: convIndex.types?.[i as any] ?? 1, - }), + for (const index of indexes) { + const convIndex = this.checkAndConvertIndex(schemaName, index, callerModule); + const indexSpecs: Indexable[] = convIndex.fields.map( + (field: string, i: number) => ({ + [field]: convIndex.types?.[i] ?? 1, + }), + ); + await collection + .createIndex(indexSpecs, { + name: convIndex.name, + ...convIndex.options, + } as IndexOptions) + .catch((e: Error) => { + throw new GrpcError(status.INTERNAL, e.message); + }); + } + // Add indexes to modelOptions + const modelOptionsIndexes = + this.models[schemaName].originalSchema.modelOptions.indexes ?? []; + const indexMap = new Map( + modelOptionsIndexes.map((i: ModelOptionsIndex) => [i.name, i]), ); - await collection - .createIndex(indexSpecs, { - name: convIndex.name, - ...convIndex.options, - } as IndexOptions) - .catch((e: Error) => { - throw new GrpcError(status.INTERNAL, e.message); - }); - // Add index to modelOptions - const schema = this.models[schemaName].originalSchema; - const indexes = schema.modelOptions.indexes ?? []; - if (!indexes.map((i: ModelOptionsIndex) => i.name).includes(convIndex.name)) { - indexes.push(convIndex); + for (const i of indexes) { + if (!indexMap.has(i.name)) { + indexMap.set(i.name, i); + } } - Object.assign(schema.modelOptions, { indexes }); const foundSchema = await this.models['_DeclaredSchema'].findOne({ name: schemaName, }); await this.models['_DeclaredSchema'].findByIdAndUpdate(foundSchema!._id, { - modelOptions: schema.modelOptions, + 'modelOptions.indexes': [...indexMap.values()], }); - return 'Index created!'; + return 'Indexes created!'; } async getIndexes(schemaName: string): Promise { if (!this.models[schemaName]) throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); - const indexes: ModelOptionsIndex[] = []; // Find schema field indexes and convert them to modelOption indexes - for (const [field, value] of Object.entries( - this.models[schemaName].originalSchema.fields, - )) { + const indexes = []; + const ogSchema = this.models[schemaName].originalSchema; + const fields = ogSchema.fields; + + for (const [field, value] of Object.entries(fields)) { const index = (value as ConduitModelField).index; if (index) { indexes.push({ name: index.name, fields: [field], - types: index.type ? [index.type as MongoIndexType] : undefined, + types: index.type + ? [ReverseMongoIndexTypeMap[index.type.toString()]] + : undefined, options: index.options ?? undefined, }); } } - indexes.push(...(this.models[schemaName].originalSchema.modelOptions.indexes ?? [])); + const modelOptionsIndexes = (ogSchema.modelOptions.indexes ?? []).map( + (i: ModelOptionsIndex) => ({ + ...i, + types: i.types?.map( + (t: string | number) => ReverseMongoIndexTypeMap[t.toString()], + ), + }), + ); + indexes.push(...modelOptionsIndexes); return indexes; } async deleteIndexes(schemaName: string, indexNames: string[]): Promise { if (!this.models[schemaName]) - throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); + if (!this.models[schemaName]) + throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); const foundSchema = await this.models['_DeclaredSchema'].findOne({ name: schemaName, }); @@ -464,9 +484,7 @@ export class MongooseAdapter extends DatabaseAdapter { await this.saveSchemaToDatabase(schema); } if (indexes && !isInstanceSync) { - for (const i of indexes) { - await this.createIndex(schema.name, i, schema.ownerModule); - } + await this.createIndexes(schema.name, indexes, schema.ownerModule); } return this.models[schema.name]; } @@ -476,49 +494,25 @@ export class MongooseAdapter extends DatabaseAdapter { index: ModelOptionsIndex, callerModule: string, ) { - const { fields, types, options } = index; + const { types, options } = index; + validateIndexFields(this.models[schemaName].originalSchema, index, callerModule); + if (options && !checkIfMongoOptions(options)) { + throw new GrpcError(status.INTERNAL, 'Invalid index options for mongoDB'); + } + if (!types) return index; if ( - fields.some( - (field: string) => - !Object.keys(this.models[schemaName].originalSchema.compiledFields).includes( - field, - ), + types.length !== index.fields.length || + types.some( + type => + !(type in MongoIndexType) && !Object.values(MongoIndexType).includes(type), ) ) { - throw new Error(`Invalid fields for index creation`); - } - if (!options && !types) return index; - if (options) { - if (!checkIfMongoOptions(options)) { - throw new GrpcError(status.INTERNAL, 'Invalid index options for mongoDB'); - } - if ( - Object.keys(options).includes('unique') && - this.models[schemaName].originalSchema.ownerModule !== callerModule - ) { - throw new GrpcError( - status.PERMISSION_DENIED, - 'Not authorized to create unique index', - ); - } - } - if (types) { - if ( - types.length !== index.fields.length || - types.some( - (type: string | MongoIndexType.Ascending | MongoIndexType.Descending) => - !(type in MongoIndexType) && !Object.values(MongoIndexType).includes(type), - ) - ) { - throw new GrpcError(status.INTERNAL, 'Invalid index types'); - } - // Convert index types (if called by endpoint index.types contains the keys of MongoIndexType enum) - index.types = types.map((type: unknown) => { - if (Object.keys(MongoIndexType).includes(type as keyof typeof MongoIndexType)) - return MongoIndexType[type as unknown as keyof typeof MongoIndexType]; - return type; - }) as MongoIndexType[]; + throw new GrpcError(status.INTERNAL, 'Invalid index types'); } + // Convert index types (if called by endpoint index.types contains the keys of MongoIndexType enum) + index.types = types.map( + type => MongoIndexType[type as unknown as keyof typeof MongoIndexType] || type, + ) as MongoIndexType[]; return index; } } diff --git a/modules/database/src/adapters/sequelize-adapter/index.ts b/modules/database/src/adapters/sequelize-adapter/index.ts index fb90ece81..814d518a1 100644 --- a/modules/database/src/adapters/sequelize-adapter/index.ts +++ b/modules/database/src/adapters/sequelize-adapter/index.ts @@ -33,6 +33,7 @@ import { sqlSchemaConverter } from './sql-adapter/SqlSchemaConverter.js'; import { pgSchemaConverter } from './postgres-adapter/PgSchemaConverter.js'; import { isEqual, isNil } from 'lodash-es'; import { findAndRemoveIndex } from '../utils/index.js'; +import { validateIndexFields } from '../utils/indexValidations.js'; const sqlSchemaName = process.env.SQL_SCHEMA ?? 'public'; @@ -346,22 +347,29 @@ export abstract class SequelizeAdapter extends DatabaseAdapter return this.sequelize.getDialect(); } - async createIndex( + async createIndexes( schemaName: string, - index: ModelOptionsIndex, + indexes: ModelOptionsIndex[], callerModule: string, ): Promise { if (!this.models[schemaName]) throw new GrpcError(status.NOT_FOUND, 'Requested schema not found'); - index = this.checkAndConvertIndex(schemaName, index, callerModule); + const convertedIndexes = indexes.map(i => + this.checkAndConvertIndex(schemaName, i, callerModule), + ); const schema = this.models[schemaName].originalSchema; - const indexes = schema.modelOptions.indexes ?? []; - if (!indexes.map(i => i.name).includes(index.name)) { - indexes.push(index); + const modelOptionsIndexes = schema.modelOptions.indexes ?? []; + const indexMap = new Map( + modelOptionsIndexes.map((i: ModelOptionsIndex) => [i.name, i]), + ); + for (const i of convertedIndexes) { + if (!indexMap.has(i.name)) { + indexMap.set(i.name, i); + } } - Object.assign(schema.modelOptions, { indexes }); + Object.assign(schema.modelOptions, { indexes: [...indexMap.values()] }); await this.createSchemaFromAdapter(schema); - return 'Index created!'; + return 'Indexes created!'; } async getIndexes(schemaName: string): Promise { @@ -455,49 +463,27 @@ export abstract class SequelizeAdapter extends DatabaseAdapter callerModule: string, ) { const dialect = this.sequelize.getDialect(); - const { fields, types, options } = index; - if ( - fields.some( - (field: string) => - !Object.keys(this.models[schemaName].originalSchema.compiledFields).includes( - field, - ), - ) - ) { - throw new Error(`Invalid fields for index creation`); + const { types, options } = index; + validateIndexFields(this.models[schemaName].originalSchema, index, callerModule); + if (options && !checkIfSequelizeIndexOptions(options, dialect)) { + throw new GrpcError( + status.INVALID_ARGUMENT, + `Invalid index options for ${dialect}`, + ); } - if (!types && !options) return index; - if (options) { - if (!checkIfSequelizeIndexOptions(options, dialect)) { - throw new GrpcError( - status.INVALID_ARGUMENT, - `Invalid index options for ${dialect}`, - ); - } - if ( - Object.keys(options).includes('unique') && - this.models[schemaName].originalSchema.ownerModule !== callerModule - ) { - throw new GrpcError( - status.PERMISSION_DENIED, - 'Not authorized to create unique index', - ); - } + if (!types) return index; + if (types.length !== 1 || !checkIfSequelizeIndexType(types[0], dialect)) { + throw new GrpcError(status.INVALID_ARGUMENT, `Invalid index type for ${dialect}`); } - if (types) { - if (types.length !== 1 || !checkIfSequelizeIndexType(types[0], dialect)) { - throw new GrpcError(status.INVALID_ARGUMENT, `Invalid index type for ${dialect}`); - } - if ( - (dialect === 'mysql' || dialect === 'mariadb') && - ['UNIQUE', 'FULLTEXT', 'SPATIAL'].includes(types[0] as string) - ) { - index.options = { ...index.options, type: types[0] } as MySQLMariaDBIndexOptions; - } else { - index.options = { ...index.options, using: types[0] } as PgIndexOptions; - } - delete index.types; + if ( + (dialect === 'mysql' || dialect === 'mariadb') && + ['UNIQUE', 'FULLTEXT', 'SPATIAL'].includes(types[0] as string) + ) { + index.options = { ...index.options, type: types[0] } as MySQLMariaDBIndexOptions; + } else { + index.options = { ...index.options, using: types[0] } as PgIndexOptions; } + delete index.types; return index; } } diff --git a/modules/database/src/adapters/sequelize-adapter/postgres-adapter/PgSchemaConverter.ts b/modules/database/src/adapters/sequelize-adapter/postgres-adapter/PgSchemaConverter.ts index cc335ab16..cfdf4d98b 100644 --- a/modules/database/src/adapters/sequelize-adapter/postgres-adapter/PgSchemaConverter.ts +++ b/modules/database/src/adapters/sequelize-adapter/postgres-adapter/PgSchemaConverter.ts @@ -41,7 +41,7 @@ export function pgSchemaConverter( delete copy.fields['_id']; } if (copy.modelOptions.indexes) { - copy = convertModelOptionsIndexes(copy, dialect); + copy.modelOptions.indexes = convertModelOptionsIndexes(copy, dialect); } const objectPaths: any = {}; convertObjectToDotNotation(jsonSchema.fields, copy.fields, objectPaths); diff --git a/modules/database/src/adapters/sequelize-adapter/sql-adapter/SqlSchemaConverter.ts b/modules/database/src/adapters/sequelize-adapter/sql-adapter/SqlSchemaConverter.ts index 7174e1587..70167dce9 100644 --- a/modules/database/src/adapters/sequelize-adapter/sql-adapter/SqlSchemaConverter.ts +++ b/modules/database/src/adapters/sequelize-adapter/sql-adapter/SqlSchemaConverter.ts @@ -39,7 +39,7 @@ export function sqlSchemaConverter( delete copy.fields['_id']; } if (copy.modelOptions.indexes) { - copy = convertModelOptionsIndexes(copy, dialect); + copy.modelOptions.indexes = convertModelOptionsIndexes(copy, dialect); } const objectPaths: any = {}; convertObjectToDotNotation(jsonSchema.fields, copy.fields, objectPaths); diff --git a/modules/database/src/adapters/utils/database-transform-utils.ts b/modules/database/src/adapters/utils/database-transform-utils.ts index d2e53fa74..45667f669 100644 --- a/modules/database/src/adapters/utils/database-transform-utils.ts +++ b/modules/database/src/adapters/utils/database-transform-utils.ts @@ -9,6 +9,7 @@ import { has, } from 'lodash-es'; import { + ConduitGrpcSdk, ConduitModelField, Indexable, ModelOptionsIndex, @@ -43,6 +44,8 @@ export function checkDefaultValue(type: string, value: string) { } export function convertModelOptionsIndexes(copy: ConduitDatabaseSchema, dialect: string) { + const convertedIndexes = []; + for (const index of copy.modelOptions.indexes!) { const { fields, types, options } = index; const compiledFields = Object.keys(copy.compiledFields); @@ -55,7 +58,10 @@ export function convertModelOptionsIndexes(copy: ConduitDatabaseSchema, dialect: // Convert conduit indexes to sequelize indexes if (options) { if (!checkIfSequelizeIndexOptions(options, dialect)) { - throw new Error(`Invalid index options for ${dialect}`); + ConduitGrpcSdk.Logger.warn( + `Invalid index options for ${dialect} found in: ${copy.name}. Index ignored`, + ); + continue; } // Used instead of ModelOptionsIndexes fields for more complex index definitions const seqOptions = options as SequelizeIndexOptions; @@ -71,7 +77,10 @@ export function convertModelOptionsIndexes(copy: ConduitDatabaseSchema, dialect: } if (types) { if (types.length !== 1 || !checkIfSequelizeIndexType(types[0], dialect)) { - throw new Error(`Invalid index type for ${dialect}`); + ConduitGrpcSdk.Logger.warn( + `Invalid index type for ${dialect} found in: ${copy.name}. Index ignored`, + ); + continue; } if ( (dialect === 'mysql' || dialect === 'mariadb') && @@ -83,8 +92,9 @@ export function convertModelOptionsIndexes(copy: ConduitDatabaseSchema, dialect: } delete index.types; } + convertedIndexes.push(index); } - return copy; + return convertedIndexes; } export function convertSchemaFieldIndexes(copy: ConduitDatabaseSchema, dialect: string) { @@ -97,7 +107,11 @@ export function convertSchemaFieldIndexes(copy: ConduitDatabaseSchema, dialect: const newIndex: any = { name, fields: [fieldName] }; if (type) { if (isArray(type) || !checkIfSequelizeIndexType(type, dialect)) { - throw new Error(`Invalid index type for ${dialect}`); + ConduitGrpcSdk.Logger.warn( + `Invalid index type for ${dialect} found in: ${copy.name}. Index ignored`, + ); + delete (copy.fields[fieldName] as ConduitModelField).index; + continue; } if ( (dialect === 'mysql' || dialect === 'mariadb') && @@ -109,7 +123,11 @@ export function convertSchemaFieldIndexes(copy: ConduitDatabaseSchema, dialect: } } if (options && !checkIfSequelizeIndexOptions(options, dialect)) { - throw new Error(`Invalid index options for ${dialect}`); + ConduitGrpcSdk.Logger.warn( + `Invalid index options for ${dialect} found in: ${copy.name}. Index ignored`, + ); + delete (copy.fields[fieldName] as ConduitModelField).index; + continue; } Object.assign(newIndex, options); indexes.push(newIndex); diff --git a/modules/database/src/adapters/utils/indexValidations.ts b/modules/database/src/adapters/utils/indexValidations.ts new file mode 100644 index 000000000..47a0b9e99 --- /dev/null +++ b/modules/database/src/adapters/utils/indexValidations.ts @@ -0,0 +1,32 @@ +import { ConduitModel, GrpcError, ModelOptionsIndex } from '@conduitplatform/grpc-sdk'; +import { status } from '@grpc/grpc-js'; +import { ConduitDatabaseSchema } from '../../interfaces/index.js'; + +export function validateIndexFields( + schema: ConduitDatabaseSchema, + index: ModelOptionsIndex, + callerModule: string, +) { + const compiledFields = schema.compiledFields; + if (index.fields.some(field => !Object.keys(compiledFields).includes(field))) { + throw new GrpcError(status.INVALID_ARGUMENT, 'Invalid fields for index creation'); + } + + let ownedExtensionFields: ConduitModel = {}; + for (const ext of schema.extensions) { + if (ext.ownerModule === callerModule) { + ownedExtensionFields = ext.fields; + break; + } + } + + const isOwnerOfSchema = schema.ownerModule === callerModule; + for (const field of index.fields) { + if (index.options?.unique && !(field in ownedExtensionFields) && !isOwnerOfSchema) { + throw new GrpcError( + status.PERMISSION_DENIED, + 'Not authorized to create unique index', + ); + } + } +} diff --git a/modules/database/src/admin/index.ts b/modules/database/src/admin/index.ts index 12cc8b46e..4bd4a0db6 100644 --- a/modules/database/src/admin/index.ts +++ b/modules/database/src/admin/index.ts @@ -4,6 +4,10 @@ import { ConduitRouteActions, ConduitRouteReturnDefinition, TYPE, + ConduitModelFieldJSON, + ConduitModel, + allowedTypes, + ConduitModelField, } from '@conduitplatform/grpc-sdk'; import { ConduitBoolean, @@ -603,7 +607,15 @@ export class AdminHandlers { action: ConduitRouteActions.POST, description: `Imports indexes.`, bodyParams: { - indexes: [ConduitJson.Required], + indexes: [ + { + schemaName: ConduitString.Required, + name: ConduitString.Required, + fields: [ConduitString.Required], + types: [ConduitString.Optional], + options: ConduitJson.Optional, + } as never, + ], }, }, new ConduitRouteReturnDefinition('ImportIndexes', 'String'), @@ -619,8 +631,8 @@ export class AdminHandlers { indexes: [ { schemaName: ConduitString.Required, - name: ConduitString.Required, fields: [ConduitString.Required], + name: ConduitString.Optional, types: [ConduitString.Optional], options: ConduitJson.Optional, }, @@ -632,19 +644,23 @@ export class AdminHandlers { { path: '/schemas/:id/indexes', action: ConduitRouteActions.POST, - description: `Creates an index for a schema.`, + description: `Creates indexes for a schema.`, urlParams: { id: { type: TYPE.String, required: true }, }, bodyParams: { - name: ConduitString.Required, - fields: [ConduitString.Required], - types: [ConduitString.Required], - options: ConduitJson.Optional, - }, - }, - new ConduitRouteReturnDefinition('CreateSchemaIndex', 'String'), - this.schemaAdmin.createIndex.bind(this.schemaAdmin), + indexes: [ + { + fields: [ConduitString.Required], + types: [ConduitString.Required], + name: ConduitString.Optional, + options: ConduitJson.Optional, + } as never, + ], + }, + }, + new ConduitRouteReturnDefinition('CreateSchemaIndexes', 'String'), + this.schemaAdmin.createIndexes.bind(this.schemaAdmin), ); this.routingManager.route( { diff --git a/modules/database/src/admin/schema.admin.ts b/modules/database/src/admin/schema.admin.ts index 496f4f38e..9c7268361 100644 --- a/modules/database/src/admin/schema.admin.ts +++ b/modules/database/src/admin/schema.admin.ts @@ -3,6 +3,7 @@ import { ConduitSchema, GrpcError, Indexable, + ModelOptionsIndex, ParsedRouterRequest, UnparsedRouterResponse, } from '@conduitplatform/grpc-sdk'; @@ -642,9 +643,19 @@ export class SchemaAdmin { } async importIndexes(call: ParsedRouterRequest): Promise { - for (const index of call.request.params.indexes) { - await this.database.createIndex(index.schemaName, index, 'database').catch(e => { - throw new GrpcError(status.INTERNAL, `${index.name}: ${e.message}`); + const { indexes } = call.request.params; + const indexMap = new Map(); + for (const i of indexes) { + const { schemaName, ...rest } = i; + if (indexMap.has(schemaName)) { + indexMap.get(schemaName)!.push(rest); + } else { + indexMap.set(i.schemaName, [rest]); + } + } + for (const [schemaName, indexes] of indexMap) { + await this.database.createIndexes(schemaName, indexes, 'database').catch(e => { + throw new GrpcError(status.INTERNAL, `Index creation failed: ${e.message}`); }); } return 'Indexes imported successfully'; @@ -652,9 +663,18 @@ export class SchemaAdmin { async exportIndexes(): Promise { const indexes = []; - const schemas = await this.database - .getSchemaModel('_DeclaredSchema') - .model.findMany({}); + const schemas = await this.database.getSchemaModel('_DeclaredSchema').model.findMany({ + $or: [ + { 'modelOptions.conduit.cms.enabled': true }, + { + $and: [ + { 'modelOptions.conduit.cms': { $exists: false } }, + { 'modelOptions.conduit.permissions.extendable': true }, + { extensions: { $exists: true } }, + ], + }, + ], + }); for (const schema of schemas) { const schemaIndexes = await this.database.getIndexes(schema.name); if (!isNil(schemaIndexes) && !isEmpty(schemaIndexes)) { @@ -666,19 +686,15 @@ export class SchemaAdmin { return { indexes }; } - async createIndex(call: ParsedRouterRequest): Promise { - const { id, name, fields, types, options } = call.request.params; + async createIndexes(call: ParsedRouterRequest): Promise { + const { id, indexes } = call.request.params; const requestedSchema = await this.database .getSchemaModel('_DeclaredSchema') .model.findOne({ _id: id }); if (isNil(requestedSchema)) { throw new GrpcError(status.NOT_FOUND, 'Schema does not exist'); } - return await this.database.createIndex( - requestedSchema.name, - { name, fields, types, options }, - 'database', - ); + return await this.database.createIndexes(requestedSchema.name, indexes, 'database'); } async getIndexes(call: ParsedRouterRequest): Promise { From 4ad9b01a43556246d4988425ee21218602eef8bb Mon Sep 17 00:00:00 2001 From: chris Date: Thu, 1 Aug 2024 11:52:54 +0300 Subject: [PATCH 16/17] chore: unused imports --- modules/authentication/src/models/User.schema.ts | 2 -- modules/database/src/admin/index.ts | 4 ---- 2 files changed, 6 deletions(-) diff --git a/modules/authentication/src/models/User.schema.ts b/modules/authentication/src/models/User.schema.ts index e1eb6e29a..a7a5bcc7e 100644 --- a/modules/authentication/src/models/User.schema.ts +++ b/modules/authentication/src/models/User.schema.ts @@ -2,8 +2,6 @@ import { ConduitModel, DatabaseProvider, Indexable, - ModelOptionsIndex, - MongoIndexType, TYPE, } from '@conduitplatform/grpc-sdk'; import { ConduitActiveSchema } from '@conduitplatform/module-tools'; diff --git a/modules/database/src/admin/index.ts b/modules/database/src/admin/index.ts index 4bd4a0db6..cfe0c17d6 100644 --- a/modules/database/src/admin/index.ts +++ b/modules/database/src/admin/index.ts @@ -4,10 +4,6 @@ import { ConduitRouteActions, ConduitRouteReturnDefinition, TYPE, - ConduitModelFieldJSON, - ConduitModel, - allowedTypes, - ConduitModelField, } from '@conduitplatform/grpc-sdk'; import { ConduitBoolean, From ea54457652646256342a4ee8fe5d7a880f6718e4 Mon Sep 17 00:00:00 2001 From: chris Date: Fri, 2 Aug 2024 14:31:01 +0300 Subject: [PATCH 17/17] fix: mongoose index types bug --- modules/database/src/adapters/mongoose-adapter/index.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/modules/database/src/adapters/mongoose-adapter/index.ts b/modules/database/src/adapters/mongoose-adapter/index.ts index 85568eca1..ff19561a1 100644 --- a/modules/database/src/adapters/mongoose-adapter/index.ts +++ b/modules/database/src/adapters/mongoose-adapter/index.ts @@ -510,9 +510,11 @@ export class MongooseAdapter extends DatabaseAdapter { throw new GrpcError(status.INTERNAL, 'Invalid index types'); } // Convert index types (if called by endpoint index.types contains the keys of MongoIndexType enum) - index.types = types.map( - type => MongoIndexType[type as unknown as keyof typeof MongoIndexType] || type, - ) as MongoIndexType[]; + index.types = types.map((type: unknown) => { + if (Object.keys(MongoIndexType).includes(type as keyof typeof MongoIndexType)) + return MongoIndexType[type as unknown as keyof typeof MongoIndexType]; + return type; + }) as MongoIndexType[]; return index; } }