diff --git a/packages/datalib/src/PerspectiveCache.ts b/packages/datalib/src/PerspectiveCache.ts index 5446d87a8..44d99c7c5 100644 --- a/packages/datalib/src/PerspectiveCache.ts +++ b/packages/datalib/src/PerspectiveCache.ts @@ -105,7 +105,6 @@ export class PerspectiveCache { 'databaseConfig', 'orderBy', 'sqlCondition', - 'mongoCondition', ]) ); let res = this.tables[tableKey]; diff --git a/packages/datalib/src/PerspectiveDataLoader.ts b/packages/datalib/src/PerspectiveDataLoader.ts index 1c9df1439..ea94b452b 100644 --- a/packages/datalib/src/PerspectiveDataLoader.ts +++ b/packages/datalib/src/PerspectiveDataLoader.ts @@ -5,6 +5,7 @@ import _zipObject from 'lodash/zipObject'; import _mapValues from 'lodash/mapValues'; import _isArray from 'lodash/isArray'; import { safeJsonParse } from 'dbgate-tools'; +import { CollectionAggregateDefinition } from 'dbgate-types'; function normalizeLoadedRow(row) { return _mapValues(row, v => safeJsonParse(v) || v); @@ -59,24 +60,6 @@ export class PerspectiveDataLoader { : null; } - buildMongoCondition(props: PerspectiveDataLoadProps): {} { - const { schemaName, pureName, bindingColumns, bindingValues, dataColumns, orderBy, mongoCondition } = props; - - const conditions = []; - - if (mongoCondition) { - conditions.push(mongoCondition); - } - - if (bindingColumns?.length == 1) { - conditions.push({ - [bindingColumns[0]]: { $in: bindingValues.map(x => x[0]) }, - }); - } - - return conditions.length == 1 ? conditions[0] : conditions.length > 0 ? { $and: conditions } : null; - } - async loadGroupingSqlDb(props: PerspectiveDataLoadProps) { const { schemaName, pureName, bindingColumns } = props; @@ -135,18 +118,28 @@ export class PerspectiveDataLoader { async loadGroupingDocDb(props: PerspectiveDataLoadProps) { const { schemaName, pureName, bindingColumns } = props; - const aggregate = [ - { $match: this.buildMongoCondition(props) }, - { - $group: { - _id: _zipObject( - bindingColumns, - bindingColumns.map(col => '$' + col) - ), - count: { $sum: 1 }, + const aggregate: CollectionAggregateDefinition = { + condition: this.buildSqlCondition(props), + groupByColumns: bindingColumns, + aggregateColumns: [ + { + alias: 'acount', + aggregateFunction: 'count', }, - }, - ]; + ], + }; + // const aggregate = [ + // { $match: this.buildMongoCondition(props) }, + // { + // $group: { + // _id: _zipObject( + // bindingColumns, + // bindingColumns.map(col => '$' + col) + // ), + // count: { $sum: 1 }, + // }, + // }, + // ]; if (dbg?.enabled) { dbg(`LOAD COUNTS, table=${props.pureName}, columns=${bindingColumns?.join(',')}`); @@ -244,7 +237,7 @@ export class PerspectiveDataLoader { const { pureName } = props; const res: any = { pureName, - condition: this.buildMongoCondition(props), + condition: this.buildSqlCondition(props), skip: props.range?.offset, limit: props.range?.limit, }; diff --git a/packages/datalib/src/PerspectiveDataProvider.ts b/packages/datalib/src/PerspectiveDataProvider.ts index d1dd11dd1..23e0171a7 100644 --- a/packages/datalib/src/PerspectiveDataProvider.ts +++ b/packages/datalib/src/PerspectiveDataProvider.ts @@ -25,7 +25,6 @@ export interface PerspectiveDataLoadProps { range?: RangeDefinition; topCount?: number; sqlCondition?: Condition; - mongoCondition?: any; engineType: PerspectiveDatabaseEngineType; } diff --git a/packages/datalib/src/PerspectiveTreeNode.ts b/packages/datalib/src/PerspectiveTreeNode.ts index 5176cafd4..8d9f7d9c2 100644 --- a/packages/datalib/src/PerspectiveTreeNode.ts +++ b/packages/datalib/src/PerspectiveTreeNode.ts @@ -349,8 +349,23 @@ export abstract class PerspectiveTreeNode { ); } + getMutliColumnCondition(source): Condition { + if (!this.nodeConfig?.multiColumnFilter) return null; + + const base = this.getBaseTableFromThis() as TableInfo | ViewInfo | CollectionInfo; + if (!base) return null; + + const isDocDb = isCollectionInfo(base); + if (isDocDb) { + return this.getMutliColumnNoSqlCondition(); + } else { + return this.getMutliColumnSqlCondition(source); + } + } + getMutliColumnSqlCondition(source): Condition { if (!this.nodeConfig?.multiColumnFilter) return null; + const base = this.getBaseTableFromThis() as TableInfo | ViewInfo; if (!base) return null; try { @@ -383,32 +398,40 @@ export abstract class PerspectiveTreeNode { return null; } - getMutliColumnMongoCondition(): {} { + getMutliColumnNoSqlCondition(): Condition { if (!this.nodeConfig?.multiColumnFilter) return null; const pattern = this.dataProvider?.dataPatterns?.[this.designerId]; if (!pattern) return null; const condition = parseFilter(this.nodeConfig?.multiColumnFilter, mongoFilterBehaviour); if (!condition) return null; - const res = pattern.columns.map(col => { - return _cloneDeepWith(condition, expr => { - if (expr.__placeholder__) { - return { - [col.name]: expr.__placeholder__, - }; - } - }); - }); - return { - $or: res, + + const orCondition: CompoudCondition = { + conditionType: 'or', + conditions: [], }; + for (const column of pattern.columns || []) { + orCondition.conditions.push( + _cloneDeepWith(condition, (expr: Expression) => { + if (expr.exprType == 'placeholder') { + return { + exprType: 'column', + columnName: column.name, + }; + } + }) + ); + } + if (orCondition.conditions.length > 0) { + return orCondition; + } } getChildrenSqlCondition(source = null): Condition { const conditions = _compact([ ...this.childNodes.map(x => x.parseFilterCondition(source)), ...this.buildParentFilterConditions(), - this.getMutliColumnSqlCondition(source), + this.getMutliColumnCondition(source), ]); if (conditions.length == 0) { return null; @@ -422,20 +445,6 @@ export abstract class PerspectiveTreeNode { }; } - getChildrenMongoCondition(source = null): {} { - const conditions = _compact([ - ...this.childNodes.map(x => x.parseFilterCondition(source)), - this.getMutliColumnMongoCondition(), - ]); - if (conditions.length == 0) { - return null; - } - if (conditions.length == 1) { - return conditions[0]; - } - return { $and: conditions }; - } - getOrderBy(table: TableInfo | ViewInfo | CollectionInfo): PerspectiveDataLoadProps['orderBy'] { const res = _compact( this.childNodes.map(node => { @@ -1158,17 +1167,16 @@ export class PerspectiveTableNode extends PerspectiveTreeNode { } getNodeLoadProps(parentRows: any[]): PerspectiveDataLoadProps { - const isMongo = isCollectionInfo(this.table); + const isDocDb = isCollectionInfo(this.table); return { schemaName: this.table.schemaName, pureName: this.table.pureName, dataColumns: this.getDataLoadColumns(), - allColumns: isMongo, + allColumns: isDocDb, databaseConfig: this.databaseConfig, orderBy: this.getOrderBy(this.table), - sqlCondition: isMongo ? null : this.getChildrenSqlCondition(), - mongoCondition: isMongo ? this.getChildrenMongoCondition() : null, - engineType: isMongo ? 'docdb' : 'sqldb', + sqlCondition: this.getChildrenSqlCondition(), + engineType: isDocDb ? 'docdb' : 'sqldb', }; } @@ -1372,7 +1380,7 @@ export class PerspectiveCustomJoinTreeNode extends PerspectiveTableNode { // console.log('PARENT ROWS', parentRows); // console.log('this.getDataLoadColumns()', this.getDataLoadColumns()); - const isMongo = isCollectionInfo(this.table); + const isDocDb = isCollectionInfo(this.table); // const bindingValues = []; @@ -1432,12 +1440,12 @@ export class PerspectiveCustomJoinTreeNode extends PerspectiveTableNode { bindingColumns: this.getParentMatchColumns(), bindingValues: _uniqBy(bindingValues, x => JSON.stringify(x)), dataColumns: this.getDataLoadColumns(), - allColumns: isMongo, + allColumns: isDocDb, databaseConfig: this.databaseConfig, orderBy: this.getOrderBy(this.table), - sqlCondition: isMongo ? null : this.getChildrenSqlCondition(), - mongoCondition: isMongo ? this.getChildrenMongoCondition() : null, - engineType: isMongo ? 'docdb' : 'sqldb', + sqlCondition: this.getChildrenSqlCondition(), + // mongoCondition: isMongo ? this.getChildrenMongoCondition() : null, + engineType: isDocDb ? 'docdb' : 'sqldb', }; } diff --git a/packages/types/engines.d.ts b/packages/types/engines.d.ts index 6cd8ee45f..cf8c1d2b4 100644 --- a/packages/types/engines.d.ts +++ b/packages/types/engines.d.ts @@ -41,6 +41,8 @@ export interface ReadCollectionOptions { countDocuments?: boolean; skip?: number; limit?: number; + condition?: any; + aggregate?: CollectionAggregateDefinition; } export interface NewObjectTemplate { @@ -72,6 +74,17 @@ export interface ServerSummary { databases: ServerSummaryDatabase[]; } +export type CollectionAggregateFunction = 'count' | 'sum' | 'avg' | 'min' | 'max'; +export interface CollectionAggregateDefinition { + condition: any; // SQL tree condition + groupByColumns: string[]; + aggregateColumns: { + alias: string; + aggregateFunction: CollectionAggregateFunction; + columnArgument?: string; + }[]; +} + export interface FilterBehaviourProvider { getFilterBehaviour(dataType: string, standardFilterBehaviours: { [id: string]: FilterBehaviour }): FilterBehaviour; } diff --git a/packages/types/filter-type.d.ts b/packages/types/filter-type.d.ts index 1b36ae04c..d3fd5dde7 100644 --- a/packages/types/filter-type.d.ts +++ b/packages/types/filter-type.d.ts @@ -1,5 +1,3 @@ -export type FilterParserCompilerType = 'sqlTree' | 'mongoCondition' | 'datetime'; - export interface FilterBehaviour { supportEquals?: boolean; supportStringInclusion?: boolean; diff --git a/plugins/dbgate-plugin-mongo/package.json b/plugins/dbgate-plugin-mongo/package.json index d56803534..1a0b80879 100644 --- a/plugins/dbgate-plugin-mongo/package.json +++ b/plugins/dbgate-plugin-mongo/package.json @@ -33,6 +33,7 @@ "devDependencies": { "dbgate-plugin-tools": "^1.0.7", "dbgate-query-splitter": "^4.10.1", + "lodash": "^4.17.21", "webpack": "^5.91.0", "webpack-cli": "^5.1.4", "dbgate-tools": "^5.0.0-alpha.1", diff --git a/plugins/dbgate-plugin-mongo/src/backend/driver.js b/plugins/dbgate-plugin-mongo/src/backend/driver.js index b548b470f..baa963fe3 100644 --- a/plugins/dbgate-plugin-mongo/src/backend/driver.js +++ b/plugins/dbgate-plugin-mongo/src/backend/driver.js @@ -7,7 +7,7 @@ const MongoClient = require('mongodb').MongoClient; const ObjectId = require('mongodb').ObjectId; const AbstractCursor = require('mongodb').AbstractCursor; const createBulkInsertStream = require('./createBulkInsertStream'); -const { convertToMongoCondition } = require('../frontend/convertToMongoCondition'); +const { convertToMongoCondition, convertToMongoAggregate } = require('../frontend/convertToMongoCondition'); function transformMongoData(row) { return _.cloneDeepWith(row, (x) => { @@ -280,7 +280,7 @@ const driver = { const count = await collection.countDocuments(convertObjectId(mongoCondition) || {}); return { count }; } else if (options.aggregate) { - let cursor = await collection.aggregate(convertObjectId(options.aggregate)); + let cursor = await collection.aggregate(convertObjectId(convertToMongoAggregate(options.aggregate))); const rows = await cursor.toArray(); return { rows: rows.map(transformMongoData) }; } else { diff --git a/plugins/dbgate-plugin-mongo/src/frontend/convertToMongoCondition.js b/plugins/dbgate-plugin-mongo/src/frontend/convertToMongoCondition.js index 1ac599d9e..66fbe8e3c 100644 --- a/plugins/dbgate-plugin-mongo/src/frontend/convertToMongoCondition.js +++ b/plugins/dbgate-plugin-mongo/src/frontend/convertToMongoCondition.js @@ -1,3 +1,5 @@ +const _zipObject = require('lodash/zipObject'); + function convertLeftOperandToMongoColumn(left) { if (left.exprType == 'placeholder') return '__placeholder__'; if (left.exprType == 'column') return left.columnName; @@ -121,11 +123,55 @@ function convertToMongoCondition(filter) { }; } + case 'in': + return { + [convertLeftOperandToMongoColumn(filter.expr)]: { + $in: filter.values, + }, + }; + default: throw new Error(`Unknown condition type ${filter.conditionType}`); } } +function convertToMongoAggregateFunction(aggregate) { + switch (aggregate.aggregateFunction) { + case 'count': + return { $sum: 1 }; + case 'sum': + return { $sum: `$${aggregate.columnArgument}` }; + case 'avg': + return { $avg: `$${aggregate.columnArgument}` }; + case 'min': + return { $min: `$${aggregate.columnArgument}` }; + case 'max': + return { $max: `$${aggregate.columnArgument}` }; + default: + throw new Error(`Unknown aggregate function ${aggregate.aggregateFunction}`); + } +} + +function convertToMongoAggregate(collectionAggregate) { + return [ + { $match: convertToMongoCondition(collectionAggregate.condition) }, + { + $group: { + _id: _zipObject( + collectionAggregate.groupByColumns, + collectionAggregate.groupByColumns.map((col) => '$' + col) + ), + ..._zipObject( + collectionAggregate.aggregateColumns.map((col) => col.alias), + collectionAggregate.aggregateColumns.map((col) => convertToMongoAggregateFunction(col)) + ), + count: { $sum: 1 }, + }, + }, + ]; +} + module.exports = { convertToMongoCondition, + convertToMongoAggregate, }; diff --git a/yarn.lock b/yarn.lock index c0150cc76..817e109d5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -31,7 +31,7 @@ dependencies: tslib "^2.6.2" -"@azure/core-auth@^1.3.0", "@azure/core-auth@^1.4.0", "@azure/core-auth@^1.5.0", "@azure/core-auth@^1.7.1": +"@azure/core-auth@^1.3.0", "@azure/core-auth@^1.4.0", "@azure/core-auth@^1.5.0": version "1.7.2" resolved "https://registry.yarnpkg.com/@azure/core-auth/-/core-auth-1.7.2.tgz#558b7cb7dd12b00beec07ae5df5907d74df1ebd9" integrity sha512-Igm/S3fDYmnMq1uKS38Ae1/m37B3zigdlZw+kocwEhh5GjyKjPrXKO2J6rzpC1wAxrNil/jX9BJRqBshyjnF3g== @@ -93,21 +93,7 @@ https-proxy-agent "^7.0.0" tslib "^2.6.2" -"@azure/core-rest-pipeline@^1.15.1": - version "1.16.3" - resolved "https://registry.yarnpkg.com/@azure/core-rest-pipeline/-/core-rest-pipeline-1.16.3.tgz#bde3bc3ebad7f885ddd9de6af5e5a8fc254b287e" - integrity sha512-VxLk4AHLyqcHsfKe4MZ6IQ+D+ShuByy+RfStKfSjxJoL3WBWq17VNmrz8aT8etKzqc2nAeIyLxScjpzsS4fz8w== - dependencies: - "@azure/abort-controller" "^2.0.0" - "@azure/core-auth" "^1.4.0" - "@azure/core-tracing" "^1.0.1" - "@azure/core-util" "^1.9.0" - "@azure/logger" "^1.0.0" - http-proxy-agent "^7.0.0" - https-proxy-agent "^7.0.0" - tslib "^2.6.2" - -"@azure/core-tracing@^1.0.0", "@azure/core-tracing@^1.0.1", "@azure/core-tracing@^1.1.1": +"@azure/core-tracing@^1.0.0", "@azure/core-tracing@^1.0.1": version "1.1.2" resolved "https://registry.yarnpkg.com/@azure/core-tracing/-/core-tracing-1.1.2.tgz#065dab4e093fb61899988a1cdbc827d9ad90b4ee" integrity sha512-dawW9ifvWAWmUm9/h+/UQ2jrdvjCJ7VJEuCJ6XVNudzcOwm53BFZH4Q845vjfgoUAM8ZxokvVNxNxAITc502YA== @@ -122,30 +108,6 @@ "@azure/abort-controller" "^2.0.0" tslib "^2.6.2" -"@azure/core-util@^1.8.1": - version "1.9.2" - resolved "https://registry.yarnpkg.com/@azure/core-util/-/core-util-1.9.2.tgz#1dc37dc5b0dae34c578be62cf98905ba7c0cafe7" - integrity sha512-l1Qrqhi4x1aekkV+OlcqsJa4AnAkj5p0JV8omgwjaV9OAbP41lvrMvs+CptfetKkeEaGRGSzby7sjPZEX7+kkQ== - dependencies: - "@azure/abort-controller" "^2.0.0" - tslib "^2.6.2" - -"@azure/cosmos@^4.1.0": - version "4.1.0" - resolved "https://registry.yarnpkg.com/@azure/cosmos/-/cosmos-4.1.0.tgz#97014d8110d94c4b47911350a018ad2d726493b0" - integrity sha512-+m085WKIGkf6wyw4vT85FFXl9j3U35u+LFFVwmLqfPbolnQAtoX24cowXz+vseW4BWKyx6Lamb+Zz+jl69zn6g== - dependencies: - "@azure/abort-controller" "^2.0.0" - "@azure/core-auth" "^1.7.1" - "@azure/core-rest-pipeline" "^1.15.1" - "@azure/core-tracing" "^1.1.1" - "@azure/core-util" "^1.8.1" - fast-json-stable-stringify "^2.1.0" - jsbi "^4.3.0" - priorityqueuejs "^2.0.0" - semaphore "^1.1.0" - tslib "^2.6.2" - "@azure/identity@^3.4.1": version "3.4.2" resolved "https://registry.yarnpkg.com/@azure/identity/-/identity-3.4.2.tgz#6b01724c9caac7cadab6b63c76584345bda8e2de" @@ -3245,26 +3207,6 @@ dbgate-query-splitter@^4.10.1: resolved "https://registry.yarnpkg.com/dbgate-query-splitter/-/dbgate-query-splitter-4.10.1.tgz#dc40d792de06f779a743cad054d5e786006b03a9" integrity sha512-KqrB7NLP1jXbx8rN7gSmYUVorm6ICeqOV+oR+jHaBLXqqhWepHsKr6JJlFEeb/LhoVjnTDY/cy5zhW1dMIQF6A== -dbgate-sqltree@^5.3.4: - version "5.3.4" - resolved "https://registry.yarnpkg.com/dbgate-sqltree/-/dbgate-sqltree-5.3.4.tgz#d91bbb1a3264dc8d88898fbb5427ee15aacc14a3" - integrity sha512-pvfjuI51plcmwErxxDCl8ZLXb9VfDnT+NukEMExiytYrTg3dDF2j9xwsYR9MZ/UaOQjwEO4LZ2FBLg/B776CuA== - dependencies: - lodash "^4.17.21" - -dbgate-tools@^5.0.0: - version "5.3.4" - resolved "https://registry.yarnpkg.com/dbgate-tools/-/dbgate-tools-5.3.4.tgz#168662ccd92e404a31fe3d5e29f732a91a0ea9b6" - integrity sha512-EsZafhQIGx8AlUT5PIMXoS0LTCxtANuQnxFUMwL20DfW/CCJtwWALHwwi2Am+1/YbDeM9Uh3FhWPTiZh5fhLYA== - dependencies: - dbgate-query-splitter "^4.10.1" - dbgate-sqltree "^5.3.4" - debug "^4.3.4" - json-stable-stringify "^1.0.1" - lodash "^4.17.21" - pinomin "^1.0.4" - uuid "^3.4.0" - debug@2.6.9, debug@^2.2.0, debug@^2.3.3: version "2.6.9" resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" @@ -4141,7 +4083,7 @@ fast-glob@^3.0.3: merge2 "^1.3.0" micromatch "^4.0.4" -fast-json-stable-stringify@2.x, fast-json-stable-stringify@^2.0.0, fast-json-stable-stringify@^2.1.0: +fast-json-stable-stringify@2.x, fast-json-stable-stringify@^2.0.0: version "2.1.0" resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz#874bf69c6f404c2b5d99c481341399fd55892633" integrity sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw== @@ -6702,11 +6644,6 @@ js-yaml@^4.1.0: dependencies: argparse "^2.0.1" -jsbi@^4.3.0: - version "4.3.0" - resolved "https://registry.yarnpkg.com/jsbi/-/jsbi-4.3.0.tgz#b54ee074fb6fcbc00619559305c8f7e912b04741" - integrity sha512-SnZNcinB4RIcnEyZqFPdGPVgrg2AcnykiBy0sHVJQKHYeaLUvi3Exj+iaPpLnFVkDPZIV4U0yvgC9/R4uEAZ9g== - jsbn@1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-1.1.0.tgz#b01307cb29b618a1ed26ec79e911f803c4da0040" @@ -8521,11 +8458,6 @@ printj@~1.1.0: resolved "https://registry.yarnpkg.com/printj/-/printj-1.1.2.tgz#d90deb2975a8b9f600fb3a1c94e3f4c53c78a222" integrity sha512-zA2SmoLaxZyArQTOPj5LXecR+RagfPSU5Kw1qP+jkWeNlrq+eJZyY2oS68SU1Z/7/myXM4lo9716laOFAVStCQ== -priorityqueuejs@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/priorityqueuejs/-/priorityqueuejs-2.0.0.tgz#96064040edd847ee9dd3013d8e16297399a6bd4f" - integrity sha512-19BMarhgpq3x4ccvVi8k2QpJZcymo/iFUcrhPd4V96kYGovOdTsWwy7fxChYi4QY+m2EnGBWSX9Buakz+tWNQQ== - process-nextick-args@~1.0.6: version "1.0.7" resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-1.0.7.tgz#150e20b756590ad3f91093f25a4f2ad8bff30ba3" @@ -9235,11 +9167,6 @@ secure-json-parse@^2.4.0: resolved "https://registry.yarnpkg.com/secure-json-parse/-/secure-json-parse-2.7.0.tgz#5a5f9cd6ae47df23dba3151edd06855d47e09862" integrity sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw== -semaphore@^1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/semaphore/-/semaphore-1.1.0.tgz#aaad8b86b20fe8e9b32b16dc2ee682a8cd26a8aa" - integrity sha512-O4OZEaNtkMd/K0i6js9SL+gqy0ZCBMgUvlSqHKi4IBdjhe7wB8pwztUk1BbZ1fmrvpwFrPbHzqd2w5pTcJH6LA== - semiver@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/semiver/-/semiver-1.1.0.tgz#9c97fb02c21c7ce4fcf1b73e2c7a24324bdddd5f"