diff --git a/packages/datalib/src/PerspectiveCache.ts b/packages/datalib/src/PerspectiveCache.ts index 5446d87a8..44d99c7c5 100644 --- a/packages/datalib/src/PerspectiveCache.ts +++ b/packages/datalib/src/PerspectiveCache.ts @@ -105,7 +105,6 @@ export class PerspectiveCache { 'databaseConfig', 'orderBy', 'sqlCondition', - 'mongoCondition', ]) ); let res = this.tables[tableKey]; diff --git a/packages/datalib/src/PerspectiveDataLoader.ts b/packages/datalib/src/PerspectiveDataLoader.ts index 1c9df1439..ea94b452b 100644 --- a/packages/datalib/src/PerspectiveDataLoader.ts +++ b/packages/datalib/src/PerspectiveDataLoader.ts @@ -5,6 +5,7 @@ import _zipObject from 'lodash/zipObject'; import _mapValues from 'lodash/mapValues'; import _isArray from 'lodash/isArray'; import { safeJsonParse } from 'dbgate-tools'; +import { CollectionAggregateDefinition } from 'dbgate-types'; function normalizeLoadedRow(row) { return _mapValues(row, v => safeJsonParse(v) || v); @@ -59,24 +60,6 @@ export class PerspectiveDataLoader { : null; } - buildMongoCondition(props: PerspectiveDataLoadProps): {} { - const { schemaName, pureName, bindingColumns, bindingValues, dataColumns, orderBy, mongoCondition } = props; - - const conditions = []; - - if (mongoCondition) { - conditions.push(mongoCondition); - } - - if (bindingColumns?.length == 1) { - conditions.push({ - [bindingColumns[0]]: { $in: bindingValues.map(x => x[0]) }, - }); - } - - return conditions.length == 1 ? conditions[0] : conditions.length > 0 ? { $and: conditions } : null; - } - async loadGroupingSqlDb(props: PerspectiveDataLoadProps) { const { schemaName, pureName, bindingColumns } = props; @@ -135,18 +118,28 @@ export class PerspectiveDataLoader { async loadGroupingDocDb(props: PerspectiveDataLoadProps) { const { schemaName, pureName, bindingColumns } = props; - const aggregate = [ - { $match: this.buildMongoCondition(props) }, - { - $group: { - _id: _zipObject( - bindingColumns, - bindingColumns.map(col => '$' + col) - ), - count: { $sum: 1 }, + const aggregate: CollectionAggregateDefinition = { + condition: this.buildSqlCondition(props), + groupByColumns: bindingColumns, + aggregateColumns: [ + { + alias: 'acount', + aggregateFunction: 'count', }, - }, - ]; + ], + }; + // const aggregate = [ + // { $match: this.buildMongoCondition(props) }, + // { + // $group: { + // _id: _zipObject( + // bindingColumns, + // bindingColumns.map(col => '$' + col) + // ), + // count: { $sum: 1 }, + // }, + // }, + // ]; if (dbg?.enabled) { dbg(`LOAD COUNTS, table=${props.pureName}, columns=${bindingColumns?.join(',')}`); @@ -244,7 +237,7 @@ export class PerspectiveDataLoader { const { pureName } = props; const res: any = { pureName, - condition: this.buildMongoCondition(props), + condition: this.buildSqlCondition(props), skip: props.range?.offset, limit: props.range?.limit, }; diff --git a/packages/datalib/src/PerspectiveDataProvider.ts b/packages/datalib/src/PerspectiveDataProvider.ts index d1dd11dd1..23e0171a7 100644 --- a/packages/datalib/src/PerspectiveDataProvider.ts +++ b/packages/datalib/src/PerspectiveDataProvider.ts @@ -25,7 +25,6 @@ export interface PerspectiveDataLoadProps { range?: RangeDefinition; topCount?: number; sqlCondition?: Condition; - mongoCondition?: any; engineType: PerspectiveDatabaseEngineType; } diff --git a/packages/datalib/src/PerspectiveTreeNode.ts b/packages/datalib/src/PerspectiveTreeNode.ts index 5176cafd4..8d9f7d9c2 100644 --- a/packages/datalib/src/PerspectiveTreeNode.ts +++ b/packages/datalib/src/PerspectiveTreeNode.ts @@ -349,8 +349,23 @@ export abstract class PerspectiveTreeNode { ); } + getMutliColumnCondition(source): Condition { + if (!this.nodeConfig?.multiColumnFilter) return null; + + const base = this.getBaseTableFromThis() as TableInfo | ViewInfo | CollectionInfo; + if (!base) return null; + + const isDocDb = isCollectionInfo(base); + if (isDocDb) { + return this.getMutliColumnNoSqlCondition(); + } else { + return this.getMutliColumnSqlCondition(source); + } + } + getMutliColumnSqlCondition(source): Condition { if (!this.nodeConfig?.multiColumnFilter) return null; + const base = this.getBaseTableFromThis() as TableInfo | ViewInfo; if (!base) return null; try { @@ -383,32 +398,40 @@ export abstract class PerspectiveTreeNode { return null; } - getMutliColumnMongoCondition(): {} { + getMutliColumnNoSqlCondition(): Condition { if (!this.nodeConfig?.multiColumnFilter) return null; const pattern = this.dataProvider?.dataPatterns?.[this.designerId]; if (!pattern) return null; const condition = parseFilter(this.nodeConfig?.multiColumnFilter, mongoFilterBehaviour); if (!condition) return null; - const res = pattern.columns.map(col => { - return _cloneDeepWith(condition, expr => { - if (expr.__placeholder__) { - return { - [col.name]: expr.__placeholder__, - }; - } - }); - }); - return { - $or: res, + + const orCondition: CompoudCondition = { + conditionType: 'or', + conditions: [], }; + for (const column of pattern.columns || []) { + orCondition.conditions.push( + _cloneDeepWith(condition, (expr: Expression) => { + if (expr.exprType == 'placeholder') { + return { + exprType: 'column', + columnName: column.name, + }; + } + }) + ); + } + if (orCondition.conditions.length > 0) { + return orCondition; + } } getChildrenSqlCondition(source = null): Condition { const conditions = _compact([ ...this.childNodes.map(x => x.parseFilterCondition(source)), ...this.buildParentFilterConditions(), - this.getMutliColumnSqlCondition(source), + this.getMutliColumnCondition(source), ]); if (conditions.length == 0) { return null; @@ -422,20 +445,6 @@ export abstract class PerspectiveTreeNode { }; } - getChildrenMongoCondition(source = null): {} { - const conditions = _compact([ - ...this.childNodes.map(x => x.parseFilterCondition(source)), - this.getMutliColumnMongoCondition(), - ]); - if (conditions.length == 0) { - return null; - } - if (conditions.length == 1) { - return conditions[0]; - } - return { $and: conditions }; - } - getOrderBy(table: TableInfo | ViewInfo | CollectionInfo): PerspectiveDataLoadProps['orderBy'] { const res = _compact( this.childNodes.map(node => { @@ -1158,17 +1167,16 @@ export class PerspectiveTableNode extends PerspectiveTreeNode { } getNodeLoadProps(parentRows: any[]): PerspectiveDataLoadProps { - const isMongo = isCollectionInfo(this.table); + const isDocDb = isCollectionInfo(this.table); return { schemaName: this.table.schemaName, pureName: this.table.pureName, dataColumns: this.getDataLoadColumns(), - allColumns: isMongo, + allColumns: isDocDb, databaseConfig: this.databaseConfig, orderBy: this.getOrderBy(this.table), - sqlCondition: isMongo ? null : this.getChildrenSqlCondition(), - mongoCondition: isMongo ? this.getChildrenMongoCondition() : null, - engineType: isMongo ? 'docdb' : 'sqldb', + sqlCondition: this.getChildrenSqlCondition(), + engineType: isDocDb ? 'docdb' : 'sqldb', }; } @@ -1372,7 +1380,7 @@ export class PerspectiveCustomJoinTreeNode extends PerspectiveTableNode { // console.log('PARENT ROWS', parentRows); // console.log('this.getDataLoadColumns()', this.getDataLoadColumns()); - const isMongo = isCollectionInfo(this.table); + const isDocDb = isCollectionInfo(this.table); // const bindingValues = []; @@ -1432,12 +1440,12 @@ export class PerspectiveCustomJoinTreeNode extends PerspectiveTableNode { bindingColumns: this.getParentMatchColumns(), bindingValues: _uniqBy(bindingValues, x => JSON.stringify(x)), dataColumns: this.getDataLoadColumns(), - allColumns: isMongo, + allColumns: isDocDb, databaseConfig: this.databaseConfig, orderBy: this.getOrderBy(this.table), - sqlCondition: isMongo ? null : this.getChildrenSqlCondition(), - mongoCondition: isMongo ? this.getChildrenMongoCondition() : null, - engineType: isMongo ? 'docdb' : 'sqldb', + sqlCondition: this.getChildrenSqlCondition(), + // mongoCondition: isMongo ? this.getChildrenMongoCondition() : null, + engineType: isDocDb ? 'docdb' : 'sqldb', }; } diff --git a/packages/filterparser/src/datetimeParser.ts b/packages/filterparser/src/datetimeParser.ts deleted file mode 100644 index 06f501340..000000000 --- a/packages/filterparser/src/datetimeParser.ts +++ /dev/null @@ -1,308 +0,0 @@ -import P from 'parsimmon'; -import moment from 'moment'; -import type { TransformType } from 'dbgate-types'; -import { token, word, whitespace } from './common'; - -const compoudCondition = conditionType => conditions => { - if (conditions.length == 1) return conditions[0]; - return { - [conditionType]: conditions, - }; -}; - -function getTransformCondition(transform: TransformType, value) { - return { - conditionType: 'binary', - operator: '=', - left: { - exprType: 'transform', - transform, - expr: { - exprType: 'placeholder', - }, - }, - right: { - exprType: 'value', - value, - }, - }; -} - -const yearCondition = () => value => { - return getTransformCondition('YEAR', value); -}; - -const yearMonthCondition = () => value => { - const m = value.match(/(\d\d\d\d)-(\d\d?)/); - - return { - conditionType: 'and', - conditions: [getTransformCondition('YEAR', m[1]), getTransformCondition('MONTH', m[2])], - }; -}; - -const yearMonthDayCondition = () => value => { - const m = value.match(/(\d\d\d\d)-(\d\d?)-(\d\d?)/); - - return { - conditionType: 'and', - conditions: [ - getTransformCondition('YEAR', m[1]), - getTransformCondition('MONTH', m[2]), - getTransformCondition('DAY', m[3]), - ], - }; -}; - -const yearEdge = edgeFunction => value => { - return moment(new Date(parseInt(value), 0, 1)) - [edgeFunction]('year') - .format('YYYY-MM-DDTHH:mm:ss.SSS'); -}; - -const yearMonthEdge = edgeFunction => value => { - const m = value.match(/(\d\d\d\d)-(\d\d?)/); - - return moment(new Date(parseInt(m[1]), parseInt(m[2]) - 1, 1)) - [edgeFunction]('month') - .format('YYYY-MM-DDTHH:mm:ss.SSS'); -}; - -const yearMonthDayEdge = edgeFunction => value => { - const m = value.match(/(\d\d\d\d)-(\d\d?)-(\d\d?)/); - - return moment(new Date(parseInt(m[1]), parseInt(m[2]) - 1, parseInt(m[3]))) - [edgeFunction]('day') - .format('YYYY-MM-DDTHH:mm:ss.SSS'); -}; - -const yearMonthDayMinuteEdge = edgeFunction => value => { - const m = value.match(/(\d\d\d\d)-(\d\d?)-(\d\d?)\s+(\d\d?):(\d\d?)/); - const year = m[1]; - const month = m[2]; - const day = m[3]; - const hour = m[4]; - const minute = m[5]; - const dateObject = new Date(year, month - 1, day, hour, minute); - - return moment(dateObject)[edgeFunction]('minute').format('YYYY-MM-DDTHH:mm:ss.SSS'); -}; - -const yearMonthDayMinuteSecondEdge = edgeFunction => value => { - const m = value.match(/(\d\d\d\d)-(\d\d?)-(\d\d?)(T|\s+)(\d\d?):(\d\d?):(\d\d?)/); - const year = m[1]; - const month = m[2]; - const day = m[3]; - const hour = m[5]; - const minute = m[6]; - const second = m[7]; - const dateObject = new Date(year, month - 1, day, hour, minute, second); - - return moment(dateObject)[edgeFunction]('second').format('YYYY-MM-DDTHH:mm:ss.SSS'); -}; - -const createIntervalCondition = (start, end) => { - return { - conditionType: 'and', - conditions: [ - { - conditionType: 'binary', - operator: '>=', - left: { - exprType: 'placeholder', - }, - right: { - exprType: 'value', - value: start, - }, - }, - { - conditionType: 'binary', - operator: '<=', - left: { - exprType: 'placeholder', - }, - right: { - exprType: 'value', - value: end, - }, - }, - ], - }; -}; - -const createDateIntervalCondition = (start, end) => { - return createIntervalCondition(start.format('YYYY-MM-DDTHH:mm:ss.SSS'), end.format('YYYY-MM-DDTHH:mm:ss.SSS')); -}; - -const fixedMomentIntervalCondition = (intervalType, diff) => () => { - return createDateIntervalCondition( - moment().add(intervalType, diff).startOf(intervalType), - moment().add(intervalType, diff).endOf(intervalType) - ); -}; - -const yearMonthDayMinuteCondition = () => value => { - const m = value.match(/(\d\d\d\d)-(\d\d?)-(\d\d?)\s+(\d\d?):(\d\d?)/); - const year = m[1]; - const month = m[2]; - const day = m[3]; - const hour = m[4]; - const minute = m[5]; - const dateObject = new Date(year, month - 1, day, hour, minute); - - return createDateIntervalCondition(moment(dateObject).startOf('minute'), moment(dateObject).endOf('minute')); -}; - -const yearMonthDaySecondCondition = () => value => { - const m = value.match(/(\d\d\d\d)-(\d\d?)-(\d\d?)(T|\s+)(\d\d?):(\d\d?):(\d\d?)/); - const year = m[1]; - const month = m[2]; - const day = m[3]; - const hour = m[5]; - const minute = m[6]; - const second = m[7]; - const dateObject = new Date(year, month - 1, day, hour, minute, second); - - return createDateIntervalCondition(moment(dateObject).startOf('second'), moment(dateObject).endOf('second')); -}; - -const binaryCondition = operator => value => ({ - conditionType: 'binary', - operator, - left: { - exprType: 'placeholder', - }, - right: { - exprType: 'value', - value, - }, -}); - -const unaryCondition = conditionType => () => { - return { - conditionType, - expr: { - exprType: 'placeholder', - }, - }; -}; - -const sqlTemplate = templateSql => { - return { - conditionType: 'rawTemplate', - templateSql, - expr: { - exprType: 'placeholder', - }, - }; -}; - -const createParser = () => { - const langDef = { - comma: () => word(','), - - not: () => word('NOT'), - notNull: r => r.not.then(r.null).map(unaryCondition('isNotNull')), - null: () => word('NULL').map(unaryCondition('isNull')), - - sql: () => - token(P.regexp(/\{(.*?)\}/, 1)) - .map(sqlTemplate) - .desc('sql literal'), - - yearNum: () => P.regexp(/\d\d\d\d/).map(yearCondition()), - yearMonthNum: () => P.regexp(/\d\d\d\d-\d\d?/).map(yearMonthCondition()), - yearMonthDayNum: () => P.regexp(/\d\d\d\d-\d\d?-\d\d?/).map(yearMonthDayCondition()), - yearMonthDayMinute: () => P.regexp(/\d\d\d\d-\d\d?-\d\d?\s+\d\d?:\d\d?/).map(yearMonthDayMinuteCondition()), - yearMonthDaySecond: () => - P.regexp(/\d\d\d\d-\d\d?-\d\d?(\s+|T)\d\d?:\d\d?:\d\d?/).map(yearMonthDaySecondCondition()), - - yearNumStart: () => P.regexp(/\d\d\d\d/).map(yearEdge('startOf')), - yearNumEnd: () => P.regexp(/\d\d\d\d/).map(yearEdge('endOf')), - yearMonthStart: () => P.regexp(/\d\d\d\d-\d\d?/).map(yearMonthEdge('startOf')), - yearMonthEnd: () => P.regexp(/\d\d\d\d-\d\d?/).map(yearMonthEdge('endOf')), - yearMonthDayStart: () => P.regexp(/\d\d\d\d-\d\d?-\d\d?/).map(yearMonthDayEdge('startOf')), - yearMonthDayEnd: () => P.regexp(/\d\d\d\d-\d\d?-\d\d?/).map(yearMonthDayEdge('endOf')), - yearMonthDayMinuteStart: () => - P.regexp(/\d\d\d\d-\d\d?-\d\d?\s+\d\d?:\d\d?/).map(yearMonthDayMinuteEdge('startOf')), - yearMonthDayMinuteEnd: () => P.regexp(/\d\d\d\d-\d\d?-\d\d?\s+\d\d?:\d\d?/).map(yearMonthDayMinuteEdge('endOf')), - yearMonthDayMinuteSecondStart: () => - P.regexp(/\d\d\d\d-\d\d?-\d\d?(\s+|T)\d\d?:\d\d?:\d\d?/).map(yearMonthDayMinuteSecondEdge('startOf')), - yearMonthDayMinuteSecondEnd: () => - P.regexp(/\d\d\d\d-\d\d?-\d\d?(\s+|T)\d\d?:\d\d?:\d\d?/).map(yearMonthDayMinuteSecondEdge('endOf')), - - this: () => word('THIS'), - last: () => word('LAST'), - next: () => word('NEXT'), - week: () => word('WEEK'), - month: () => word('MONTH'), - year: () => word('YEAR'), - - yesterday: () => word('YESTERDAY').map(fixedMomentIntervalCondition('day', -1)), - today: () => word('TODAY').map(fixedMomentIntervalCondition('day', 0)), - tomorrow: () => word('TOMORROW').map(fixedMomentIntervalCondition('day', 1)), - - lastWeek: r => r.last.then(r.week).map(fixedMomentIntervalCondition('week', -1)), - thisWeek: r => r.this.then(r.week).map(fixedMomentIntervalCondition('week', 0)), - nextWeek: r => r.next.then(r.week).map(fixedMomentIntervalCondition('week', 1)), - - lastMonth: r => r.last.then(r.month).map(fixedMomentIntervalCondition('month', -1)), - thisMonth: r => r.this.then(r.month).map(fixedMomentIntervalCondition('month', 0)), - nextMonth: r => r.next.then(r.month).map(fixedMomentIntervalCondition('month', 1)), - - lastYear: r => r.last.then(r.year).map(fixedMomentIntervalCondition('year', -1)), - thisYear: r => r.this.then(r.year).map(fixedMomentIntervalCondition('year', 0)), - nextYear: r => r.next.then(r.year).map(fixedMomentIntervalCondition('year', 1)), - - valueStart: r => - P.alt( - r.yearMonthDayMinuteSecondStart, - r.yearMonthDayMinuteStart, - r.yearMonthDayStart, - r.yearMonthStart, - r.yearNumStart - ), - valueEnd: r => - P.alt(r.yearMonthDayMinuteSecondEnd, r.yearMonthDayMinuteEnd, r.yearMonthDayEnd, r.yearMonthEnd, r.yearNumEnd), - - le: r => word('<=').then(r.valueEnd).map(binaryCondition('<=')), - ge: r => word('>=').then(r.valueStart).map(binaryCondition('>=')), - lt: r => word('<').then(r.valueStart).map(binaryCondition('<')), - gt: r => word('>').then(r.valueEnd).map(binaryCondition('>')), - - element: r => - P.alt( - r.yearMonthDaySecond, - r.yearMonthDayMinute, - r.yearMonthDayNum, - r.yearMonthNum, - r.yearNum, - r.yesterday, - r.today, - r.tomorrow, - r.lastWeek, - r.thisWeek, - r.nextWeek, - r.lastMonth, - r.thisMonth, - r.nextMonth, - r.lastYear, - r.thisYear, - r.nextYear, - r.null, - r.notNull, - r.le, - r.lt, - r.ge, - r.gt, - r.sql - ).trim(whitespace), - factor: r => r.element.sepBy(whitespace).map(compoudCondition('$and')), - list: r => r.factor.sepBy(r.comma).map(compoudCondition('$or')), - }; - - return P.createLanguage(langDef); -}; - -export const datetimeParser = createParser(); diff --git a/packages/filterparser/src/mongoParser.ts b/packages/filterparser/src/mongoParser.ts deleted file mode 100644 index 79c67f43c..000000000 --- a/packages/filterparser/src/mongoParser.ts +++ /dev/null @@ -1,151 +0,0 @@ -import P from 'parsimmon'; -import { interpretEscapes, token, word, whitespace } from './common'; - -const operatorCondition = operator => value => ({ - __placeholder__: { - [operator]: value, - }, -}); - -const regexCondition = regexString => value => ({ - __placeholder__: { - $regex: regexString.replace('#VALUE#', value), - $options: 'i', - }, -}); - -const numberTestCondition = () => value => ({ - $or: [ - { - __placeholder__: { - $regex: `.*${value}.*`, - $options: 'i', - }, - }, - { - __placeholder__: value, - }, - ], -}); - -const idRegex = /[('"]([0-9a-f]{24})['")]/; - -const objectIdTestCondition = () => value => ({ - $or: [ - { - __placeholder__: { $oid: value.match(idRegex)[1] }, - }, - ], -}); - -const testCondition = (operator, value) => () => ({ - __placeholder__: { - [operator]: value, - }, -}); - -const multiTestCondition = condition => () => ({ - __placeholder__: condition, -}); - -const compoudCondition = conditionType => conditions => { - if (conditions.length == 1) return conditions[0]; - return { - [conditionType]: conditions, - }; -}; - -const negateCondition = condition => ({ - __placeholder__: { - $not: condition.__placeholder__, - }, -}); - -const createParser = () => { - const langDef = { - string1: () => - token(P.regexp(/"((?:\\.|.)*?)"/, 1)) - .map(interpretEscapes) - .desc('string quoted'), - - string2: () => - token(P.regexp(/'((?:\\.|.)*?)'/, 1)) - .map(interpretEscapes) - .desc('string quoted'), - - number: () => - token(P.regexp(/-?(0|[1-9][0-9]*)([.][0-9]+)?([eE][+-]?[0-9]+)?/)) - .map(Number) - .desc('number'), - - objectid: () => token(P.regexp(/ObjectId\(['"]?[0-9a-f]{24}['"]?\)/)).desc('ObjectId'), - - noQuotedString: () => P.regexp(/[^\s^,^'^"]+/).desc('string unquoted'), - - value: r => P.alt(r.objectid, r.string1, r.string2, r.number, r.noQuotedString), - valueTestObjectId: r => r.objectid.map(objectIdTestCondition()), - valueTestNum: r => r.number.map(numberTestCondition()), - valueTest: r => r.value.map(regexCondition('.*#VALUE#.*')), - - comma: () => word(','), - not: () => word('NOT'), - empty: () => word('EMPTY'), - array: () => word('ARRAY'), - notExists: r => r.not.then(r.exists).map(testCondition('$exists', false)), - notEmptyArray: r => - r.not - .then(r.empty) - .then(r.array) - .map(multiTestCondition({ $exists: true, $type: 'array', $ne: [] })), - emptyArray: r => r.empty.then(r.array).map(multiTestCondition({ $exists: true, $eq: [] })), - exists: () => word('EXISTS').map(testCondition('$exists', true)), - true: () => word('TRUE').map(testCondition('$eq', true)), - false: () => word('FALSE').map(testCondition('$eq', false)), - - eq: r => word('=').then(r.value).map(operatorCondition('$eq')), - ne: r => word('!=').then(r.value).map(operatorCondition('$ne')), - ne2: r => word('<>').then(r.value).map(operatorCondition('$ne')), - lt: r => word('<').then(r.value).map(operatorCondition('$lt')), - gt: r => word('>').then(r.value).map(operatorCondition('$gt')), - le: r => word('<=').then(r.value).map(operatorCondition('$lte')), - ge: r => word('>=').then(r.value).map(operatorCondition('$gte')), - startsWith: r => word('^').then(r.value).map(regexCondition('#VALUE#.*')), - endsWith: r => word('$').then(r.value).map(regexCondition('.*#VALUE#')), - contains: r => word('+').then(r.value).map(regexCondition('.*#VALUE#.*')), - startsWithNot: r => word('!^').then(r.value).map(regexCondition('#VALUE#.*')).map(negateCondition), - endsWithNot: r => word('!$').then(r.value).map(regexCondition('.*#VALUE#')).map(negateCondition), - containsNot: r => word('~').then(r.value).map(regexCondition('.*#VALUE#.*')).map(negateCondition), - - element: r => - P.alt( - r.exists, - r.notExists, - r.true, - r.false, - r.eq, - r.ne, - r.ne2, - r.lt, - r.gt, - r.le, - r.ge, - r.notEmptyArray, - r.emptyArray, - r.startsWith, - r.endsWith, - r.contains, - r.startsWithNot, - r.endsWithNot, - r.containsNot, - r.valueTestObjectId, - r.valueTestNum, - r.valueTest - ).trim(whitespace), - factor: r => r.element.sepBy(whitespace).map(compoudCondition('$and')), - list: r => r.factor.sepBy(r.comma).map(compoudCondition('$or')), - }; - - return P.createLanguage(langDef); -}; - -export const mongoParser = createParser(); diff --git a/packages/filterparser/src/parseFilter.ts b/packages/filterparser/src/parseFilter.ts index 9850462ce..7625ce10d 100644 --- a/packages/filterparser/src/parseFilter.ts +++ b/packages/filterparser/src/parseFilter.ts @@ -1,10 +1,9 @@ import P from 'parsimmon'; +import moment from 'moment'; import { Condition } from 'dbgate-sqltree'; import { interpretEscapes, token, word, whitespace } from './common'; -import { mongoParser } from './mongoParser'; -import { datetimeParser } from './datetimeParser'; import { hexStringToArray } from 'dbgate-tools'; -import { FilterBehaviour } from 'dbgate-types'; +import { FilterBehaviour, TransformType } from 'dbgate-types'; const binaryCondition = operator => value => ({ conditionType: 'binary', @@ -67,6 +66,57 @@ const negateCondition = condition => { }; }; +const numberTestCondition = () => value => { + return { + conditionType: 'or', + conditions: [ + { + conditionType: 'like', + left: { + exprType: 'placeholder', + }, + right: { + exprType: 'value', + value: `.*${value}.*`, + }, + }, + { + conditionType: 'binary', + operator: '=', + left: { + exprType: 'placeholder', + }, + right: { + exprType: 'value', + value, + }, + }, + ], + }; +}; + +const idRegex = /[('"]([0-9a-f]{24})['")]/; + +const objectIdTestCondition = () => value => ({ + conditionType: 'binary', + operator: '=', + left: { + exprType: 'placeholder', + }, + right: { + exprType: 'value', + value: { $oid: value.match(idRegex)[1] }, + }, +}); + +const specificPredicateCondition = predicate => () => ({ + conditionType: 'specificPredicate', + predicate, + expr: { + exprType: 'placeholder', + }, +}); + const sqlTemplate = templateSql => { return { conditionType: 'rawTemplate', @@ -77,6 +127,163 @@ const sqlTemplate = templateSql => { }; }; +function getTransformCondition(transform: TransformType, value) { + return { + conditionType: 'binary', + operator: '=', + left: { + exprType: 'transform', + transform, + expr: { + exprType: 'placeholder', + }, + }, + right: { + exprType: 'value', + value, + }, + }; +} + +const yearCondition = () => value => { + return getTransformCondition('YEAR', value); +}; + +const yearMonthCondition = () => value => { + const m = value.match(/(\d\d\d\d)-(\d\d?)/); + + return { + conditionType: 'and', + conditions: [getTransformCondition('YEAR', m[1]), getTransformCondition('MONTH', m[2])], + }; +}; + +const yearMonthDayCondition = () => value => { + const m = value.match(/(\d\d\d\d)-(\d\d?)-(\d\d?)/); + + return { + conditionType: 'and', + conditions: [ + getTransformCondition('YEAR', m[1]), + getTransformCondition('MONTH', m[2]), + getTransformCondition('DAY', m[3]), + ], + }; +}; + +const yearEdge = edgeFunction => value => { + return moment(new Date(parseInt(value), 0, 1)) + [edgeFunction]('year') + .format('YYYY-MM-DDTHH:mm:ss.SSS'); +}; + +const yearMonthEdge = edgeFunction => value => { + const m = value.match(/(\d\d\d\d)-(\d\d?)/); + + return moment(new Date(parseInt(m[1]), parseInt(m[2]) - 1, 1)) + [edgeFunction]('month') + .format('YYYY-MM-DDTHH:mm:ss.SSS'); +}; + +const yearMonthDayEdge = edgeFunction => value => { + const m = value.match(/(\d\d\d\d)-(\d\d?)-(\d\d?)/); + + return moment(new Date(parseInt(m[1]), parseInt(m[2]) - 1, parseInt(m[3]))) + [edgeFunction]('day') + .format('YYYY-MM-DDTHH:mm:ss.SSS'); +}; + +const yearMonthDayMinuteEdge = edgeFunction => value => { + const m = value.match(/(\d\d\d\d)-(\d\d?)-(\d\d?)\s+(\d\d?):(\d\d?)/); + const year = m[1]; + const month = m[2]; + const day = m[3]; + const hour = m[4]; + const minute = m[5]; + const dateObject = new Date(year, month - 1, day, hour, minute); + + return moment(dateObject)[edgeFunction]('minute').format('YYYY-MM-DDTHH:mm:ss.SSS'); +}; + +const yearMonthDayMinuteSecondEdge = edgeFunction => value => { + const m = value.match(/(\d\d\d\d)-(\d\d?)-(\d\d?)(T|\s+)(\d\d?):(\d\d?):(\d\d?)/); + const year = m[1]; + const month = m[2]; + const day = m[3]; + const hour = m[5]; + const minute = m[6]; + const second = m[7]; + const dateObject = new Date(year, month - 1, day, hour, minute, second); + + return moment(dateObject)[edgeFunction]('second').format('YYYY-MM-DDTHH:mm:ss.SSS'); +}; + +const createIntervalCondition = (start, end) => { + return { + conditionType: 'and', + conditions: [ + { + conditionType: 'binary', + operator: '>=', + left: { + exprType: 'placeholder', + }, + right: { + exprType: 'value', + value: start, + }, + }, + { + conditionType: 'binary', + operator: '<=', + left: { + exprType: 'placeholder', + }, + right: { + exprType: 'value', + value: end, + }, + }, + ], + }; +}; + +const createDateIntervalCondition = (start, end) => { + return createIntervalCondition(start.format('YYYY-MM-DDTHH:mm:ss.SSS'), end.format('YYYY-MM-DDTHH:mm:ss.SSS')); +}; + +const fixedMomentIntervalCondition = (intervalType, diff) => () => { + return createDateIntervalCondition( + moment().add(intervalType, diff).startOf(intervalType), + moment().add(intervalType, diff).endOf(intervalType) + ); +}; + +const yearMonthDayMinuteCondition = () => value => { + const m = value.match(/(\d\d\d\d)-(\d\d?)-(\d\d?)\s+(\d\d?):(\d\d?)/); + const year = m[1]; + const month = m[2]; + const day = m[3]; + const hour = m[4]; + const minute = m[5]; + const dateObject = new Date(year, month - 1, day, hour, minute); + + return createDateIntervalCondition(moment(dateObject).startOf('minute'), moment(dateObject).endOf('minute')); +}; + +const yearMonthDaySecondCondition = () => value => { + const m = value.match(/(\d\d\d\d)-(\d\d?)-(\d\d?)(T|\s+)(\d\d?):(\d\d?):(\d\d?)/); + const year = m[1]; + const month = m[2]; + const day = m[3]; + const hour = m[5]; + const minute = m[6]; + const second = m[7]; + const dateObject = new Date(year, month - 1, day, hour, minute, second); + + return createDateIntervalCondition(moment(dateObject).startOf('second'), moment(dateObject).endOf('second')); +}; + const createParser = (filterBehaviour: FilterBehaviour) => { const langDef = { string1: () => @@ -104,6 +311,8 @@ const createParser = (filterBehaviour: FilterBehaviour) => { .map(Number) .desc('number'), + objectid: () => token(P.regexp(/ObjectId\(['"]?[0-9a-f]{24}['"]?\)/)).desc('ObjectId'), + hexstring: () => token(P.regexp(/0x(([0-9a-fA-F][0-9a-fA-F])+)/, 1)) .map(x => ({ @@ -123,13 +332,82 @@ const createParser = (filterBehaviour: FilterBehaviour) => { valueTestEq: r => r.value.map(binaryCondition('=')), hexTestEq: r => r.hexstring.map(binaryCondition('=')), valueTestStr: r => r.value.map(likeCondition('like', '%#VALUE#%')), + valueTestNum: r => r.number.map(numberTestCondition()), + valueTestObjectId: r => r.objectid.map(objectIdTestCondition()), + + notExists: r => r.not.then(r.exists).map(specificPredicateCondition('notExists')), + notEmptyArray: r => r.not.then(r.empty).then(r.array).map(specificPredicateCondition('notEmptyArray')), + emptyArray: r => r.empty.then(r.array).map(specificPredicateCondition('emptyArray')), + exists: () => word('EXISTS').map(specificPredicateCondition('exists')), + + this: () => word('THIS'), + last: () => word('LAST'), + next: () => word('NEXT'), + week: () => word('WEEK'), + month: () => word('MONTH'), + year: () => word('YEAR'), + + yesterday: () => word('YESTERDAY').map(fixedMomentIntervalCondition('day', -1)), + today: () => word('TODAY').map(fixedMomentIntervalCondition('day', 0)), + tomorrow: () => word('TOMORROW').map(fixedMomentIntervalCondition('day', 1)), + + lastWeek: r => r.last.then(r.week).map(fixedMomentIntervalCondition('week', -1)), + thisWeek: r => r.this.then(r.week).map(fixedMomentIntervalCondition('week', 0)), + nextWeek: r => r.next.then(r.week).map(fixedMomentIntervalCondition('week', 1)), + + lastMonth: r => r.last.then(r.month).map(fixedMomentIntervalCondition('month', -1)), + thisMonth: r => r.this.then(r.month).map(fixedMomentIntervalCondition('month', 0)), + nextMonth: r => r.next.then(r.month).map(fixedMomentIntervalCondition('month', 1)), + + lastYear: r => r.last.then(r.year).map(fixedMomentIntervalCondition('year', -1)), + thisYear: r => r.this.then(r.year).map(fixedMomentIntervalCondition('year', 0)), + nextYear: r => r.next.then(r.year).map(fixedMomentIntervalCondition('year', 1)), + + dateValueStart: r => + P.alt( + r.yearMonthDayMinuteSecondStart, + r.yearMonthDayMinuteStart, + r.yearMonthDayStart, + r.yearMonthStart, + r.yearNumStart + ), + dateValueEnd: r => + P.alt(r.yearMonthDayMinuteSecondEnd, r.yearMonthDayMinuteEnd, r.yearMonthDayEnd, r.yearMonthEnd, r.yearNumEnd), + + dateLe: r => word('<=').then(r.dateValueEnd).map(binaryCondition('<=')), + dateGe: r => word('>=').then(r.dateValueStart).map(binaryCondition('>=')), + dateLt: r => word('<').then(r.dateValueStart).map(binaryCondition('<')), + dateGt: r => word('>').then(r.dateValueEnd).map(binaryCondition('>')), + + yearNum: () => P.regexp(/\d\d\d\d/).map(yearCondition()), + yearMonthNum: () => P.regexp(/\d\d\d\d-\d\d?/).map(yearMonthCondition()), + yearMonthDayNum: () => P.regexp(/\d\d\d\d-\d\d?-\d\d?/).map(yearMonthDayCondition()), + yearMonthDayMinute: () => P.regexp(/\d\d\d\d-\d\d?-\d\d?\s+\d\d?:\d\d?/).map(yearMonthDayMinuteCondition()), + yearMonthDaySecond: () => + P.regexp(/\d\d\d\d-\d\d?-\d\d?(\s+|T)\d\d?:\d\d?:\d\d?/).map(yearMonthDaySecondCondition()), + + yearNumStart: () => P.regexp(/\d\d\d\d/).map(yearEdge('startOf')), + yearNumEnd: () => P.regexp(/\d\d\d\d/).map(yearEdge('endOf')), + yearMonthStart: () => P.regexp(/\d\d\d\d-\d\d?/).map(yearMonthEdge('startOf')), + yearMonthEnd: () => P.regexp(/\d\d\d\d-\d\d?/).map(yearMonthEdge('endOf')), + yearMonthDayStart: () => P.regexp(/\d\d\d\d-\d\d?-\d\d?/).map(yearMonthDayEdge('startOf')), + yearMonthDayEnd: () => P.regexp(/\d\d\d\d-\d\d?-\d\d?/).map(yearMonthDayEdge('endOf')), + yearMonthDayMinuteStart: () => + P.regexp(/\d\d\d\d-\d\d?-\d\d?\s+\d\d?:\d\d?/).map(yearMonthDayMinuteEdge('startOf')), + yearMonthDayMinuteEnd: () => P.regexp(/\d\d\d\d-\d\d?-\d\d?\s+\d\d?:\d\d?/).map(yearMonthDayMinuteEdge('endOf')), + yearMonthDayMinuteSecondStart: () => + P.regexp(/\d\d\d\d-\d\d?-\d\d?(\s+|T)\d\d?:\d\d?:\d\d?/).map(yearMonthDayMinuteSecondEdge('startOf')), + yearMonthDayMinuteSecondEnd: () => + P.regexp(/\d\d\d\d-\d\d?-\d\d?(\s+|T)\d\d?:\d\d?:\d\d?/).map(yearMonthDayMinuteSecondEdge('endOf')), comma: () => word(','), not: () => word('NOT'), + empty: () => word('EMPTY'), + array: () => word('ARRAY'), notNull: r => r.not.then(r.null).map(unaryCondition('isNotNull')), null: () => word('NULL').map(unaryCondition('isNull')), - empty: () => word('EMPTY').map(unaryCondition('isEmpty')), - notEmpty: r => r.not.then(r.empty).map(unaryCondition('isNotEmpty')), + isEmpty: r => r.empty.map(unaryCondition('isEmpty')), + isNotEmpty: r => r.not.then(r.empty).map(unaryCondition('isNotEmpty')), true: () => P.regexp(/true/i).map(binaryFixedValueCondition('1')), false: () => P.regexp(/false/i).map(binaryFixedValueCondition('0')), trueNum: () => word('1').map(binaryFixedValueCondition('1')), @@ -155,6 +433,7 @@ const createParser = (filterBehaviour: FilterBehaviour) => { }; const allowedValues = []; // 'string1', 'string2', 'number', 'noQuotedString']; + if (filterBehaviour.allowStringToken) { allowedValues.push('string1', 'string2', 'noQuotedString'); } @@ -164,6 +443,38 @@ const createParser = (filterBehaviour: FilterBehaviour) => { const allowedElements = []; + if (filterBehaviour.supportDatetimeComparison) { + allowedElements.push('yearMonthDaySecond', 'yearMonthDayMinute', 'yearMonthDayNum', 'yearMonthNum', 'yearNum'); + } + + if (filterBehaviour.supportDatetimeSymbols) { + allowedElements.push( + 'today', + 'tomorrow', + 'lastWeek', + 'thisWeek', + 'nextWeek', + 'lastMonth', + 'thisMonth', + 'nextMonth', + 'lastYear', + 'thisYear', + 'nextYear' + ); + } + + if (filterBehaviour.supportDatetimeComparison) { + allowedElements.push('dateLe', 'dateGe', 'dateLt', 'dateGt'); + } + + if (filterBehaviour.supportExistsTesting) { + allowedElements.push('exists', 'notExists'); + } + + if (filterBehaviour.supportArrayTesting) { + allowedElements.push('emptyArray', 'notEmptyArray'); + } + if (filterBehaviour.supportNullTesting) { allowedElements.push('null', 'notNull'); } @@ -181,7 +492,7 @@ const createParser = (filterBehaviour: FilterBehaviour) => { } if (filterBehaviour.supportEmpty) { - allowedElements.push('empty', 'notEmpty'); + allowedElements.push('isEmpty', 'isNotEmpty'); } if (filterBehaviour.allowHexString) { @@ -199,6 +510,14 @@ const createParser = (filterBehaviour: FilterBehaviour) => { } } + if (filterBehaviour.allowNumberDualTesting) { + allowedElements.push('valueTestNum'); + } + + if (filterBehaviour.allowObjectIdTesting) { + allowedElements.push('valueTestObjectId'); + } + // must be last if (filterBehaviour.allowStringToken) { allowedElements.push('valueTestStr'); @@ -212,12 +531,6 @@ const createParser = (filterBehaviour: FilterBehaviour) => { const cachedFilters: { [key: string]: P.Language } = {}; function getParser(filterBehaviour: FilterBehaviour) { - if (filterBehaviour.compilerType == 'mongoCondition') { - return mongoParser; - } - if (filterBehaviour.compilerType == 'datetime') { - return datetimeParser; - } const key = JSON.stringify(filterBehaviour); if (!cachedFilters[key]) { cachedFilters[key] = createParser(filterBehaviour); diff --git a/packages/sqltree/src/types.ts b/packages/sqltree/src/types.ts index 5a8390e11..db2b609d9 100644 --- a/packages/sqltree/src/types.ts +++ b/packages/sqltree/src/types.ts @@ -79,6 +79,11 @@ export interface TestCondition extends UnaryCondition { conditionType: 'isNull' | 'isNotNull' | 'isEmpty' | 'isNotEmpty'; } +export interface SpecificPredicateCondition extends UnaryCondition { + conditionType: 'specificPredicate'; + predicate: string; +} + export interface CompoudCondition { conditionType: 'and' | 'or'; conditions: Condition[]; @@ -135,7 +140,8 @@ export type Condition = | InCondition | NotInCondition | RawTemplateCondition - | AnyColumnPassEvalOnlyCondition; + | AnyColumnPassEvalOnlyCondition + | SpecificPredicateCondition; export interface Source { name?: NamedObjectInfo; diff --git a/packages/tools/src/driverBase.ts b/packages/tools/src/driverBase.ts index 1517269c6..d76318b9d 100644 --- a/packages/tools/src/driverBase.ts +++ b/packages/tools/src/driverBase.ts @@ -154,4 +154,11 @@ export const driverBase = { getFilterBehaviour(dataType: string, standardFilterBehaviours) { return detectSqlFilterBehaviour(dataType); }, + + getCollectionExportQueryScript(collection: string, condition: any, sort: any) { + return null; + }, + getCollectionExportQueryJson(collection: string, condition: any, sort: any) { + return null; + }, }; diff --git a/packages/tools/src/filterBehaviours.ts b/packages/tools/src/filterBehaviours.ts index 1a1e07100..f600c2e3c 100644 --- a/packages/tools/src/filterBehaviours.ts +++ b/packages/tools/src/filterBehaviours.ts @@ -1,7 +1,6 @@ import { FilterBehaviour } from 'dbgate-types'; export const numberFilterBehaviour: FilterBehaviour = { - compilerType: 'sqlTree', supportEquals: true, supportNumberLikeComparison: true, supportNullTesting: true, @@ -11,7 +10,6 @@ export const numberFilterBehaviour: FilterBehaviour = { }; export const stringFilterBehaviour: FilterBehaviour = { - compilerType: 'sqlTree', supportEquals: true, supportStringInclusion: true, supportEmpty: true, @@ -24,14 +22,12 @@ export const stringFilterBehaviour: FilterBehaviour = { }; export const logicalFilterBehaviour: FilterBehaviour = { - compilerType: 'sqlTree', supportBooleanValues: true, supportNullTesting: true, supportSqlCondition: true, }; export const datetimeFilterBehaviour: FilterBehaviour = { - compilerType: 'datetime', supportNullTesting: true, supportSqlCondition: true, supportDatetimeSymbols: true, @@ -39,17 +35,19 @@ export const datetimeFilterBehaviour: FilterBehaviour = { }; export const mongoFilterBehaviour: FilterBehaviour = { - compilerType: 'mongoCondition', supportEquals: true, supportArrayTesting: true, supportNumberLikeComparison: true, supportStringInclusion: true, supportBooleanValues: true, supportExistsTesting: true, + + allowStringToken: true, + allowNumberDualTesting: true, + allowObjectIdTesting: true, }; export const evalFilterBehaviour: FilterBehaviour = { - compilerType: 'sqlTree', supportEquals: true, supportStringInclusion: true, supportEmpty: true, diff --git a/packages/types/engines.d.ts b/packages/types/engines.d.ts index 5523cea3f..cf8c1d2b4 100644 --- a/packages/types/engines.d.ts +++ b/packages/types/engines.d.ts @@ -41,6 +41,8 @@ export interface ReadCollectionOptions { countDocuments?: boolean; skip?: number; limit?: number; + condition?: any; + aggregate?: CollectionAggregateDefinition; } export interface NewObjectTemplate { @@ -72,6 +74,17 @@ export interface ServerSummary { databases: ServerSummaryDatabase[]; } +export type CollectionAggregateFunction = 'count' | 'sum' | 'avg' | 'min' | 'max'; +export interface CollectionAggregateDefinition { + condition: any; // SQL tree condition + groupByColumns: string[]; + aggregateColumns: { + alias: string; + aggregateFunction: CollectionAggregateFunction; + columnArgument?: string; + }[]; +} + export interface FilterBehaviourProvider { getFilterBehaviour(dataType: string, standardFilterBehaviours: { [id: string]: FilterBehaviour }): FilterBehaviour; } @@ -158,6 +171,8 @@ export interface EngineDriver extends FilterBehaviourProvider { getRedirectAuthUrl(connection, options): Promise<{ url: string; sid: string }>; getAuthTokenFromCode(connection, options): Promise; getAccessTokenFromAuth(connection, req): Promise; + getCollectionExportQueryScript(collection: string, condition: any, sort: any): string; + getCollectionExportQueryJson(collection: string, condition: any, sort: any): {}; analyserClass?: any; dumperClass?: any; diff --git a/packages/types/filter-type.d.ts b/packages/types/filter-type.d.ts index 2fa506479..d3fd5dde7 100644 --- a/packages/types/filter-type.d.ts +++ b/packages/types/filter-type.d.ts @@ -1,8 +1,4 @@ -export type FilterParserCompilerType = 'sqlTree' | 'mongoCondition' | 'datetime'; - export interface FilterBehaviour { - compilerType: FilterParserCompilerType; - supportEquals?: boolean; supportStringInclusion?: boolean; supportEmpty?: boolean; @@ -18,4 +14,6 @@ export interface FilterBehaviour { allowStringToken?: boolean; allowNumberToken?: boolean; allowHexString?: boolean; + allowNumberDualTesting?: boolean; + allowObjectIdTesting?: boolean; } diff --git a/packages/web/src/datagrid/CollectionDataGridCore.svelte b/packages/web/src/datagrid/CollectionDataGridCore.svelte index a4e72e838..85ec10afd 100644 --- a/packages/web/src/datagrid/CollectionDataGridCore.svelte +++ b/packages/web/src/datagrid/CollectionDataGridCore.svelte @@ -19,7 +19,7 @@ onClick: () => getCurrentEditor().exportGrid(), }); - function buildGridMongoCondition(props) { + function buildConditionForGrid(props) { const filters = props?.display?.config?.filters; const filterBehaviour = props?.display?.driver?.getFilterBehaviour(null, standardFilterBehaviours) ?? mongoFilterBehaviour; @@ -33,11 +33,18 @@ const ast = parseFilter(filters[uniqueName], filterBehaviour); // console.log('AST', ast); const cond = _.cloneDeepWith(ast, expr => { - if (expr.__placeholder__) { + if (expr.exprType == 'placeholder') { return { - [uniqueName]: expr.__placeholder__, + exprType: 'column', + columnName: uniqueName, }; } + + // if (expr.__placeholder__) { + // return { + // [uniqueName]: expr.__placeholder__, + // }; + // } }); conditions.push(cond); } catch (err) { @@ -47,7 +54,8 @@ return conditions.length > 0 ? { - $and: conditions, + conditionType: 'and', + conditions, } : undefined; } @@ -75,7 +83,7 @@ pureName: props.pureName, limit, skip: offset, - condition: buildGridMongoCondition(props), + condition: buildConditionForGrid(props), sort: buildMongoSort(props), }, }); @@ -100,7 +108,7 @@ options: { pureName: props.pureName, countDocuments: true, - condition: buildGridMongoCondition(props), + condition: buildConditionForGrid(props), }, }); @@ -164,17 +172,27 @@ // $: if (onChangeGrider) onChangeGrider(grider); function getExportQuery() { - return `db.collection('${pureName}') - .find(${JSON.stringify(buildGridMongoCondition($$props) || {})}) - .sort(${JSON.stringify(buildMongoSort($$props) || {})})`; + return display?.driver?.getCollectionExportQueryScript?.( + pureName, + buildConditionForGrid($$props), + buildMongoSort($$props) + ); + // return `db.collection('${pureName}') + // .find(${JSON.stringify(buildConditionForGrid($$props) || {})}) + // .sort(${JSON.stringify(buildMongoSort($$props) || {})})`; } function getExportQueryJson() { - return { - collection: pureName, - condition: buildGridMongoCondition($$props) || {}, - sort: buildMongoSort($$props) || {}, - }; + return display?.driver?.getCollectionExportQueryJson?.( + pureName, + buildConditionForGrid($$props), + buildMongoSort($$props) + ); + // return { + // collection: pureName, + // condition: buildConditionForGrid($$props) || {}, + // sort: buildMongoSort($$props) || {}, + // }; } export async function exportGrid() { diff --git a/plugins/dbgate-plugin-mongo/package.json b/plugins/dbgate-plugin-mongo/package.json index d56803534..1a0b80879 100644 --- a/plugins/dbgate-plugin-mongo/package.json +++ b/plugins/dbgate-plugin-mongo/package.json @@ -33,6 +33,7 @@ "devDependencies": { "dbgate-plugin-tools": "^1.0.7", "dbgate-query-splitter": "^4.10.1", + "lodash": "^4.17.21", "webpack": "^5.91.0", "webpack-cli": "^5.1.4", "dbgate-tools": "^5.0.0-alpha.1", diff --git a/plugins/dbgate-plugin-mongo/src/backend/driver.js b/plugins/dbgate-plugin-mongo/src/backend/driver.js index 64762e051..baa963fe3 100644 --- a/plugins/dbgate-plugin-mongo/src/backend/driver.js +++ b/plugins/dbgate-plugin-mongo/src/backend/driver.js @@ -7,6 +7,7 @@ const MongoClient = require('mongodb').MongoClient; const ObjectId = require('mongodb').ObjectId; const AbstractCursor = require('mongodb').AbstractCursor; const createBulkInsertStream = require('./createBulkInsertStream'); +const { convertToMongoCondition, convertToMongoAggregate } = require('../frontend/convertToMongoCondition'); function transformMongoData(row) { return _.cloneDeepWith(row, (x) => { @@ -270,17 +271,21 @@ const driver = { }, async readCollection(pool, options) { try { + const mongoCondition = convertToMongoCondition(options.condition); + // console.log('******************* mongoCondition *****************'); + // console.log(JSON.stringify(mongoCondition, undefined, 2)); + const collection = pool.__getDatabase().collection(options.pureName); if (options.countDocuments) { - const count = await collection.countDocuments(convertObjectId(options.condition) || {}); + const count = await collection.countDocuments(convertObjectId(mongoCondition) || {}); return { count }; } else if (options.aggregate) { - let cursor = await collection.aggregate(convertObjectId(options.aggregate)); + let cursor = await collection.aggregate(convertObjectId(convertToMongoAggregate(options.aggregate))); const rows = await cursor.toArray(); return { rows: rows.map(transformMongoData) }; } else { // console.log('options.condition', JSON.stringify(options.condition, undefined, 2)); - let cursor = await collection.find(convertObjectId(options.condition) || {}); + let cursor = await collection.find(convertObjectId(mongoCondition) || {}); if (options.sort) cursor = cursor.sort(options.sort); if (options.skip) cursor = cursor.skip(options.skip); if (options.limit) cursor = cursor.limit(options.limit); diff --git a/plugins/dbgate-plugin-mongo/src/frontend/convertToMongoCondition.js b/plugins/dbgate-plugin-mongo/src/frontend/convertToMongoCondition.js new file mode 100644 index 000000000..66fbe8e3c --- /dev/null +++ b/plugins/dbgate-plugin-mongo/src/frontend/convertToMongoCondition.js @@ -0,0 +1,177 @@ +const _zipObject = require('lodash/zipObject'); + +function convertLeftOperandToMongoColumn(left) { + if (left.exprType == 'placeholder') return '__placeholder__'; + if (left.exprType == 'column') return left.columnName; + throw new Error(`Unknown left operand type ${left.exprType}`); +} + +function convertRightOperandToMongoValue(right) { + if (right.exprType == 'value') return right.value; + throw new Error(`Unknown right operand type ${right.exprType}`); +} + +function convertToMongoCondition(filter) { + if (!filter) { + return null; + } + switch (filter.conditionType) { + case 'and': + return { + $and: filter.conditions.map((x) => convertToMongoCondition(x)), + }; + case 'or': + return { + $or: filter.conditions.map((x) => convertToMongoCondition(x)), + }; + case 'binary': + switch (filter.operator) { + case '=': + return { + [convertLeftOperandToMongoColumn(filter.left)]: { + $eq: convertRightOperandToMongoValue(filter.right), + }, + }; + case '!=': + case '<>': + return { + [convertLeftOperandToMongoColumn(filter.left)]: { + $ne: convertRightOperandToMongoValue(filter.right), + }, + }; + case '<': + return { + [convertLeftOperandToMongoColumn(filter.left)]: { + $lt: convertRightOperandToMongoValue(filter.right), + }, + }; + case '<=': + return { + [convertLeftOperandToMongoColumn(filter.left)]: { + $lte: convertRightOperandToMongoValue(filter.right), + }, + }; + case '>': + return { + [convertLeftOperandToMongoColumn(filter.left)]: { + $gt: convertRightOperandToMongoValue(filter.right), + }, + }; + case '>=': + return { + [convertLeftOperandToMongoColumn(filter.left)]: { + $gte: convertRightOperandToMongoValue(filter.right), + }, + }; + } + break; + + case 'isNull': + return { + [convertLeftOperandToMongoColumn(filter.expr)]: { + $exists: false, + }, + }; + + case 'isNotNull': + return { + [convertLeftOperandToMongoColumn(filter.expr)]: { + $exists: true, + }, + }; + + case 'not': + return { + $not: convertToMongoCondition(filter.condition), + }; + case 'like': + return { + [convertLeftOperandToMongoColumn(filter.left)]: { + $regex: `${convertRightOperandToMongoValue(filter.right)}`.replace(/%/g, '.*'), + $options: 'i', + }, + }; + + case 'specificPredicate': + switch (filter.predicate) { + case 'exists': + return { + [convertLeftOperandToMongoColumn(filter.expr)]: { + $exists: true, + }, + }; + case 'notExists': + return { + [convertLeftOperandToMongoColumn(filter.expr)]: { + $exists: false, + }, + }; + case 'emptyArray': + return { + [convertLeftOperandToMongoColumn(filter.expr)]: { + $exists: true, + $eq: [], + }, + }; + case 'notEmptyArray': + return { + [convertLeftOperandToMongoColumn(filter.expr)]: { + $exists: true, + $type: 'array', + $ne: [], + }, + }; + } + + case 'in': + return { + [convertLeftOperandToMongoColumn(filter.expr)]: { + $in: filter.values, + }, + }; + + default: + throw new Error(`Unknown condition type ${filter.conditionType}`); + } +} + +function convertToMongoAggregateFunction(aggregate) { + switch (aggregate.aggregateFunction) { + case 'count': + return { $sum: 1 }; + case 'sum': + return { $sum: `$${aggregate.columnArgument}` }; + case 'avg': + return { $avg: `$${aggregate.columnArgument}` }; + case 'min': + return { $min: `$${aggregate.columnArgument}` }; + case 'max': + return { $max: `$${aggregate.columnArgument}` }; + default: + throw new Error(`Unknown aggregate function ${aggregate.aggregateFunction}`); + } +} + +function convertToMongoAggregate(collectionAggregate) { + return [ + { $match: convertToMongoCondition(collectionAggregate.condition) }, + { + $group: { + _id: _zipObject( + collectionAggregate.groupByColumns, + collectionAggregate.groupByColumns.map((col) => '$' + col) + ), + ..._zipObject( + collectionAggregate.aggregateColumns.map((col) => col.alias), + collectionAggregate.aggregateColumns.map((col) => convertToMongoAggregateFunction(col)) + ), + count: { $sum: 1 }, + }, + }, + ]; +} + +module.exports = { + convertToMongoCondition, + convertToMongoAggregate, +}; diff --git a/plugins/dbgate-plugin-mongo/src/frontend/driver.js b/plugins/dbgate-plugin-mongo/src/frontend/driver.js index 4945c19e3..897a14ab6 100644 --- a/plugins/dbgate-plugin-mongo/src/frontend/driver.js +++ b/plugins/dbgate-plugin-mongo/src/frontend/driver.js @@ -1,4 +1,5 @@ const { driverBase } = global.DBGATE_TOOLS; +const { convertToMongoCondition } = require('./convertToMongoCondition'); const Dumper = require('./Dumper'); const { mongoSplitterOptions } = require('dbgate-query-splitter/lib/options'); @@ -97,6 +98,19 @@ const driver = { getFilterBehaviour(dataType, standardFilterBehaviours) { return standardFilterBehaviours.mongoFilterBehaviour; }, + + getCollectionExportQueryScript(collection, condition, sort) { + return `db.collection('${collection}') + .find(${JSON.stringify(convertToMongoCondition(condition || {}))}) + .sort(${JSON.stringify(sort || {})})`; + }, + getCollectionExportQueryJson(collection, condition, sort) { + return { + collection, + condition: convertToMongoCondition(condition || {}), + sort: sort || {}, + }; + }, }; module.exports = driver; diff --git a/yarn.lock b/yarn.lock index c0150cc76..817e109d5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -31,7 +31,7 @@ dependencies: tslib "^2.6.2" -"@azure/core-auth@^1.3.0", "@azure/core-auth@^1.4.0", "@azure/core-auth@^1.5.0", "@azure/core-auth@^1.7.1": +"@azure/core-auth@^1.3.0", "@azure/core-auth@^1.4.0", "@azure/core-auth@^1.5.0": version "1.7.2" resolved "https://registry.yarnpkg.com/@azure/core-auth/-/core-auth-1.7.2.tgz#558b7cb7dd12b00beec07ae5df5907d74df1ebd9" integrity sha512-Igm/S3fDYmnMq1uKS38Ae1/m37B3zigdlZw+kocwEhh5GjyKjPrXKO2J6rzpC1wAxrNil/jX9BJRqBshyjnF3g== @@ -93,21 +93,7 @@ https-proxy-agent "^7.0.0" tslib "^2.6.2" -"@azure/core-rest-pipeline@^1.15.1": - version "1.16.3" - resolved "https://registry.yarnpkg.com/@azure/core-rest-pipeline/-/core-rest-pipeline-1.16.3.tgz#bde3bc3ebad7f885ddd9de6af5e5a8fc254b287e" - integrity sha512-VxLk4AHLyqcHsfKe4MZ6IQ+D+ShuByy+RfStKfSjxJoL3WBWq17VNmrz8aT8etKzqc2nAeIyLxScjpzsS4fz8w== - dependencies: - "@azure/abort-controller" "^2.0.0" - "@azure/core-auth" "^1.4.0" - "@azure/core-tracing" "^1.0.1" - "@azure/core-util" "^1.9.0" - "@azure/logger" "^1.0.0" - http-proxy-agent "^7.0.0" - https-proxy-agent "^7.0.0" - tslib "^2.6.2" - -"@azure/core-tracing@^1.0.0", "@azure/core-tracing@^1.0.1", "@azure/core-tracing@^1.1.1": +"@azure/core-tracing@^1.0.0", "@azure/core-tracing@^1.0.1": version "1.1.2" resolved "https://registry.yarnpkg.com/@azure/core-tracing/-/core-tracing-1.1.2.tgz#065dab4e093fb61899988a1cdbc827d9ad90b4ee" integrity sha512-dawW9ifvWAWmUm9/h+/UQ2jrdvjCJ7VJEuCJ6XVNudzcOwm53BFZH4Q845vjfgoUAM8ZxokvVNxNxAITc502YA== @@ -122,30 +108,6 @@ "@azure/abort-controller" "^2.0.0" tslib "^2.6.2" -"@azure/core-util@^1.8.1": - version "1.9.2" - resolved "https://registry.yarnpkg.com/@azure/core-util/-/core-util-1.9.2.tgz#1dc37dc5b0dae34c578be62cf98905ba7c0cafe7" - integrity sha512-l1Qrqhi4x1aekkV+OlcqsJa4AnAkj5p0JV8omgwjaV9OAbP41lvrMvs+CptfetKkeEaGRGSzby7sjPZEX7+kkQ== - dependencies: - "@azure/abort-controller" "^2.0.0" - tslib "^2.6.2" - -"@azure/cosmos@^4.1.0": - version "4.1.0" - resolved "https://registry.yarnpkg.com/@azure/cosmos/-/cosmos-4.1.0.tgz#97014d8110d94c4b47911350a018ad2d726493b0" - integrity sha512-+m085WKIGkf6wyw4vT85FFXl9j3U35u+LFFVwmLqfPbolnQAtoX24cowXz+vseW4BWKyx6Lamb+Zz+jl69zn6g== - dependencies: - "@azure/abort-controller" "^2.0.0" - "@azure/core-auth" "^1.7.1" - "@azure/core-rest-pipeline" "^1.15.1" - "@azure/core-tracing" "^1.1.1" - "@azure/core-util" "^1.8.1" - fast-json-stable-stringify "^2.1.0" - jsbi "^4.3.0" - priorityqueuejs "^2.0.0" - semaphore "^1.1.0" - tslib "^2.6.2" - "@azure/identity@^3.4.1": version "3.4.2" resolved "https://registry.yarnpkg.com/@azure/identity/-/identity-3.4.2.tgz#6b01724c9caac7cadab6b63c76584345bda8e2de" @@ -3245,26 +3207,6 @@ dbgate-query-splitter@^4.10.1: resolved "https://registry.yarnpkg.com/dbgate-query-splitter/-/dbgate-query-splitter-4.10.1.tgz#dc40d792de06f779a743cad054d5e786006b03a9" integrity sha512-KqrB7NLP1jXbx8rN7gSmYUVorm6ICeqOV+oR+jHaBLXqqhWepHsKr6JJlFEeb/LhoVjnTDY/cy5zhW1dMIQF6A== -dbgate-sqltree@^5.3.4: - version "5.3.4" - resolved "https://registry.yarnpkg.com/dbgate-sqltree/-/dbgate-sqltree-5.3.4.tgz#d91bbb1a3264dc8d88898fbb5427ee15aacc14a3" - integrity sha512-pvfjuI51plcmwErxxDCl8ZLXb9VfDnT+NukEMExiytYrTg3dDF2j9xwsYR9MZ/UaOQjwEO4LZ2FBLg/B776CuA== - dependencies: - lodash "^4.17.21" - -dbgate-tools@^5.0.0: - version "5.3.4" - resolved "https://registry.yarnpkg.com/dbgate-tools/-/dbgate-tools-5.3.4.tgz#168662ccd92e404a31fe3d5e29f732a91a0ea9b6" - integrity sha512-EsZafhQIGx8AlUT5PIMXoS0LTCxtANuQnxFUMwL20DfW/CCJtwWALHwwi2Am+1/YbDeM9Uh3FhWPTiZh5fhLYA== - dependencies: - dbgate-query-splitter "^4.10.1" - dbgate-sqltree "^5.3.4" - debug "^4.3.4" - json-stable-stringify "^1.0.1" - lodash "^4.17.21" - pinomin "^1.0.4" - uuid "^3.4.0" - debug@2.6.9, debug@^2.2.0, debug@^2.3.3: version "2.6.9" resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" @@ -4141,7 +4083,7 @@ fast-glob@^3.0.3: merge2 "^1.3.0" micromatch "^4.0.4" -fast-json-stable-stringify@2.x, fast-json-stable-stringify@^2.0.0, fast-json-stable-stringify@^2.1.0: +fast-json-stable-stringify@2.x, fast-json-stable-stringify@^2.0.0: version "2.1.0" resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz#874bf69c6f404c2b5d99c481341399fd55892633" integrity sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw== @@ -6702,11 +6644,6 @@ js-yaml@^4.1.0: dependencies: argparse "^2.0.1" -jsbi@^4.3.0: - version "4.3.0" - resolved "https://registry.yarnpkg.com/jsbi/-/jsbi-4.3.0.tgz#b54ee074fb6fcbc00619559305c8f7e912b04741" - integrity sha512-SnZNcinB4RIcnEyZqFPdGPVgrg2AcnykiBy0sHVJQKHYeaLUvi3Exj+iaPpLnFVkDPZIV4U0yvgC9/R4uEAZ9g== - jsbn@1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-1.1.0.tgz#b01307cb29b618a1ed26ec79e911f803c4da0040" @@ -8521,11 +8458,6 @@ printj@~1.1.0: resolved "https://registry.yarnpkg.com/printj/-/printj-1.1.2.tgz#d90deb2975a8b9f600fb3a1c94e3f4c53c78a222" integrity sha512-zA2SmoLaxZyArQTOPj5LXecR+RagfPSU5Kw1qP+jkWeNlrq+eJZyY2oS68SU1Z/7/myXM4lo9716laOFAVStCQ== -priorityqueuejs@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/priorityqueuejs/-/priorityqueuejs-2.0.0.tgz#96064040edd847ee9dd3013d8e16297399a6bd4f" - integrity sha512-19BMarhgpq3x4ccvVi8k2QpJZcymo/iFUcrhPd4V96kYGovOdTsWwy7fxChYi4QY+m2EnGBWSX9Buakz+tWNQQ== - process-nextick-args@~1.0.6: version "1.0.7" resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-1.0.7.tgz#150e20b756590ad3f91093f25a4f2ad8bff30ba3" @@ -9235,11 +9167,6 @@ secure-json-parse@^2.4.0: resolved "https://registry.yarnpkg.com/secure-json-parse/-/secure-json-parse-2.7.0.tgz#5a5f9cd6ae47df23dba3151edd06855d47e09862" integrity sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw== -semaphore@^1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/semaphore/-/semaphore-1.1.0.tgz#aaad8b86b20fe8e9b32b16dc2ee682a8cd26a8aa" - integrity sha512-O4OZEaNtkMd/K0i6js9SL+gqy0ZCBMgUvlSqHKi4IBdjhe7wB8pwztUk1BbZ1fmrvpwFrPbHzqd2w5pTcJH6LA== - semiver@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/semiver/-/semiver-1.1.0.tgz#9c97fb02c21c7ce4fcf1b73e2c7a24324bdddd5f"