dennii/dbgate
1
0
Fork 0
mirror of https://github.com/DeNNiiInc/dbgate.git synced 2026-04-25 12:06:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

perspectives - remove mongo hardcodes

Browse Source
This commit is contained in:
Jan Prochazka
2024-08-20 10:28:02 +02:00
parent 30e3bc6eeb
commit 80a4d3f238
10 changed files with 133 additions and 149 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
packages/datalib/src/PerspectiveCache.ts
View File

@@ -105,7 +105,6 @@ export class PerspectiveCache {
'databaseConfig',
'orderBy',
'sqlCondition',
'mongoCondition',
])
);
let res = this.tables[tableKey];

53
packages/datalib/src/PerspectiveDataLoader.ts
View File

@@ -5,6 +5,7 @@ import _zipObject from 'lodash/zipObject';
import _mapValues from 'lodash/mapValues';
import _isArray from 'lodash/isArray';
import { safeJsonParse } from 'dbgate-tools';
import { CollectionAggregateDefinition } from 'dbgate-types';
function normalizeLoadedRow(row) {
return _mapValues(row, v => safeJsonParse(v) || v);
@@ -59,24 +60,6 @@ export class PerspectiveDataLoader {
: null;
}
buildMongoCondition(props: PerspectiveDataLoadProps): {} {
const { schemaName, pureName, bindingColumns, bindingValues, dataColumns, orderBy, mongoCondition } = props;
const conditions = [];
if (mongoCondition) {
conditions.push(mongoCondition);
}
if (bindingColumns?.length == 1) {
conditions.push({
[bindingColumns[0]]: { $in: bindingValues.map(x => x[0]) },
});
}
return conditions.length == 1 ? conditions[0] : conditions.length > 0 ? { $and: conditions } : null;
}
async loadGroupingSqlDb(props: PerspectiveDataLoadProps) {
const { schemaName, pureName, bindingColumns } = props;
@@ -135,18 +118,28 @@ export class PerspectiveDataLoader {
async loadGroupingDocDb(props: PerspectiveDataLoadProps) {
const { schemaName, pureName, bindingColumns } = props;
const aggregate = [
{ $match: this.buildMongoCondition(props) },
{
$group: {
_id: _zipObject(
bindingColumns,
bindingColumns.map(col => '$' + col)
),
count: { $sum: 1 },
const aggregate: CollectionAggregateDefinition = {
condition: this.buildSqlCondition(props),
groupByColumns: bindingColumns,
aggregateColumns: [
{
alias: 'acount',
aggregateFunction: 'count',
},
},
];
],
};
// const aggregate = [
// { $match: this.buildMongoCondition(props) },
// {
// $group: {
// _id: _zipObject(
// bindingColumns,
// bindingColumns.map(col => '$' + col)
// ),
// count: { $sum: 1 },
// },
// },
// ];
if (dbg?.enabled) {
dbg(`LOAD COUNTS, table=${props.pureName}, columns=${bindingColumns?.join(',')}`);
@@ -244,7 +237,7 @@ export class PerspectiveDataLoader {
const { pureName } = props;
const res: any = {
pureName,
condition: this.buildMongoCondition(props),
condition: this.buildSqlCondition(props),
skip: props.range?.offset,
limit: props.range?.limit,
};

1
packages/datalib/src/PerspectiveDataProvider.ts
View File

@@ -25,7 +25,6 @@ export interface PerspectiveDataLoadProps {
range?: RangeDefinition;
topCount?: number;
sqlCondition?: Condition;
mongoCondition?: any;
engineType: PerspectiveDatabaseEngineType;
}

82
packages/datalib/src/PerspectiveTreeNode.ts
View File

@@ -349,8 +349,23 @@ export abstract class PerspectiveTreeNode {
);
}
getMutliColumnCondition(source): Condition {
if (!this.nodeConfig?.multiColumnFilter) return null;
const base = this.getBaseTableFromThis() as TableInfo | ViewInfo | CollectionInfo;
if (!base) return null;
const isDocDb = isCollectionInfo(base);
if (isDocDb) {
return this.getMutliColumnNoSqlCondition();
} else {
return this.getMutliColumnSqlCondition(source);
}
}
getMutliColumnSqlCondition(source): Condition {
if (!this.nodeConfig?.multiColumnFilter) return null;
const base = this.getBaseTableFromThis() as TableInfo | ViewInfo;
if (!base) return null;
try {
@@ -383,32 +398,40 @@ export abstract class PerspectiveTreeNode {
return null;
}
getMutliColumnMongoCondition(): {} {
getMutliColumnNoSqlCondition(): Condition {
if (!this.nodeConfig?.multiColumnFilter) return null;
const pattern = this.dataProvider?.dataPatterns?.[this.designerId];
if (!pattern) return null;
const condition = parseFilter(this.nodeConfig?.multiColumnFilter, mongoFilterBehaviour);
if (!condition) return null;
const res = pattern.columns.map(col => {
return _cloneDeepWith(condition, expr => {
if (expr.__placeholder__) {
return {
[col.name]: expr.__placeholder__,
};
}
});
});
return {
$or: res,
const orCondition: CompoudCondition = {
conditionType: 'or',
conditions: [],
};
for (const column of pattern.columns || []) {
orCondition.conditions.push(
_cloneDeepWith(condition, (expr: Expression) => {
if (expr.exprType == 'placeholder') {
return {
exprType: 'column',
columnName: column.name,
};
}
})
);
}
if (orCondition.conditions.length > 0) {
return orCondition;
}
}
getChildrenSqlCondition(source = null): Condition {
const conditions = _compact([
...this.childNodes.map(x => x.parseFilterCondition(source)),
...this.buildParentFilterConditions(),
this.getMutliColumnSqlCondition(source),
this.getMutliColumnCondition(source),
]);
if (conditions.length == 0) {
return null;
@@ -422,20 +445,6 @@ export abstract class PerspectiveTreeNode {
};
}
getChildrenMongoCondition(source = null): {} {
const conditions = _compact([
...this.childNodes.map(x => x.parseFilterCondition(source)),
this.getMutliColumnMongoCondition(),
]);
if (conditions.length == 0) {
return null;
}
if (conditions.length == 1) {
return conditions[0];
}
return { $and: conditions };
}
getOrderBy(table: TableInfo | ViewInfo | CollectionInfo): PerspectiveDataLoadProps['orderBy'] {
const res = _compact(
this.childNodes.map(node => {
@@ -1158,17 +1167,16 @@ export class PerspectiveTableNode extends PerspectiveTreeNode {
}
getNodeLoadProps(parentRows: any[]): PerspectiveDataLoadProps {
const isMongo = isCollectionInfo(this.table);
const isDocDb = isCollectionInfo(this.table);
return {
schemaName: this.table.schemaName,
pureName: this.table.pureName,
dataColumns: this.getDataLoadColumns(),
allColumns: isMongo,
allColumns: isDocDb,
databaseConfig: this.databaseConfig,
orderBy: this.getOrderBy(this.table),
sqlCondition: isMongo ? null : this.getChildrenSqlCondition(),
mongoCondition: isMongo ? this.getChildrenMongoCondition() : null,
engineType: isMongo ? 'docdb' : 'sqldb',
sqlCondition: this.getChildrenSqlCondition(),
engineType: isDocDb ? 'docdb' : 'sqldb',
};
}
@@ -1372,7 +1380,7 @@ export class PerspectiveCustomJoinTreeNode extends PerspectiveTableNode {
// console.log('PARENT ROWS', parentRows);
// console.log('this.getDataLoadColumns()', this.getDataLoadColumns());
const isMongo = isCollectionInfo(this.table);
const isDocDb = isCollectionInfo(this.table);
// const bindingValues = [];
@@ -1432,12 +1440,12 @@ export class PerspectiveCustomJoinTreeNode extends PerspectiveTableNode {
bindingColumns: this.getParentMatchColumns(),
bindingValues: _uniqBy(bindingValues, x => JSON.stringify(x)),
dataColumns: this.getDataLoadColumns(),
allColumns: isMongo,
allColumns: isDocDb,
databaseConfig: this.databaseConfig,
orderBy: this.getOrderBy(this.table),
sqlCondition: isMongo ? null : this.getChildrenSqlCondition(),
mongoCondition: isMongo ? this.getChildrenMongoCondition() : null,
engineType: isMongo ? 'docdb' : 'sqldb',
sqlCondition: this.getChildrenSqlCondition(),
// mongoCondition: isMongo ? this.getChildrenMongoCondition() : null,
engineType: isDocDb ? 'docdb' : 'sqldb',
};
}

13
packages/types/engines.d.ts vendored
View File

@@ -41,6 +41,8 @@ export interface ReadCollectionOptions {
countDocuments?: boolean;
skip?: number;
limit?: number;
condition?: any;
aggregate?: CollectionAggregateDefinition;
}
export interface NewObjectTemplate {
@@ -72,6 +74,17 @@ export interface ServerSummary {
databases: ServerSummaryDatabase[];
}
export type CollectionAggregateFunction = 'count' | 'sum' | 'avg' | 'min' | 'max';
export interface CollectionAggregateDefinition {
condition: any; // SQL tree condition
groupByColumns: string[];
aggregateColumns: {
alias: string;
aggregateFunction: CollectionAggregateFunction;
columnArgument?: string;
}[];
}
export interface FilterBehaviourProvider {
getFilterBehaviour(dataType: string, standardFilterBehaviours: { [id: string]: FilterBehaviour }): FilterBehaviour;
}

2
packages/types/filter-type.d.ts vendored
View File

@@ -1,5 +1,3 @@
export type FilterParserCompilerType = 'sqlTree' | 'mongoCondition' | 'datetime';
export interface FilterBehaviour {
supportEquals?: boolean;
supportStringInclusion?: boolean;

1
plugins/dbgate-plugin-mongo/package.json
View File

@@ -33,6 +33,7 @@
"devDependencies": {
"dbgate-plugin-tools": "^1.0.7",
"dbgate-query-splitter": "^4.10.1",
"lodash": "^4.17.21",
"webpack": "^5.91.0",
"webpack-cli": "^5.1.4",
"dbgate-tools": "^5.0.0-alpha.1",

4
plugins/dbgate-plugin-mongo/src/backend/driver.js
View File

@@ -7,7 +7,7 @@ const MongoClient = require('mongodb').MongoClient;
const ObjectId = require('mongodb').ObjectId;
const AbstractCursor = require('mongodb').AbstractCursor;
const createBulkInsertStream = require('./createBulkInsertStream');
const { convertToMongoCondition } = require('../frontend/convertToMongoCondition');
const { convertToMongoCondition, convertToMongoAggregate } = require('../frontend/convertToMongoCondition');
function transformMongoData(row) {
return _.cloneDeepWith(row, (x) => {
@@ -280,7 +280,7 @@ const driver = {
const count = await collection.countDocuments(convertObjectId(mongoCondition) || {});
return { count };
} else if (options.aggregate) {
let cursor = await collection.aggregate(convertObjectId(options.aggregate));
let cursor = await collection.aggregate(convertObjectId(convertToMongoAggregate(options.aggregate)));
const rows = await cursor.toArray();
return { rows: rows.map(transformMongoData) };
} else {

46
plugins/dbgate-plugin-mongo/src/frontend/convertToMongoCondition.js
View File

@@ -1,3 +1,5 @@
const _zipObject = require('lodash/zipObject');
function convertLeftOperandToMongoColumn(left) {
if (left.exprType == 'placeholder') return '__placeholder__';
if (left.exprType == 'column') return left.columnName;
@@ -121,11 +123,55 @@ function convertToMongoCondition(filter) {
};
}
case 'in':
return {
[convertLeftOperandToMongoColumn(filter.expr)]: {
$in: filter.values,
},
};
default:
throw new Error(`Unknown condition type ${filter.conditionType}`);
}
}
function convertToMongoAggregateFunction(aggregate) {
switch (aggregate.aggregateFunction) {
case 'count':
return { $sum: 1 };
case 'sum':
return { $sum: `$${aggregate.columnArgument}` };
case 'avg':
return { $avg: `$${aggregate.columnArgument}` };
case 'min':
return { $min: `$${aggregate.columnArgument}` };
case 'max':
return { $max: `$${aggregate.columnArgument}` };
default:
throw new Error(`Unknown aggregate function ${aggregate.aggregateFunction}`);
}
}
function convertToMongoAggregate(collectionAggregate) {
return [
{ $match: convertToMongoCondition(collectionAggregate.condition) },
{
$group: {
_id: _zipObject(
collectionAggregate.groupByColumns,
collectionAggregate.groupByColumns.map((col) => '$' + col)
),
..._zipObject(
collectionAggregate.aggregateColumns.map((col) => col.alias),
collectionAggregate.aggregateColumns.map((col) => convertToMongoAggregateFunction(col))
),
count: { $sum: 1 },
},
},
];
}
module.exports = {
convertToMongoCondition,
convertToMongoAggregate,
};

79
yarn.lock
View File

@@ -31,7 +31,7 @@
dependencies:
tslib "^2.6.2"
"@azure/core-auth@^1.3.0", "@azure/core-auth@^1.4.0", "@azure/core-auth@^1.5.0", "@azure/core-auth@^1.7.1":
"@azure/core-auth@^1.3.0", "@azure/core-auth@^1.4.0", "@azure/core-auth@^1.5.0":
version "1.7.2"
resolved "https://registry.yarnpkg.com/@azure/core-auth/-/core-auth-1.7.2.tgz#558b7cb7dd12b00beec07ae5df5907d74df1ebd9"
integrity sha512-Igm/S3fDYmnMq1uKS38Ae1/m37B3zigdlZw+kocwEhh5GjyKjPrXKO2J6rzpC1wAxrNil/jX9BJRqBshyjnF3g==
@@ -93,21 +93,7 @@
https-proxy-agent "^7.0.0"
tslib "^2.6.2"
"@azure/core-rest-pipeline@^1.15.1":
version "1.16.3"
resolved "https://registry.yarnpkg.com/@azure/core-rest-pipeline/-/core-rest-pipeline-1.16.3.tgz#bde3bc3ebad7f885ddd9de6af5e5a8fc254b287e"
integrity sha512-VxLk4AHLyqcHsfKe4MZ6IQ+D+ShuByy+RfStKfSjxJoL3WBWq17VNmrz8aT8etKzqc2nAeIyLxScjpzsS4fz8w==
dependencies:
"@azure/abort-controller" "^2.0.0"
"@azure/core-auth" "^1.4.0"
"@azure/core-tracing" "^1.0.1"
"@azure/core-util" "^1.9.0"
"@azure/logger" "^1.0.0"
http-proxy-agent "^7.0.0"
https-proxy-agent "^7.0.0"
tslib "^2.6.2"
"@azure/core-tracing@^1.0.0", "@azure/core-tracing@^1.0.1", "@azure/core-tracing@^1.1.1":
"@azure/core-tracing@^1.0.0", "@azure/core-tracing@^1.0.1":
version "1.1.2"
resolved "https://registry.yarnpkg.com/@azure/core-tracing/-/core-tracing-1.1.2.tgz#065dab4e093fb61899988a1cdbc827d9ad90b4ee"
integrity sha512-dawW9ifvWAWmUm9/h+/UQ2jrdvjCJ7VJEuCJ6XVNudzcOwm53BFZH4Q845vjfgoUAM8ZxokvVNxNxAITc502YA==
@@ -122,30 +108,6 @@
"@azure/abort-controller" "^2.0.0"
tslib "^2.6.2"
"@azure/core-util@^1.8.1":
version "1.9.2"
resolved "https://registry.yarnpkg.com/@azure/core-util/-/core-util-1.9.2.tgz#1dc37dc5b0dae34c578be62cf98905ba7c0cafe7"
integrity sha512-l1Qrqhi4x1aekkV+OlcqsJa4AnAkj5p0JV8omgwjaV9OAbP41lvrMvs+CptfetKkeEaGRGSzby7sjPZEX7+kkQ==
dependencies:
"@azure/abort-controller" "^2.0.0"
tslib "^2.6.2"
"@azure/cosmos@^4.1.0":
version "4.1.0"
resolved "https://registry.yarnpkg.com/@azure/cosmos/-/cosmos-4.1.0.tgz#97014d8110d94c4b47911350a018ad2d726493b0"
integrity sha512-+m085WKIGkf6wyw4vT85FFXl9j3U35u+LFFVwmLqfPbolnQAtoX24cowXz+vseW4BWKyx6Lamb+Zz+jl69zn6g==
dependencies:
"@azure/abort-controller" "^2.0.0"
"@azure/core-auth" "^1.7.1"
"@azure/core-rest-pipeline" "^1.15.1"
"@azure/core-tracing" "^1.1.1"
"@azure/core-util" "^1.8.1"
fast-json-stable-stringify "^2.1.0"
jsbi "^4.3.0"
priorityqueuejs "^2.0.0"
semaphore "^1.1.0"
tslib "^2.6.2"
"@azure/identity@^3.4.1":
version "3.4.2"
resolved "https://registry.yarnpkg.com/@azure/identity/-/identity-3.4.2.tgz#6b01724c9caac7cadab6b63c76584345bda8e2de"
@@ -3245,26 +3207,6 @@ dbgate-query-splitter@^4.10.1:
resolved "https://registry.yarnpkg.com/dbgate-query-splitter/-/dbgate-query-splitter-4.10.1.tgz#dc40d792de06f779a743cad054d5e786006b03a9"
integrity sha512-KqrB7NLP1jXbx8rN7gSmYUVorm6ICeqOV+oR+jHaBLXqqhWepHsKr6JJlFEeb/LhoVjnTDY/cy5zhW1dMIQF6A==
dbgate-sqltree@^5.3.4:
version "5.3.4"
resolved "https://registry.yarnpkg.com/dbgate-sqltree/-/dbgate-sqltree-5.3.4.tgz#d91bbb1a3264dc8d88898fbb5427ee15aacc14a3"
integrity sha512-pvfjuI51plcmwErxxDCl8ZLXb9VfDnT+NukEMExiytYrTg3dDF2j9xwsYR9MZ/UaOQjwEO4LZ2FBLg/B776CuA==
dependencies:
lodash "^4.17.21"
dbgate-tools@^5.0.0:
version "5.3.4"
resolved "https://registry.yarnpkg.com/dbgate-tools/-/dbgate-tools-5.3.4.tgz#168662ccd92e404a31fe3d5e29f732a91a0ea9b6"
integrity sha512-EsZafhQIGx8AlUT5PIMXoS0LTCxtANuQnxFUMwL20DfW/CCJtwWALHwwi2Am+1/YbDeM9Uh3FhWPTiZh5fhLYA==
dependencies:
dbgate-query-splitter "^4.10.1"
dbgate-sqltree "^5.3.4"
debug "^4.3.4"
json-stable-stringify "^1.0.1"
lodash "^4.17.21"
pinomin "^1.0.4"
uuid "^3.4.0"
debug@2.6.9, debug@^2.2.0, debug@^2.3.3:
version "2.6.9"
resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f"
@@ -4141,7 +4083,7 @@ fast-glob@^3.0.3:
merge2 "^1.3.0"
micromatch "^4.0.4"
fast-json-stable-stringify@2.x, fast-json-stable-stringify@^2.0.0, fast-json-stable-stringify@^2.1.0:
fast-json-stable-stringify@2.x, fast-json-stable-stringify@^2.0.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz#874bf69c6f404c2b5d99c481341399fd55892633"
integrity sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==
@@ -6702,11 +6644,6 @@ js-yaml@^4.1.0:
dependencies:
argparse "^2.0.1"
jsbi@^4.3.0:
version "4.3.0"
resolved "https://registry.yarnpkg.com/jsbi/-/jsbi-4.3.0.tgz#b54ee074fb6fcbc00619559305c8f7e912b04741"
integrity sha512-SnZNcinB4RIcnEyZqFPdGPVgrg2AcnykiBy0sHVJQKHYeaLUvi3Exj+iaPpLnFVkDPZIV4U0yvgC9/R4uEAZ9g==
jsbn@1.1.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-1.1.0.tgz#b01307cb29b618a1ed26ec79e911f803c4da0040"
@@ -8521,11 +8458,6 @@ printj@~1.1.0:
resolved "https://registry.yarnpkg.com/printj/-/printj-1.1.2.tgz#d90deb2975a8b9f600fb3a1c94e3f4c53c78a222"
integrity sha512-zA2SmoLaxZyArQTOPj5LXecR+RagfPSU5Kw1qP+jkWeNlrq+eJZyY2oS68SU1Z/7/myXM4lo9716laOFAVStCQ==
priorityqueuejs@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/priorityqueuejs/-/priorityqueuejs-2.0.0.tgz#96064040edd847ee9dd3013d8e16297399a6bd4f"
integrity sha512-19BMarhgpq3x4ccvVi8k2QpJZcymo/iFUcrhPd4V96kYGovOdTsWwy7fxChYi4QY+m2EnGBWSX9Buakz+tWNQQ==
process-nextick-args@~1.0.6:
version "1.0.7"
resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-1.0.7.tgz#150e20b756590ad3f91093f25a4f2ad8bff30ba3"
@@ -9235,11 +9167,6 @@ secure-json-parse@^2.4.0:
resolved "https://registry.yarnpkg.com/secure-json-parse/-/secure-json-parse-2.7.0.tgz#5a5f9cd6ae47df23dba3151edd06855d47e09862"
integrity sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw==
semaphore@^1.1.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/semaphore/-/semaphore-1.1.0.tgz#aaad8b86b20fe8e9b32b16dc2ee682a8cd26a8aa"
integrity sha512-O4OZEaNtkMd/K0i6js9SL+gqy0ZCBMgUvlSqHKi4IBdjhe7wB8pwztUk1BbZ1fmrvpwFrPbHzqd2w5pTcJH6LA==
semiver@^1.0.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/semiver/-/semiver-1.1.0.tgz#9c97fb02c21c7ce4fcf1b73e2c7a24324bdddd5f"