From 23a52dc79e9c9c864245af8427d6ffc35dc6530d Mon Sep 17 00:00:00 2001 From: "SPRINX0\\prochazka" Date: Fri, 23 Aug 2024 14:42:18 +0200 Subject: [PATCH 01/12] grid data types WIP --- packages/tools/src/driverBase.ts | 5 + packages/tools/src/stringTools.ts | 142 +++++++++++++++--- packages/types/engines.d.ts | 21 +++ packages/web/src/datagrid/DataGridCell.svelte | 33 ++-- packages/web/src/datagrid/DataGridCore.svelte | 2 +- packages/web/src/datagrid/DataGridRow.svelte | 2 + .../web/src/datagrid/InplaceEditor.svelte | 3 + packages/web/src/datagrid/InplaceInput.svelte | 15 +- .../web/src/datagrid/InplaceSelect.svelte | 3 +- packages/web/src/formview/FormView.svelte | 20 ++- .../formview/ShowFormDropDownButton.svelte | 44 ++++++ packages/web/src/icons/FontIcon.svelte | 10 ++ packages/web/src/utility/clipboard.ts | 7 +- .../src/frontend/driver.js | 18 +++ 14 files changed, 275 insertions(+), 50 deletions(-) create mode 100644 packages/web/src/formview/ShowFormDropDownButton.svelte diff --git a/packages/tools/src/driverBase.ts b/packages/tools/src/driverBase.ts index d76318b9d..3a304525e 100644 --- a/packages/tools/src/driverBase.ts +++ b/packages/tools/src/driverBase.ts @@ -161,4 +161,9 @@ export const driverBase = { getCollectionExportQueryJson(collection: string, condition: any, sort: any) { return null; }, + + dataEditorTypesBehaviour: { + parseSqlNull: true, + parseHexAsBuffer: true, + }, }; diff --git a/packages/tools/src/stringTools.ts b/packages/tools/src/stringTools.ts index 21be36385..723e7b110 100644 --- a/packages/tools/src/stringTools.ts +++ b/packages/tools/src/stringTools.ts @@ -1,6 +1,8 @@ import _isString from 'lodash/isString'; import _isArray from 'lodash/isArray'; +import _isNumber from 'lodash/isNumber'; import _isPlainObject from 'lodash/isPlainObject'; +import { DataEditorTypesBehaviour } from 'dbgate-types'; export function arrayToHexString(byteArray) { return byteArray.reduce((output, elem) => output + ('0' + elem.toString(16)).slice(-2), '').toUpperCase(); @@ -15,36 +17,103 @@ export function hexStringToArray(inputString) { return res; } -export function parseCellValue(value) { +export function parseCellValue(value, editorTypes?: DataEditorTypesBehaviour) { if (!_isString(value)) return value; - if (value == '(NULL)') return null; - - const mHex = value.match(/^0x([0-9a-fA-F][0-9a-fA-F])+$/); - if (mHex) { - return { - type: 'Buffer', - data: hexStringToArray(value.substring(2)), - }; + if (editorTypes?.parseSqlNull) { + if (value == '(NULL)') return null; } - const mOid = value.match(/^ObjectId\("([0-9a-f]{24})"\)$/); - if (mOid) { - return { $oid: mOid[1] }; + if (editorTypes?.parseHexAsBuffer) { + const mHex = value.match(/^0x([0-9a-fA-F][0-9a-fA-F])+$/); + if (mHex) { + return { + type: 'Buffer', + data: hexStringToArray(value.substring(2)), + }; + } + } + + if (editorTypes?.parseObjectIdAsDollar) { + const mOid = value.match(/^ObjectId\("([0-9a-f]{24})"\)$/); + if (mOid) { + return { $oid: mOid[1] }; + } + } + + if (editorTypes?.parseJsonNull) { + if (value == 'null') return null; + } + + if (editorTypes?.parseJsonBoolean) { + if (value == 'true') return true; + if (value == 'false') return false; + } + + if (editorTypes?.parseNumber) { + if (/^-?[0-9]+(?:\.[0-9]+)?$/.test(value)) { + return parseFloat(value); + } + } + + if (editorTypes?.parseJsonArray || editorTypes?.parseJsonObject) { + const jsonValue = safeJsonParse(value); + if (_isPlainObject(jsonValue) && editorTypes?.parseJsonObject) return jsonValue; + if (_isArray(jsonValue) && editorTypes?.parseJsonArray) return jsonValue; } return value; } -export function stringifyCellValue(value) { - if (value === null) return '(NULL)'; - if (value === undefined) return '(NoField)'; - if (value?.type == 'Buffer' && _isArray(value.data)) return '0x' + arrayToHexString(value.data); - if (value?.$oid) return `ObjectId("${value?.$oid}")`; - if (_isPlainObject(value) || _isArray(value)) return JSON.stringify(value); +function parseObjectIdAsDollar(value) { + if (value?.$oid) return value; + if (_isString(value)) { + if (value.match(/^[0-9a-f]{24}$/)) return { $oid: value }; + const mOid = value.match(/^ObjectId\("([0-9a-f]{24})"\)$/); + if (mOid) { + return { $oid: mOid[1] }; + } + } return value; } +export function stringifyCellValue(value, editorTypes?: DataEditorTypesBehaviour) { + if (editorTypes?.parseSqlNull) { + if (value === null) return '(NULL)'; + } + if (value === undefined) return '(NoField)'; + if (editorTypes?.parseJsonNull) { + if (value === null) return 'null'; + } + if (editorTypes?.parseJsonBoolean) { + if (value === true) return 'true'; + if (value === false) return 'false'; + } + if (editorTypes?.parseHexAsBuffer) { + if (value?.type == 'Buffer' && _isArray(value.data)) return '0x' + arrayToHexString(value.data); + } + if (editorTypes?.parseObjectIdAsDollar) { + if (value?.$oid) return `ObjectId("${value?.$oid}")`; + } + if (editorTypes?.parseJsonArray) { + if (_isArray(value)) return JSON.stringify(value); + } + if (editorTypes?.parseJsonObject) { + if (_isPlainObject(value)) return JSON.stringify(value); + } + if (editorTypes?.parseNumber) { + if (_isNumber(value)) return value.toString(); + } + + if (_isString(value)) return value; + + // fallback + if (_isNumber(value)) return value.toString(); + if (value === null || value === undefined) return ''; + + return ''; +} + export function safeJsonParse(json, defaultValue?, logError = false) { if (_isArray(json) || _isPlainObject(json)) { return json; @@ -127,3 +196,40 @@ export function parseSqlDefaultValue(value: string) { } return undefined; } + +export function detectTypeIcon(value) { + if (value === null) return 'icon type-null'; + if (value?.$oid) return 'icon type-objectid'; + if (_isString(value)) return 'icon type-string'; + if (_isNumber(value)) return 'icon type-number'; + if (_isPlainObject(value)) return 'icon type-object'; + if (_isArray(value)) return 'icon type-array'; + if (value === true || value === false) return 'icon type-boolean'; + return 'icon type-unknown'; +} + +export function getConvertValueMenu(value, onSetValue, editorTypes?: DataEditorTypesBehaviour) { + return [ + editorTypes?.supportStringType && { + text: 'String', + onClick: () => onSetValue(stringifyCellValue(value, editorTypes)), + }, + editorTypes?.supportNumberType && { text: 'Number', onClick: () => onSetValue(parseFloat(value)) }, + editorTypes?.supportNullType && { text: 'Null', onClick: () => onSetValue(null) }, + editorTypes?.supportBooleanType && { + text: 'Boolean', + onClick: () => onSetValue(value?.toString()?.toLowerCase() == 'true' || value == '1'), + }, + editorTypes?.supportObjectIdType && { text: 'ObjectId', onClick: () => onSetValue(parseObjectIdAsDollar(value)) }, + editorTypes?.supportJsonType && { + text: 'JSON', + onClick: () => { + const jsonValue = safeJsonParse(value); + if (jsonValue != null) { + console.log('**** ON SET VALUE', jsonValue); + onSetValue(jsonValue); + } + }, + }, + ]; +} diff --git a/packages/types/engines.d.ts b/packages/types/engines.d.ts index c828a4b9e..f6008c4a8 100644 --- a/packages/types/engines.d.ts +++ b/packages/types/engines.d.ts @@ -93,6 +93,26 @@ export interface CollectionSortDefinitionItem { export type CollectionSortDefinition = CollectionSortDefinitionItem[]; +export interface DataEditorTypesBehaviour { + parseSqlNull?: boolean; + parseJsonNull?: boolean; + parseJsonBoolean?: boolean; + parseNumber?: boolean; + parseJsonArray?: boolean; + parseJsonObject?: boolean; + parseHexAsBuffer?: boolean; + parseObjectIdAsDollar?: boolean; + + explicitDataType?: boolean; + supportNumberType?: boolean; + supportStringType?: boolean; + supportBooleanType?: boolean; + supportDateType?: boolean; + supportNullType?: boolean; + supportJsonType?: boolean; + supportObjectIdType?: boolean; +} + export interface FilterBehaviourProvider { getFilterBehaviour(dataType: string, standardFilterBehaviours: { [id: string]: FilterBehaviour }): FilterBehaviour; } @@ -105,6 +125,7 @@ export interface EngineDriver extends FilterBehaviourProvider { editorMode?: string; readOnlySessions: boolean; supportedKeyTypes: SupportedDbKeyType[]; + dataEditorTypesBehaviour: DataEditorTypesBehaviour; supportsDatabaseUrl?: boolean; supportsDatabaseDump?: boolean; supportsServerSummary?: boolean; diff --git a/packages/web/src/datagrid/DataGridCell.svelte b/packages/web/src/datagrid/DataGridCell.svelte index f22b394c2..cb49efa79 100644 --- a/packages/web/src/datagrid/DataGridCell.svelte +++ b/packages/web/src/datagrid/DataGridCell.svelte @@ -1,13 +1,14 @@ {/if} - {#if col.foreignKey && rowData && rowData[col.uniqueName] && !isCurrentCell} + {#if editorTypes?.explicitDataType} + {#if value !== undefined} + getConvertValueMenu(value, onSetValue, editorTypes)} + /> + {/if} + {:else if col.foreignKey && rowData && rowData[col.uniqueName] && !isCurrentCell} onSetFormView(rowData, col)} /> - {/if} - - {#if col.foreignKey && isCurrentCell && onDictionaryLookup} + {:else if col.foreignKey && isCurrentCell && onDictionaryLookup} - {/if} - - {#if isJson} + {:else if isJson} openJsonDocument(value, undefined, true)} /> - {/if} - - {#if jsonParsedValue && _.isPlainObject(jsonParsedValue)} + {:else if jsonParsedValue && _.isPlainObject(jsonParsedValue)} openJsonDocument(jsonParsedValue, undefined, true)} /> - {/if} - - {#if _.isArray(jsonParsedValue || value)} + {:else if _.isArray(jsonParsedValue || value)} { diff --git a/packages/web/src/datagrid/DataGridCore.svelte b/packages/web/src/datagrid/DataGridCore.svelte index ec51a266f..fc5031724 100644 --- a/packages/web/src/datagrid/DataGridCore.svelte +++ b/packages/web/src/datagrid/DataGridCore.svelte @@ -1557,7 +1557,7 @@ } let colIndex = startCol; for (const cell of rowData) { - setCellValue([rowIndex, colIndex], parseCellValue(cell)); + setCellValue([rowIndex, colIndex], parseCellValue(cell, display?.driver?.dataEditorTypesBehaviour)); colIndex += 1; } rowIndex += 1; diff --git a/packages/web/src/datagrid/DataGridRow.svelte b/packages/web/src/datagrid/DataGridRow.svelte index 9c6924a9d..f41f86a92 100644 --- a/packages/web/src/datagrid/DataGridRow.svelte +++ b/packages/web/src/datagrid/DataGridRow.svelte @@ -62,6 +62,7 @@ options="{col.options}" canSelectMultipleOptions="{col.canSelectMultipleOptions}" onSetValue={value => grider.setCellValue(rowIndex, col.uniqueName, value)} + {driver} /> {:else} @@ -21,6 +22,7 @@ {onSetValue} {options} {canSelectMultipleOptions} + {driver} /> {:else} {/if} diff --git a/packages/web/src/datagrid/InplaceInput.svelte b/packages/web/src/datagrid/InplaceInput.svelte index aec5d7b61..fdc8c22c4 100644 --- a/packages/web/src/datagrid/InplaceInput.svelte +++ b/packages/web/src/datagrid/InplaceInput.svelte @@ -14,6 +14,7 @@ export let onSetValue; export let width; export let cellValue; + export let driver; let domEditor; let showEditorButton = true; @@ -22,6 +23,8 @@ const isChangedRef = createRef(!!inplaceEditorState.text); + $: editorTypes = driver?.dataEditorTypesBehaviour; + function handleKeyDown(event) { showEditorButton = false; @@ -32,7 +35,7 @@ break; case keycodes.enter: if (isChangedRef.get()) { - onSetValue(parseCellValue(domEditor.value)); + onSetValue(parseCellValue(domEditor.value, editorTypes)); isChangedRef.set(false); } domEditor.blur(); @@ -41,7 +44,7 @@ break; case keycodes.tab: if (isChangedRef.get()) { - onSetValue(parseCellValue(domEditor.value)); + onSetValue(parseCellValue(domEditor.value, editorTypes)); isChangedRef.set(false); } domEditor.blur(); @@ -51,7 +54,7 @@ case keycodes.s: if (isCtrlOrCommandKey(event)) { if (isChangedRef.get()) { - onSetValue(parseCellValue(domEditor.value)); + onSetValue(parseCellValue(domEditor.value, editorTypes)); isChangedRef.set(false); } event.preventDefault(); @@ -63,7 +66,7 @@ function handleBlur() { if (isChangedRef.get()) { - onSetValue(parseCellValue(domEditor.value)); + onSetValue(parseCellValue(domEditor.value, editorTypes)); // grider.setCellValue(rowIndex, uniqueName, editor.value); isChangedRef.set(false); } @@ -71,7 +74,7 @@ } onMount(() => { - domEditor.value = inplaceEditorState.text || stringifyCellValue(cellValue); + domEditor.value = inplaceEditorState.text || stringifyCellValue(cellValue, editorTypes); domEditor.focus(); if (inplaceEditorState.selectAll) { domEditor.select(); @@ -102,7 +105,7 @@ dispatchInsplaceEditor({ type: 'close' }); showModal(EditCellDataModal, { - value: stringifyCellValue(cellValue), + value: stringifyCellValue(cellValue, editorTypes), onSave: onSetValue, }); }} diff --git a/packages/web/src/datagrid/InplaceSelect.svelte b/packages/web/src/datagrid/InplaceSelect.svelte index 7148c2837..a1d58a45f 100644 --- a/packages/web/src/datagrid/InplaceSelect.svelte +++ b/packages/web/src/datagrid/InplaceSelect.svelte @@ -11,6 +11,7 @@ export let cellValue; export let options; export let canSelectMultipleOptions; + export let driver; let value; let valueInit; @@ -18,7 +19,7 @@ let isOptionsHidden = false; onMount(() => { - value = inplaceEditorState.text || stringifyCellValue(cellValue); + value = inplaceEditorState.text || stringifyCellValue(cellValue, driver?.dataEditorTypesBehaviour); valueInit = value; const optionsSelected = value.split(','); diff --git a/packages/web/src/formview/FormView.svelte b/packages/web/src/formview/FormView.svelte index e24d90216..10e27a633 100644 --- a/packages/web/src/formview/FormView.svelte +++ b/packages/web/src/formview/FormView.svelte @@ -273,7 +273,10 @@ export function copyToClipboard() { const column = getCellColumn(currentCell); if (!column) return; - const text = currentCell[1] % 2 == 1 ? extractRowCopiedValue(rowData, column.uniqueName) : column.columnName; + const text = + currentCell[1] % 2 == 1 + ? extractRowCopiedValue(rowData, column.uniqueName, display?.driver?.dataEditorTypesBehaviour) + : column.columnName; copyTextToClipboard(text); } @@ -631,11 +634,12 @@ {#if rowData && $inplaceEditorState.cell && rowIndex == $inplaceEditorState.cell[0] && chunkIndex * 2 + 1 == $inplaceEditorState.cell[1]} { grider.setCellValue(0, col.uniqueName, value); }} @@ -644,6 +648,7 @@ handleLookup(col)} + onSetValue={value => { + grider.setCellValue(0, col.uniqueName, value); + }} /> {/if} diff --git a/packages/web/src/formview/ShowFormDropDownButton.svelte b/packages/web/src/formview/ShowFormDropDownButton.svelte new file mode 100644 index 000000000..d0977aaf3 --- /dev/null +++ b/packages/web/src/formview/ShowFormDropDownButton.svelte @@ -0,0 +1,44 @@ + + +
+ +
+ + diff --git a/packages/web/src/icons/FontIcon.svelte b/packages/web/src/icons/FontIcon.svelte index c64332de1..61be8eadd 100644 --- a/packages/web/src/icons/FontIcon.svelte +++ b/packages/web/src/icons/FontIcon.svelte @@ -186,6 +186,16 @@ 'icon num-9-outline': 'mdi mdi-numeric-9-circle-outline', 'icon num-9-plus-outline': 'mdi mdi-numeric-9-plus-circle-outline', + 'icon type-string': 'mdi mdi-code-string', + 'icon type-object': 'mdi mdi-code-braces-box', + 'icon type-array': 'mdi mdi-code-array', + 'icon type-number': 'mdi mdi-pound-box', + 'icon type-boolean': 'mdi mdi-code-equal', + 'icon type-date': 'mdi mdi-alpha-d-box', + 'icon type-objectid': 'mdi mdi-alpha-i-box', + 'icon type-null': 'mdi mdi-code-equal', + 'icon type-unknown': 'mdi mdi-help-box', + 'img ok': 'mdi mdi-check-circle color-icon-green', 'img ok-inv': 'mdi mdi-check-circle color-icon-inv-green', 'img alert': 'mdi mdi-alert-circle color-icon-blue', diff --git a/packages/web/src/utility/clipboard.ts b/packages/web/src/utility/clipboard.ts index f1885763d..516d69071 100644 --- a/packages/web/src/utility/clipboard.ts +++ b/packages/web/src/utility/clipboard.ts @@ -1,6 +1,7 @@ import _ from 'lodash'; import { arrayToHexString, stringifyCellValue } from 'dbgate-tools'; import yaml from 'js-yaml'; +import { DataEditorTypesBehaviour } from 'dbgate-types'; export function copyTextToClipboard(text) { const oldFocus = document.activeElement; @@ -71,13 +72,13 @@ export async function getClipboardText() { return await navigator.clipboard.readText(); } -export function extractRowCopiedValue(row, col) { +export function extractRowCopiedValue(row, col, editorTypes?: DataEditorTypesBehaviour) { let value = row[col]; if (value === undefined) value = _.get(row, col); - return stringifyCellValue(value); + return stringifyCellValue(value, editorTypes); } -const clipboardHeadersFormatter = (delimiter) => (columns) => { +const clipboardHeadersFormatter = delimiter => columns => { return columns.join(delimiter); }; diff --git a/plugins/dbgate-plugin-mongo/src/frontend/driver.js b/plugins/dbgate-plugin-mongo/src/frontend/driver.js index b24d07d14..6eb0db144 100644 --- a/plugins/dbgate-plugin-mongo/src/frontend/driver.js +++ b/plugins/dbgate-plugin-mongo/src/frontend/driver.js @@ -122,6 +122,24 @@ const driver = { sort: convertToMongoSort(sort) || {}, }; }, + + dataEditorTypesBehaviour: { + parseJsonNull: true, + parseJsonBoolean: true, + parseNumber: true, + parseJsonArray: true, + parseJsonObject: true, + parseObjectIdAsDollar: true, + + explicitDataType: true, + supportNumberType: true, + supportStringType: true, + supportBooleanType: true, + supportDateType: true, + supportJsonType: true, + supportObjectIdType: true, + supportNullType: true, + }, }; module.exports = driver; From 3b813e93e751e91ef8bf8d7bad5cc2cf59824adf Mon Sep 17 00:00:00 2001 From: "SPRINX0\\prochazka" Date: Fri, 23 Aug 2024 16:19:04 +0200 Subject: [PATCH 02/12] cell display refactor --- packages/tools/src/stringTools.ts | 175 +++++++++++++++--- packages/web/src/datagrid/CellValue.svelte | 97 ++-------- packages/web/src/datagrid/DataGridCell.svelte | 2 +- packages/web/src/datagrid/InplaceInput.svelte | 4 +- .../web/src/datagrid/InplaceSelect.svelte | 37 ++-- .../web/src/elements/SelectionMapView.svelte | 2 +- .../src/perspectives/PerspectiveCell.svelte | 2 +- packages/web/src/utility/clipboard.ts | 2 +- 8 files changed, 189 insertions(+), 132 deletions(-) diff --git a/packages/tools/src/stringTools.ts b/packages/tools/src/stringTools.ts index 723e7b110..3b93522d2 100644 --- a/packages/tools/src/stringTools.ts +++ b/packages/tools/src/stringTools.ts @@ -2,8 +2,12 @@ import _isString from 'lodash/isString'; import _isArray from 'lodash/isArray'; import _isNumber from 'lodash/isNumber'; import _isPlainObject from 'lodash/isPlainObject'; +import _pad from 'lodash/pad'; import { DataEditorTypesBehaviour } from 'dbgate-types'; +const dateTimeRegex = + /^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?(([Zz])|()|([\+|\-]([01][0-9]|2[0-3]):[0-5][0-9]))$/; + export function arrayToHexString(byteArray) { return byteArray.reduce((output, elem) => output + ('0' + elem.toString(16)).slice(-2), '').toUpperCase(); } @@ -77,41 +81,170 @@ function parseObjectIdAsDollar(value) { return value; } -export function stringifyCellValue(value, editorTypes?: DataEditorTypesBehaviour) { +function makeBulletString(value) { + return _pad('', value.length, '•'); +} + +function highlightSpecialCharacters(value) { + value = value.replace(/\n/g, '↲'); + value = value.replace(/\r/g, ''); + value = value.replace(/^(\s+)/, makeBulletString); + value = value.replace(/(\s+)$/, makeBulletString); + value = value.replace(/(\s\s+)/g, makeBulletString); + return value; +} + +function stringifyJsonToGrid(value): ReturnType { + if (_isPlainObject(value)) { + const svalue = JSON.stringify(value, undefined, 2); + if (svalue.length < 100) { + return { value: svalue, gridStyle: 'nullCellStyle' }; + } else { + return { value: '(JSON)', gridStyle: 'nullCellStyle', gridTitle: svalue }; + } + } + if (_isArray(value)) { + return { + value: `[${value.length} items]`, + gridStyle: 'nullCellStyle', + gridTitle: value.map(x => JSON.stringify(x)).join('\n'), + }; + } + return { value: '(JSON)', gridStyle: 'nullCellStyle' }; +} + +export function stringifyCellValue( + value, + intent: 'gridCellIntent' | 'inlineEditorIntent' | 'multilineEditorIntent' | 'stringConversionIntent' | 'exportIntent', + editorTypes?: DataEditorTypesBehaviour, + gridFormattingOptions?: { useThousandsSeparator?: boolean }, + jsonParsedValue?: any +): { + value: string; + gridStyle?: 'textCellStyle' | 'valueCellStyle' | 'nullCellStyle'; // only for gridCellIntent + gridTitle?: string; // only for gridCellIntent +} { if (editorTypes?.parseSqlNull) { - if (value === null) return '(NULL)'; + if (value === null) { + switch (intent) { + case 'exportIntent': + return { value: '' }; + default: + return { value: '(NULL)', gridStyle: 'nullCellStyle' }; + } + } + } + if (value === undefined) { + switch (intent) { + case 'gridCellIntent': + return { value: '(No Field)', gridStyle: 'nullCellStyle' }; + default: + return { value: '' }; + } } - if (value === undefined) return '(NoField)'; if (editorTypes?.parseJsonNull) { - if (value === null) return 'null'; - } - if (editorTypes?.parseJsonBoolean) { - if (value === true) return 'true'; - if (value === false) return 'false'; + if (value === null) { + return { value: 'null', gridStyle: 'valueCellStyle' }; + } } + + if (value === true) return { value: 'true', gridStyle: 'valueCellStyle' }; + if (value === false) return { value: 'false', gridStyle: 'valueCellStyle' }; + if (editorTypes?.parseHexAsBuffer) { - if (value?.type == 'Buffer' && _isArray(value.data)) return '0x' + arrayToHexString(value.data); + if (value?.type == 'Buffer' && _isArray(value.data)) { + return { value: '0x' + arrayToHexString(value.data), gridStyle: 'valueCellStyle' }; + } } if (editorTypes?.parseObjectIdAsDollar) { - if (value?.$oid) return `ObjectId("${value?.$oid}")`; + if (value?.$oid) { + switch (intent) { + case 'exportIntent': + case 'stringConversionIntent': + return { value: value.$oid }; + default: + return { value: `ObjectId("${value.$oid}")`, gridStyle: 'valueCellStyle' }; + } + } } if (editorTypes?.parseJsonArray) { - if (_isArray(value)) return JSON.stringify(value); + if (_isArray(value)) { + switch (intent) { + case 'gridCellIntent': + return stringifyJsonToGrid(value); + case 'multilineEditorIntent': + return { value: JSON.stringify(value) }; + default: + return { value: JSON.stringify(value, null, 2), gridStyle: 'valueCellStyle' }; + } + } } if (editorTypes?.parseJsonObject) { - if (_isPlainObject(value)) return JSON.stringify(value); - } - if (editorTypes?.parseNumber) { - if (_isNumber(value)) return value.toString(); + if (_isPlainObject(value)) { + switch (intent) { + case 'gridCellIntent': + return stringifyJsonToGrid(value); + case 'multilineEditorIntent': + return { value: JSON.stringify(value) }; + default: + return { value: JSON.stringify(value, null, 2), gridStyle: 'valueCellStyle' }; + } + } } - if (_isString(value)) return value; + if (_isNumber(value)) { + switch (intent) { + case 'gridCellIntent': + return { + value: + gridFormattingOptions?.useThousandsSeparator && (value >= 10000 || value <= -10000) + ? value.toLocaleString() + : value.toString(), + gridStyle: 'valueCellStyle', + }; + default: + return { value: value.toString() }; + } + } - // fallback - if (_isNumber(value)) return value.toString(); - if (value === null || value === undefined) return ''; + if (_isString(value)) { + switch (intent) { + case 'gridCellIntent': + if (jsonParsedValue && !editorTypes?.explicitDataType) { + return stringifyJsonToGrid(jsonParsedValue); + } else { + if (!editorTypes?.explicitDataType) { + // reformat datetime for implicit date types + const m = value.match(dateTimeRegex); + if (m) { + return { + value: `${m[1]}-${m[2]}-${m[3]} ${m[4]}:${m[5]}:${m[6]}`, + gridStyle: 'valueCellStyle', + }; + } + } + return { value: highlightSpecialCharacters(value), gridStyle: 'textCellStyle' }; + } + default: + return { value: value }; + } + } - return ''; + if (value === null || value === undefined) { + switch (intent) { + case 'gridCellIntent': + return { value: '(n/a)', gridStyle: 'nullCellStyle' }; + default: + return { value: '' }; + } + } + + switch (intent) { + case 'gridCellIntent': + return { value: '(Unknown)', gridStyle: 'nullCellStyle' }; + default: + return { value: '' }; + } } export function safeJsonParse(json, defaultValue?, logError = false) { @@ -212,7 +345,7 @@ export function getConvertValueMenu(value, onSetValue, editorTypes?: DataEditorT return [ editorTypes?.supportStringType && { text: 'String', - onClick: () => onSetValue(stringifyCellValue(value, editorTypes)), + onClick: () => onSetValue(stringifyCellValue(value, 'stringConversionIntent', editorTypes).value), }, editorTypes?.supportNumberType && { text: 'Number', onClick: () => onSetValue(parseFloat(value)) }, editorTypes?.supportNullType && { text: 'Null', onClick: () => onSetValue(null) }, diff --git a/packages/web/src/datagrid/CellValue.svelte b/packages/web/src/datagrid/CellValue.svelte index e2e5eb1f2..7b6bcb7a1 100644 --- a/packages/web/src/datagrid/CellValue.svelte +++ b/packages/web/src/datagrid/CellValue.svelte @@ -1,105 +1,34 @@ - - {#if rowData == null} (No row) -{:else if value === null} - (NULL) -{:else if value === undefined} - (No field) -{:else if _.isDate(value)} - {value.toString()} -{:else if value === true} - true -{:else if value === false} - false -{:else if _.isNumber(value)} - {formatNumber(value)} -{:else if _.isString(value) && !jsonParsedValue} - {#if dateTimeRegex.test(value)} - - {formatDateTime(value)} - - {:else} - {highlightSpecialCharacters(value)} - {/if} -{:else if value?.type == 'Buffer' && _.isArray(value.data)} - {#if value.data.length <= 16} - {'0x' + arrayToHexString(value.data)} - {:else} - ({value.data.length} bytes) - {/if} -{:else if value.$oid} - ObjectId("{value.$oid}") -{:else if _.isPlainObject(value)} - {@const svalue = JSON.stringify(value, undefined, 2)} - {#if svalue.length < 100}{JSON.stringify(value)}{:else}(JSON){/if} -{:else if _.isArray(value)} - JSON.stringify(x)).join('\n')}>[{value.length} items] -{:else if _.isPlainObject(jsonParsedValue)} - {@const svalue = JSON.stringify(jsonParsedValue, undefined, 2)} - {#if svalue.length < 100}{JSON.stringify(jsonParsedValue)}{:else}(JSON){/if} -{:else if _.isArray(jsonParsedValue)} - JSON.stringify(x)).join('\n')} - >[{jsonParsedValue.length} items] {:else} - {value.toString()} + {stringified.value} {/if} diff --git a/packages/web/src/datagrid/DataGridCell.svelte b/packages/web/src/datagrid/DataGridCell.svelte index cb49efa79..f6d9b80e7 100644 --- a/packages/web/src/datagrid/DataGridCell.svelte +++ b/packages/web/src/datagrid/DataGridCell.svelte @@ -72,7 +72,7 @@ class:isFocusedColumn {style} > - + {#if allowHintField && rowData && _.some(col.hintColumnNames, hintColumnName => rowData[hintColumnName])} { - domEditor.value = inplaceEditorState.text || stringifyCellValue(cellValue, editorTypes); + domEditor.value = inplaceEditorState.text || stringifyCellValue(cellValue, 'inlineEditorIntent', editorTypes).value; domEditor.focus(); if (inplaceEditorState.selectAll) { domEditor.select(); @@ -105,7 +105,7 @@ dispatchInsplaceEditor({ type: 'close' }); showModal(EditCellDataModal, { - value: stringifyCellValue(cellValue, editorTypes), + value: stringifyCellValue(cellValue, 'multilineEditorIntent', editorTypes).value, onSave: onSetValue, }); }} diff --git a/packages/web/src/datagrid/InplaceSelect.svelte b/packages/web/src/datagrid/InplaceSelect.svelte index a1d58a45f..b95254fc9 100644 --- a/packages/web/src/datagrid/InplaceSelect.svelte +++ b/packages/web/src/datagrid/InplaceSelect.svelte @@ -19,22 +19,22 @@ let isOptionsHidden = false; onMount(() => { - value = inplaceEditorState.text || stringifyCellValue(cellValue, driver?.dataEditorTypesBehaviour); + value = + inplaceEditorState.text || stringifyCellValue(cellValue, 'inlineEditorIntent', driver?.dataEditorTypesBehaviour).value; valueInit = value; const optionsSelected = value.split(','); - optionsData = options - .map(function(option) { - return { - value: option, - isSelected: optionsSelected.includes(option) - }; - }); + optionsData = options.map(function (option) { + return { + value: option, + isSelected: optionsSelected.includes(option), + }; + }); }); function handleCheckboxChanged(e, option) { if (!canSelectMultipleOptions) { - optionsData.forEach(option => option.isSelected = false); + optionsData.forEach(option => (option.isSelected = false)); option.isSelected = true; } else { option.isSelected = e.target.checked; @@ -45,8 +45,7 @@ .map(option => option.value) .join(','); - if(!canSelectMultipleOptions) - handleConfirm(); + if (!canSelectMultipleOptions) handleConfirm(); } function handleConfirm() { @@ -70,13 +69,8 @@ } -
-
isOptionsHidden = !isOptionsHidden} class="value"> +
+
(isOptionsHidden = !isOptionsHidden)} class="value"> {value}
@@ -89,11 +83,12 @@
{#each optionsData ?? [] as option} {/each} @@ -113,7 +108,7 @@ background-color: var(--theme-bg-alt); max-height: 150px; overflow: auto; - box-shadow: 0 1px 10px 1px var(--theme-bg-inv-3);; + box-shadow: 0 1px 10px 1px var(--theme-bg-inv-3); } .value { diff --git a/packages/web/src/elements/SelectionMapView.svelte b/packages/web/src/elements/SelectionMapView.svelte index ec64fde5c..60ca253b0 100644 --- a/packages/web/src/elements/SelectionMapView.svelte +++ b/packages/web/src/elements/SelectionMapView.svelte @@ -37,7 +37,7 @@ function createColumnsTable(cells) { if (cells.length == 0) return ''; return `${cells - .map(cell => ``) + .map(cell => ``) .join('\n')}
${cell.column}${stringifyCellValue(cell.value)}
${cell.column}${stringifyCellValue(cell.value, 'exportIntent').value}
`; } diff --git a/packages/web/src/perspectives/PerspectiveCell.svelte b/packages/web/src/perspectives/PerspectiveCell.svelte index e86579cd9..f72f3968d 100644 --- a/packages/web/src/perspectives/PerspectiveCell.svelte +++ b/packages/web/src/perspectives/PerspectiveCell.svelte @@ -15,7 +15,7 @@ if (force && value?.type == 'Buffer' && _.isArray(value.data)) { return String.fromCharCode.apply(String, value.data); } - return stringifyCellValue(value); + return stringifyCellValue(value, 'gridCellIntent').value; } diff --git a/packages/web/src/utility/clipboard.ts b/packages/web/src/utility/clipboard.ts index 516d69071..7adf11eae 100644 --- a/packages/web/src/utility/clipboard.ts +++ b/packages/web/src/utility/clipboard.ts @@ -75,7 +75,7 @@ export async function getClipboardText() { export function extractRowCopiedValue(row, col, editorTypes?: DataEditorTypesBehaviour) { let value = row[col]; if (value === undefined) value = _.get(row, col); - return stringifyCellValue(value, editorTypes); + return stringifyCellValue(value, 'exportIntent', editorTypes).value; } const clipboardHeadersFormatter = delimiter => columns => { From eaa943a39d005b17b70656cfe096f361aa8dd8ae Mon Sep 17 00:00:00 2001 From: "SPRINX0\\prochazka" Date: Fri, 23 Aug 2024 16:27:36 +0200 Subject: [PATCH 03/12] mongo - using ejson --- plugins/dbgate-plugin-mongo/package.json | 11 ++++++----- plugins/dbgate-plugin-mongo/src/backend/driver.js | 15 ++++----------- yarn.lock | 5 +++++ 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/plugins/dbgate-plugin-mongo/package.json b/plugins/dbgate-plugin-mongo/package.json index 1a0b80879..3d28ffd57 100644 --- a/plugins/dbgate-plugin-mongo/package.json +++ b/plugins/dbgate-plugin-mongo/package.json @@ -31,14 +31,15 @@ "prepublishOnly": "yarn build" }, "devDependencies": { + "bson": "^6.8.0", "dbgate-plugin-tools": "^1.0.7", "dbgate-query-splitter": "^4.10.1", - "lodash": "^4.17.21", - "webpack": "^5.91.0", - "webpack-cli": "^5.1.4", "dbgate-tools": "^5.0.0-alpha.1", "is-promise": "^4.0.0", + "lodash": "^4.17.21", "mongodb": "^6.3.0", - "mongodb-client-encryption": "^6.0.0" + "mongodb-client-encryption": "^6.0.0", + "webpack": "^5.91.0", + "webpack-cli": "^5.1.4" } -} \ No newline at end of file +} diff --git a/plugins/dbgate-plugin-mongo/src/backend/driver.js b/plugins/dbgate-plugin-mongo/src/backend/driver.js index f7b1aec2e..9a91dcca7 100644 --- a/plugins/dbgate-plugin-mongo/src/backend/driver.js +++ b/plugins/dbgate-plugin-mongo/src/backend/driver.js @@ -3,9 +3,8 @@ const stream = require('stream'); const isPromise = require('is-promise'); const driverBase = require('../frontend/driver'); const Analyser = require('./Analyser'); -const MongoClient = require('mongodb').MongoClient; -const ObjectId = require('mongodb').ObjectId; -const AbstractCursor = require('mongodb').AbstractCursor; +const { MongoClient, ObjectId, AbstractCursor } = require('mongodb'); +const { EJSON } = require('bson'); const createBulkInsertStream = require('./createBulkInsertStream'); const { convertToMongoCondition, @@ -14,9 +13,7 @@ const { } = require('../frontend/convertToMongoCondition'); function transformMongoData(row) { - return _.cloneDeepWith(row, (x) => { - if (x && x.constructor == ObjectId) return { $oid: x.toString() }; - }); + return EJSON.serialize(row); } async function readCursor(cursor, options) { @@ -27,11 +24,7 @@ async function readCursor(cursor, options) { } function convertObjectId(condition) { - return _.cloneDeepWith(condition, (x) => { - if (x && x.$oid) { - return ObjectId.createFromHexString(x.$oid); - } - }); + return EJSON.deserialize(condition); } function findArrayResult(resValue) { diff --git a/yarn.lock b/yarn.lock index 817e109d5..84be0550e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2458,6 +2458,11 @@ bson@^6.7.0: resolved "https://registry.yarnpkg.com/bson/-/bson-6.7.0.tgz#51973b132cdc424c8372fda3cb43e3e3e2ae2227" integrity sha512-w2IquM5mYzYZv6rs3uN2DZTOBe2a0zXLj53TGDqwF4l6Sz/XsISrisXOJihArF9+BZ6Cq/GjVht7Sjfmri7ytQ== +bson@^6.8.0: + version "6.8.0" + resolved "https://registry.yarnpkg.com/bson/-/bson-6.8.0.tgz#5063c41ba2437c2b8ff851b50d9e36cb7aaa7525" + integrity sha512-iOJg8pr7wq2tg/zSlCCHMi3hMm5JTOxLTagf3zxhcenHsFp+c6uOs6K7W5UE7A4QIJGtqh/ZovFNMP4mOPJynQ== + buffer-crc32@^0.2.5: version "0.2.13" resolved "https://registry.yarnpkg.com/buffer-crc32/-/buffer-crc32-0.2.13.tgz#0d333e3f00eac50aa1454abd30ef8c2a5d9a7242" From 4cbfa7c937a05447c0523c6d51640b4c9b2e3dc7 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Mon, 26 Aug 2024 09:32:28 +0200 Subject: [PATCH 04/12] fixes --- packages/tools/src/stringTools.ts | 40 +++++++++---------- packages/web/src/datagrid/DataGridCell.svelte | 3 -- packages/web/src/datagrid/DataGridCore.svelte | 6 +-- packages/web/src/formview/FormView.svelte | 4 +- .../web/src/modals/EditCellDataModal.svelte | 9 +++++ 5 files changed, 33 insertions(+), 29 deletions(-) diff --git a/packages/tools/src/stringTools.ts b/packages/tools/src/stringTools.ts index 3b93522d2..f2eeacff9 100644 --- a/packages/tools/src/stringTools.ts +++ b/packages/tools/src/stringTools.ts @@ -150,7 +150,7 @@ export function stringifyCellValue( if (value === true) return { value: 'true', gridStyle: 'valueCellStyle' }; if (value === false) return { value: 'false', gridStyle: 'valueCellStyle' }; - + if (editorTypes?.parseHexAsBuffer) { if (value?.type == 'Buffer' && _isArray(value.data)) { return { value: '0x' + arrayToHexString(value.data), gridStyle: 'valueCellStyle' }; @@ -167,28 +167,26 @@ export function stringifyCellValue( } } } - if (editorTypes?.parseJsonArray) { - if (_isArray(value)) { - switch (intent) { - case 'gridCellIntent': - return stringifyJsonToGrid(value); - case 'multilineEditorIntent': - return { value: JSON.stringify(value) }; - default: - return { value: JSON.stringify(value, null, 2), gridStyle: 'valueCellStyle' }; - } + + if (_isArray(value)) { + switch (intent) { + case 'gridCellIntent': + return stringifyJsonToGrid(value); + case 'multilineEditorIntent': + return { value: JSON.stringify(value, null, 2) }; + default: + return { value: JSON.stringify(value), gridStyle: 'valueCellStyle' }; } } - if (editorTypes?.parseJsonObject) { - if (_isPlainObject(value)) { - switch (intent) { - case 'gridCellIntent': - return stringifyJsonToGrid(value); - case 'multilineEditorIntent': - return { value: JSON.stringify(value) }; - default: - return { value: JSON.stringify(value, null, 2), gridStyle: 'valueCellStyle' }; - } + + if (_isPlainObject(value)) { + switch (intent) { + case 'gridCellIntent': + return stringifyJsonToGrid(value); + case 'multilineEditorIntent': + return { value: JSON.stringify(value, null, 2) }; + default: + return { value: JSON.stringify(value), gridStyle: 'valueCellStyle' }; } } diff --git a/packages/web/src/datagrid/DataGridCell.svelte b/packages/web/src/datagrid/DataGridCell.svelte index f6d9b80e7..54825b543 100644 --- a/packages/web/src/datagrid/DataGridCell.svelte +++ b/packages/web/src/datagrid/DataGridCell.svelte @@ -3,10 +3,7 @@ import ShowFormButton from '../formview/ShowFormButton.svelte'; import { detectTypeIcon, getConvertValueMenu, isJsonLikeLongString, safeJsonParse } from 'dbgate-tools'; import { openJsonDocument } from '../tabs/JsonTab.svelte'; - import openNewTab from '../utility/openNewTab'; import CellValue from './CellValue.svelte'; - import { showModal } from '../modals/modalTools'; - import EditCellDataModal from '../modals/EditCellDataModal.svelte'; import { openJsonLinesData } from '../utility/openJsonLinesData'; import ShowFormDropDownButton from '../formview/ShowFormDropDownButton.svelte'; diff --git a/packages/web/src/datagrid/DataGridCore.svelte b/packages/web/src/datagrid/DataGridCore.svelte index fc5031724..c8a637957 100644 --- a/packages/web/src/datagrid/DataGridCore.svelte +++ b/packages/web/src/datagrid/DataGridCore.svelte @@ -343,7 +343,7 @@ From 60bf682449ae839c4c65d8bc2ed4b5929113b02f Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Mon, 26 Aug 2024 09:58:09 +0200 Subject: [PATCH 05/12] multiline dialog fixes --- packages/tools/src/stringTools.ts | 22 ++++++++++++ packages/web/src/datagrid/DataGridCore.svelte | 16 ++++++--- packages/web/src/datagrid/InplaceInput.svelte | 3 +- packages/web/src/formview/FormView.svelte | 7 ++-- .../web/src/modals/EditCellDataModal.svelte | 35 ++++--------------- 5 files changed, 47 insertions(+), 36 deletions(-) diff --git a/packages/tools/src/stringTools.ts b/packages/tools/src/stringTools.ts index f2eeacff9..7f2a47015 100644 --- a/packages/tools/src/stringTools.ts +++ b/packages/tools/src/stringTools.ts @@ -259,6 +259,28 @@ export function safeJsonParse(json, defaultValue?, logError = false) { } } +export function shouldOpenMultilineDialog(value) { + if (_isString(value)) { + if (value.includes('\n')) { + return true; + } + const parsed = safeJsonParse(value); + if (parsed && (_isPlainObject(parsed) || _isArray(parsed))) { + return true; + } + } + if (value?.$oid) { + return false; + } + if (value?.$date) { + return false; + } + if (_isPlainObject(value) || _isArray(value)) { + return true; + } + return false; +} + export function isJsonLikeLongString(value) { return _isString(value) && value.length > 100 && value.match(/^\s*\{.*\}\s*$|^\s*\[.*\]\s*$/); } diff --git a/packages/web/src/datagrid/DataGridCore.svelte b/packages/web/src/datagrid/DataGridCore.svelte index c8a637957..d3034d9dd 100644 --- a/packages/web/src/datagrid/DataGridCore.svelte +++ b/packages/web/src/datagrid/DataGridCore.svelte @@ -343,7 +343,13 @@ -
@@ -23,6 +25,10 @@ border: 1px solid var(--theme-bg-1); } + .secondary { + margin-right: 20px; + } + div:hover { color: var(--theme-font-hover); border: var(--theme-border); From 8e17516d54d252e71773349fecb63634a8021469 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Mon, 26 Aug 2024 11:33:29 +0200 Subject: [PATCH 07/12] support date type --- packages/tools/src/stringTools.ts | 103 +++++++++++++++--- packages/types/engines.d.ts | 1 + .../src/frontend/driver.js | 1 + 3 files changed, 92 insertions(+), 13 deletions(-) diff --git a/packages/tools/src/stringTools.ts b/packages/tools/src/stringTools.ts index 7f2a47015..05e4fa423 100644 --- a/packages/tools/src/stringTools.ts +++ b/packages/tools/src/stringTools.ts @@ -5,9 +5,22 @@ import _isPlainObject from 'lodash/isPlainObject'; import _pad from 'lodash/pad'; import { DataEditorTypesBehaviour } from 'dbgate-types'; -const dateTimeRegex = +export type EditorDataType = + | 'null' + | 'objectid' + | 'string' + | 'number' + | 'object' + | 'date' + | 'array' + | 'boolean' + | 'unknown'; + +const dateTimeStorageRegex = /^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?(([Zz])|()|([\+|\-]([01][0-9]|2[0-3]):[0-5][0-9]))$/; +const dateTimeParseRegex = /^(\d{4})-(\d{2})-(\d{2})[Tt ](\d{2}):(\d{2}):(\d{2})(\.[0-9]+)?(([Zz])|()|([\+|\-]([01][0-9]|2[0-3]):[0-5][0-9]))$/; + export function arrayToHexString(byteArray) { return byteArray.reduce((output, elem) => output + ('0' + elem.toString(16)).slice(-2), '').toUpperCase(); } @@ -45,6 +58,15 @@ export function parseCellValue(value, editorTypes?: DataEditorTypesBehaviour) { } } + if (editorTypes?.parseDateAsDollar) { + const m = value.match(dateTimeParseRegex); + if (m) { + return { + $date: `${m[1]}-${m[2]}-${m[3]}T${m[4]}:${m[5]}:${m[6]}Z`, + }; + } + } + if (editorTypes?.parseJsonNull) { if (value == 'null') return null; } @@ -69,7 +91,7 @@ export function parseCellValue(value, editorTypes?: DataEditorTypesBehaviour) { return value; } -function parseObjectIdAsDollar(value) { +function parseFunc_ObjectIdAsDollar(value) { if (value?.$oid) return value; if (_isString(value)) { if (value.match(/^[0-9a-f]{24}$/)) return { $oid: value }; @@ -81,6 +103,17 @@ function parseObjectIdAsDollar(value) { return value; } +function parseFunc_DateAsDollar(value) { + if (value?.$date) return value; + if (_isString(value)) { + const m = value.match(dateTimeParseRegex); + if (m) { + return { $date: `${m[1]}-${m[2]}-${m[3]}T${m[4]}:${m[5]}:${m[6]}Z` }; + } + } + return value; +} + function makeBulletString(value) { return _pad('', value.length, '•'); } @@ -168,6 +201,23 @@ export function stringifyCellValue( } } + if (editorTypes?.parseDateAsDollar) { + if (value?.$date) { + switch (intent) { + case 'exportIntent': + case 'stringConversionIntent': + return { value: value.$date }; + default: + const m = value.$date.match(dateTimeStorageRegex); + if (m) { + return { value: `${m[1]}-${m[2]}-${m[3]} ${m[4]}:${m[5]}:${m[6]}`, gridStyle: 'valueCellStyle' }; + } else { + return { value: value.$date.replaCE('T', ' '), gridStyle: 'valueCellStyle' }; + } + } + } + } + if (_isArray(value)) { switch (intent) { case 'gridCellIntent': @@ -213,7 +263,7 @@ export function stringifyCellValue( } else { if (!editorTypes?.explicitDataType) { // reformat datetime for implicit date types - const m = value.match(dateTimeRegex); + const m = value.match(dateTimeStorageRegex); if (m) { return { value: `${m[1]}-${m[2]}-${m[3]} ${m[4]}:${m[5]}:${m[6]}`, @@ -350,15 +400,39 @@ export function parseSqlDefaultValue(value: string) { return undefined; } +export function detectCellDataType(value): EditorDataType { + if (value === null) return 'null'; + if (value?.$oid) return 'objectid'; + if (value?.$date) return 'date'; + if (_isString(value)) return 'string'; + if (_isNumber(value)) return 'number'; + if (_isPlainObject(value)) return 'object'; + if (_isArray(value)) return 'array'; + if (value === true || value === false) return 'boolean'; + return 'unknown'; +} + export function detectTypeIcon(value) { - if (value === null) return 'icon type-null'; - if (value?.$oid) return 'icon type-objectid'; - if (_isString(value)) return 'icon type-string'; - if (_isNumber(value)) return 'icon type-number'; - if (_isPlainObject(value)) return 'icon type-object'; - if (_isArray(value)) return 'icon type-array'; - if (value === true || value === false) return 'icon type-boolean'; - return 'icon type-unknown'; + switch (detectCellDataType(value)) { + case 'null': + return 'icon type-null'; + case 'objectid': + return 'icon type-objectid'; + case 'date': + return 'icon type-date'; + case 'string': + return 'icon type-string'; + case 'number': + return 'icon type-number'; + case 'object': + return 'icon type-object'; + case 'array': + return 'icon type-array'; + case 'boolean': + return 'icon type-boolean'; + default: + return 'icon type-unknown'; + } } export function getConvertValueMenu(value, onSetValue, editorTypes?: DataEditorTypesBehaviour) { @@ -373,13 +447,16 @@ export function getConvertValueMenu(value, onSetValue, editorTypes?: DataEditorT text: 'Boolean', onClick: () => onSetValue(value?.toString()?.toLowerCase() == 'true' || value == '1'), }, - editorTypes?.supportObjectIdType && { text: 'ObjectId', onClick: () => onSetValue(parseObjectIdAsDollar(value)) }, + editorTypes?.supportObjectIdType && { + text: 'ObjectId', + onClick: () => onSetValue(parseFunc_ObjectIdAsDollar(value)), + }, + editorTypes?.supportDateType && { text: 'Date', onClick: () => onSetValue(parseFunc_DateAsDollar(value)) }, editorTypes?.supportJsonType && { text: 'JSON', onClick: () => { const jsonValue = safeJsonParse(value); if (jsonValue != null) { - console.log('**** ON SET VALUE', jsonValue); onSetValue(jsonValue); } }, diff --git a/packages/types/engines.d.ts b/packages/types/engines.d.ts index f6008c4a8..c58ca18ff 100644 --- a/packages/types/engines.d.ts +++ b/packages/types/engines.d.ts @@ -102,6 +102,7 @@ export interface DataEditorTypesBehaviour { parseJsonObject?: boolean; parseHexAsBuffer?: boolean; parseObjectIdAsDollar?: boolean; + parseDateAsDollar?: boolean; explicitDataType?: boolean; supportNumberType?: boolean; diff --git a/plugins/dbgate-plugin-mongo/src/frontend/driver.js b/plugins/dbgate-plugin-mongo/src/frontend/driver.js index 6eb0db144..5e0fdd7cf 100644 --- a/plugins/dbgate-plugin-mongo/src/frontend/driver.js +++ b/plugins/dbgate-plugin-mongo/src/frontend/driver.js @@ -130,6 +130,7 @@ const driver = { parseJsonArray: true, parseJsonObject: true, parseObjectIdAsDollar: true, + parseDateAsDollar: true, explicitDataType: true, supportNumberType: true, From 32ebd86171d4fd13f61eccf7a89ff2bb39346a06 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Mon, 26 Aug 2024 13:25:51 +0200 Subject: [PATCH 08/12] support remove fields for mongo --- packages/types/engines.d.ts | 2 ++ packages/web/src/datagrid/DataGridCore.svelte | 20 +++++++++++++++-- packages/web/src/formview/FormView.svelte | 22 +++++++++++++++---- .../web/src/tabs/CollectionDataTab.svelte | 8 ++++++- .../dbgate-plugin-mongo/src/backend/driver.js | 9 +++++++- .../src/frontend/driver.js | 11 +++++++++- 6 files changed, 63 insertions(+), 9 deletions(-) diff --git a/packages/types/engines.d.ts b/packages/types/engines.d.ts index c58ca18ff..dea2c0409 100644 --- a/packages/types/engines.d.ts +++ b/packages/types/engines.d.ts @@ -112,6 +112,8 @@ export interface DataEditorTypesBehaviour { supportNullType?: boolean; supportJsonType?: boolean; supportObjectIdType?: boolean; + + supportFieldRemoval?: boolean; } export interface FilterBehaviourProvider { diff --git a/packages/web/src/datagrid/DataGridCore.svelte b/packages/web/src/datagrid/DataGridCore.svelte index d3034d9dd..a608cff9f 100644 --- a/packages/web/src/datagrid/DataGridCore.svelte +++ b/packages/web/src/datagrid/DataGridCore.svelte @@ -81,10 +81,21 @@ category: 'Data grid', name: 'Set NULL', keyText: 'CtrlOrCommand+0', - testEnabled: () => getCurrentDataGrid()?.getGrider()?.editable, + testEnabled: () => + getCurrentDataGrid()?.getGrider()?.editable && !getCurrentDataGrid()?.getEditorTypes()?.supportFieldRemoval, onClick: () => getCurrentDataGrid().setFixedValue(null), }); + registerCommand({ + id: 'dataGrid.removeField', + category: 'Data grid', + name: 'Remove field', + keyText: 'CtrlOrCommand+0', + testEnabled: () => + getCurrentDataGrid()?.getGrider()?.editable && getCurrentDataGrid()?.getEditorTypes()?.supportFieldRemoval, + onClick: () => getCurrentDataGrid().setFixedValue(undefined), + }); + registerCommand({ id: 'dataGrid.undo', category: 'Data grid', @@ -823,6 +834,10 @@ }); } + export function getEditorTypes() { + return display?.driver?.dataEditorTypesBehaviour; + } + export function addJsonDocumentEnabled() { return grider.editable; } @@ -1703,7 +1718,8 @@ { command: 'dataGrid.deleteSelectedRows' }, { command: 'dataGrid.insertNewRow' }, { command: 'dataGrid.cloneRows' }, - { command: 'dataGrid.setNull' }, + { command: 'dataGrid.setNull', hideDisabled: true }, + { command: 'dataGrid.removeField', hideDisabled: true }, { placeTag: 'edit' }, { divider: true }, { command: 'dataGrid.findColumn' }, diff --git a/packages/web/src/formview/FormView.svelte b/packages/web/src/formview/FormView.svelte index 32b51f156..7dce34966 100644 --- a/packages/web/src/formview/FormView.svelte +++ b/packages/web/src/formview/FormView.svelte @@ -48,10 +48,19 @@ category: 'Data form', name: 'Set NULL', keyText: 'CtrlOrCommand+0', - testEnabled: () => getCurrentDataForm() != null, + testEnabled: () => getCurrentDataForm() != null && !getCurrentDataForm()?.getEditorTypes()?.supportFieldRemoval, onClick: () => getCurrentDataForm().setFixedValue(null), }); + registerCommand({ + id: 'dataForm.removeField', + category: 'Data form', + name: 'Remove field', + keyText: 'CtrlOrCommand+0', + testEnabled: () => getCurrentDataForm() != null && getCurrentDataForm()?.getEditorTypes()?.supportFieldRemoval, + onClick: () => getCurrentDataForm().setFixedValue(undefined), + }); + registerCommand({ id: 'dataForm.undo', category: 'Data form', @@ -321,6 +330,10 @@ export const activator = createActivator('FormView', false); + export function getEditorTypes() { + return display?.driver?.dataEditorTypesBehaviour; + } + const handleTableMouseDown = event => { if (event.target.closest('.buttonLike')) return; if (event.target.closest('.resizeHandleControl')) return; @@ -411,10 +424,11 @@ { divider: true }, { placeTag: 'save' }, { command: 'dataForm.revertRowChanges' }, - { command: 'dataForm.setNull' }, + { command: 'dataForm.setNull', hideDisabled: true }, + { command: 'dataForm.removeField', hideDisabled: true }, { divider: true }, - { command: 'dataForm.undo' }, - { command: 'dataForm.redo' }, + { command: 'dataForm.undo', hideDisabled: true }, + { command: 'dataForm.redo', hideDisabled: true }, { divider: true }, { command: 'dataForm.goToFirst' }, { command: 'dataForm.goToPrevious' }, diff --git a/packages/web/src/tabs/CollectionDataTab.svelte b/packages/web/src/tabs/CollectionDataTab.svelte index 2dd8fb6d6..4417742d0 100644 --- a/packages/web/src/tabs/CollectionDataTab.svelte +++ b/packages/web/src/tabs/CollectionDataTab.svelte @@ -121,7 +121,13 @@ const resp = await apiCall('database-connections/update-collection', { conid, database, - changeSet, + changeSet: { + ...changeSet, + updates: changeSet.updates.map(update => ({ + ...update, + fields: _.mapValues(update.fields, (v, k) => (v === undefined ? { $undefined: true } : v)), + })), + }, }); const { errorMessage } = resp || {}; if (errorMessage) { diff --git a/plugins/dbgate-plugin-mongo/src/backend/driver.js b/plugins/dbgate-plugin-mongo/src/backend/driver.js index 9a91dcca7..58f0a5b6d 100644 --- a/plugins/dbgate-plugin-mongo/src/backend/driver.js +++ b/plugins/dbgate-plugin-mongo/src/backend/driver.js @@ -330,7 +330,14 @@ const driver = { } } else { const resdoc = await collection.updateOne(convertObjectId(update.condition), { - $set: convertObjectId(update.fields), + $set: convertObjectId(_.pickBy(update.fields, (v, k) => !v?.$undefined)), + $unset: _.fromPairs( + Object.keys(update.fields) + .filter((k) => update.fields[k]?.$undefined) + .map((k) => [k, '']) + ), + + // $set: convertObjectId(update.fields), }); res.updated.push(resdoc._id); } diff --git a/plugins/dbgate-plugin-mongo/src/frontend/driver.js b/plugins/dbgate-plugin-mongo/src/frontend/driver.js index 5e0fdd7cf..4f3f1bfea 100644 --- a/plugins/dbgate-plugin-mongo/src/frontend/driver.js +++ b/plugins/dbgate-plugin-mongo/src/frontend/driver.js @@ -2,6 +2,8 @@ const { driverBase } = global.DBGATE_PACKAGES['dbgate-tools']; const { convertToMongoCondition, convertToMongoSort } = require('./convertToMongoCondition'); const Dumper = require('./Dumper'); const { mongoSplitterOptions } = require('dbgate-query-splitter/lib/options'); +const _pickBy = require('lodash/pickBy'); +const _fromPairs = require('lodash/fromPairs'); function jsonStringifyWithObjectId(obj) { return JSON.stringify(obj, undefined, 2).replace( @@ -96,7 +98,12 @@ const driver = { res += `db.${update.pureName}.updateOne(${jsonStringifyWithObjectId( update.condition )}, ${jsonStringifyWithObjectId({ - $set: update.fields, + $set: _pickBy(update.fields, (v, k) => v !== undefined), + $unset: _fromPairs( + Object.keys(update.fields) + .filter((k) => update.fields[k] === undefined) + .map((k) => [k, '']) + ), })});\n`; } } @@ -140,6 +147,8 @@ const driver = { supportJsonType: true, supportObjectIdType: true, supportNullType: true, + + supportFieldRemoval: true, }, }; From 2232a7bab1d582e255231bc6d011972a87910b1e Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Mon, 26 Aug 2024 14:26:38 +0200 Subject: [PATCH 09/12] JSONL data editor supports data types --- packages/api/src/shell/modifyJsonLinesReader.js | 11 +++++++---- packages/web/src/datagrid/DataGridCore.svelte | 11 +++++++---- packages/web/src/datagrid/DataGridRow.svelte | 11 +++++++---- packages/web/src/datagrid/InplaceEditor.svelte | 3 +++ packages/web/src/datagrid/InplaceInput.svelte | 4 +++- packages/web/src/datagrid/InplaceSelect.svelte | 9 ++++++++- packages/web/src/datagrid/JslDataGrid.svelte | 17 +++++++++++++++++ .../web/src/datagrid/JslDataGridCore.svelte | 4 ++-- packages/web/src/formview/FormView.svelte | 2 ++ packages/web/src/tabs/ArchiveFileTab.svelte | 9 ++++++++- packages/web/src/tabs/CollectionDataTab.svelte | 2 +- .../dbgate-plugin-mongo/src/backend/driver.js | 4 ++-- 12 files changed, 67 insertions(+), 20 deletions(-) diff --git a/packages/api/src/shell/modifyJsonLinesReader.js b/packages/api/src/shell/modifyJsonLinesReader.js index acffb32f9..7aa007369 100644 --- a/packages/api/src/shell/modifyJsonLinesReader.js +++ b/packages/api/src/shell/modifyJsonLinesReader.js @@ -61,10 +61,13 @@ class ParseStream extends stream.Transform { if (update.document) { obj = update.document; } else { - obj = { - ...obj, - ...update.fields, - }; + obj = _.omitBy( + { + ...obj, + ...update.fields, + }, + (v, k) => v.$$undefined$$ + ); } } diff --git a/packages/web/src/datagrid/DataGridCore.svelte b/packages/web/src/datagrid/DataGridCore.svelte index a608cff9f..5a96ad8fe 100644 --- a/packages/web/src/datagrid/DataGridCore.svelte +++ b/packages/web/src/datagrid/DataGridCore.svelte @@ -457,6 +457,8 @@ export const activator = createActivator('DataGridCore', false); + export let dataEditorTypesBehaviourOverride = null; + const wheelRowCount = 5; const tabVisible: any = getContext('tabVisible'); @@ -829,13 +831,13 @@ showModal(EditCellDataModal, { value: cellData, - dataEditorTypesBehaviour: display?.driver?.dataEditorTypesBehaviour, + dataEditorTypesBehaviour: getEditorTypes(), onSave: value => grider.setCellValue(currentCell[0], realColumnUniqueNames[currentCell[1]], value), }); } export function getEditorTypes() { - return display?.driver?.dataEditorTypesBehaviour; + return dataEditorTypesBehaviourOverride ?? display?.driver?.dataEditorTypesBehaviour; } export function addJsonDocumentEnabled() { @@ -1268,7 +1270,7 @@ const cellData = rowData[realColumnUniqueNames[cell[1]]]; if (shouldOpenMultilineDialog(cellData)) { showModal(EditCellDataModal, { - dataEditorTypesBehaviour: display?.driver?.dataEditorTypesBehaviour, + dataEditorTypesBehaviour: getEditorTypes(), value: cellData, onSave: value => grider.setCellValue(cell[0], realColumnUniqueNames[cell[1]], value), }); @@ -1580,7 +1582,7 @@ } let colIndex = startCol; for (const cell of rowData) { - setCellValue([rowIndex, colIndex], parseCellValue(cell, display?.driver?.dataEditorTypesBehaviour)); + setCellValue([rowIndex, colIndex], parseCellValue(cell, getEditorTypes())); colIndex += 1; } rowIndex += 1; @@ -1965,6 +1967,7 @@ {dispatchInsplaceEditor} {frameSelection} onSetFormView={formViewAvailable && display?.baseTable?.primaryKey ? handleSetFormView : null} + {dataEditorTypesBehaviourOverride} /> {/each} diff --git a/packages/web/src/datagrid/DataGridRow.svelte b/packages/web/src/datagrid/DataGridRow.svelte index f41f86a92..d7acaaf1e 100644 --- a/packages/web/src/datagrid/DataGridRow.svelte +++ b/packages/web/src/datagrid/DataGridRow.svelte @@ -27,6 +27,8 @@ export let database; export let driver; + export let dataEditorTypesBehaviourOverride = null; + $: rowData = grider.getRowData(rowIndex); $: rowStatus = grider.getRowStatus(rowIndex); @@ -59,11 +61,12 @@ {inplaceEditorState} {dispatchInsplaceEditor} cellValue={rowData[col.uniqueName]} - options="{col.options}" - canSelectMultipleOptions="{col.canSelectMultipleOptions}" + options={col.options} + canSelectMultipleOptions={col.canSelectMultipleOptions} onSetValue={value => grider.setCellValue(rowIndex, col.uniqueName, value)} {driver} - /> + {dataEditorTypesBehaviourOverride} + /> {:else} @@ -23,6 +24,7 @@ {options} {canSelectMultipleOptions} {driver} + {dataEditorTypesBehaviourOverride} /> {:else} {/if}
diff --git a/packages/web/src/datagrid/InplaceInput.svelte b/packages/web/src/datagrid/InplaceInput.svelte index 4160816d1..de2ab375d 100644 --- a/packages/web/src/datagrid/InplaceInput.svelte +++ b/packages/web/src/datagrid/InplaceInput.svelte @@ -16,6 +16,8 @@ export let cellValue; export let driver; + export let dataEditorTypesBehaviourOverride = null; + let domEditor; let showEditorButton = true; @@ -23,7 +25,7 @@ const isChangedRef = createRef(!!inplaceEditorState.text); - $: editorTypes = driver?.dataEditorTypesBehaviour; + $: editorTypes = dataEditorTypesBehaviourOverride ?? driver?.dataEditorTypesBehaviour; function handleKeyDown(event) { showEditorButton = false; diff --git a/packages/web/src/datagrid/InplaceSelect.svelte b/packages/web/src/datagrid/InplaceSelect.svelte index b95254fc9..dbc3a0241 100644 --- a/packages/web/src/datagrid/InplaceSelect.svelte +++ b/packages/web/src/datagrid/InplaceSelect.svelte @@ -13,6 +13,8 @@ export let canSelectMultipleOptions; export let driver; + export let dataEditorTypesBehaviourOverride = null; + let value; let valueInit; let optionsData; @@ -20,7 +22,12 @@ onMount(() => { value = - inplaceEditorState.text || stringifyCellValue(cellValue, 'inlineEditorIntent', driver?.dataEditorTypesBehaviour).value; + inplaceEditorState.text || + stringifyCellValue( + cellValue, + 'inlineEditorIntent', + dataEditorTypesBehaviourOverride ?? driver?.dataEditorTypesBehaviour + ).value; valueInit = value; const optionsSelected = value.split(','); diff --git a/packages/web/src/datagrid/JslDataGrid.svelte b/packages/web/src/datagrid/JslDataGrid.svelte index 72e62a8d0..da9acad6a 100644 --- a/packages/web/src/datagrid/JslDataGrid.svelte +++ b/packages/web/src/datagrid/JslDataGrid.svelte @@ -99,5 +99,22 @@ preprocessLoadedRow={changeSetState?.value?.dataUpdateCommands ? row => processJsonDataUpdateCommands(row, changeSetState?.value?.dataUpdateCommands) : null} + dataEditorTypesBehaviourOverride={{ + parseJsonNull: true, + parseJsonBoolean: true, + parseNumber: true, + parseJsonArray: true, + parseJsonObject: true, + + explicitDataType: true, + + supportNumberType: true, + supportStringType: true, + supportBooleanType: true, + supportNullType: true, + supportJsonType: true, + + supportFieldRemoval: true, + }} /> {/key} diff --git a/packages/web/src/datagrid/JslDataGridCore.svelte b/packages/web/src/datagrid/JslDataGridCore.svelte index f089fe0b8..c25164e6b 100644 --- a/packages/web/src/datagrid/JslDataGridCore.svelte +++ b/packages/web/src/datagrid/JslDataGridCore.svelte @@ -69,7 +69,7 @@ export let macroPreview; export let macroValues; - export let onPublishedCellsChanged + export let onPublishedCellsChanged; export const activator = createActivator('JslDataGridCore', false); export let setLoadedRows; @@ -201,7 +201,7 @@ bind:this={domGrid} {...$$props} setLoadedRows={handleSetLoadedRows} - onPublishedCellsChanged={value => { + onPublishedCellsChanged={value => { publishedCells = value; if (onPublishedCellsChanged) { onPublishedCellsChanged(value); diff --git a/packages/web/src/formview/FormView.svelte b/packages/web/src/formview/FormView.svelte index 7dce34966..2493db23c 100644 --- a/packages/web/src/formview/FormView.svelte +++ b/packages/web/src/formview/FormView.svelte @@ -210,6 +210,7 @@ export let rowCountNotAvailable; // export let formDisplay; export let onNavigate; + export let dataEditorTypesBehaviourOverride = null; let wrapperHeight = 1; let wrapperWidth = 1; @@ -652,6 +653,7 @@ driver={display?.driver} inplaceEditorState={$inplaceEditorState} {dispatchInsplaceEditor} + {dataEditorTypesBehaviourOverride} cellValue={rowData[col.uniqueName]} options={col.options} canSelectMultipleOptions={col.canSelectMultipleOptions} diff --git a/packages/web/src/tabs/ArchiveFileTab.svelte b/packages/web/src/tabs/ArchiveFileTab.svelte index 1756d7484..f7391a580 100644 --- a/packages/web/src/tabs/ArchiveFileTab.svelte +++ b/packages/web/src/tabs/ArchiveFileTab.svelte @@ -30,6 +30,7 @@ import { changeSetContainsChanges, createChangeSet } from 'dbgate-datalib'; import localforage from 'localforage'; import { onMount, tick } from 'svelte'; + import _ from 'lodash'; import ToolStripCommandButton from '../buttons/ToolStripCommandButton.svelte'; import ToolStripCommandSplitButton from '../buttons/ToolStripCommandSplitButton.svelte'; @@ -129,7 +130,13 @@ await apiCall('archive/modify-file', { folder: archiveFolder, file: archiveFile, - changeSet: $changeSetStore.value, + changeSet: { + ...$changeSetStore.value, + updates: $changeSetStore.value.updates.map(update => ({ + ...update, + fields: _.mapValues(update.fields, (v, k) => (v === undefined ? { $$undefined$$: true } : v)), + })), + }, }); await afterSaveChangeSet(); } diff --git a/packages/web/src/tabs/CollectionDataTab.svelte b/packages/web/src/tabs/CollectionDataTab.svelte index 4417742d0..e684aad29 100644 --- a/packages/web/src/tabs/CollectionDataTab.svelte +++ b/packages/web/src/tabs/CollectionDataTab.svelte @@ -125,7 +125,7 @@ ...changeSet, updates: changeSet.updates.map(update => ({ ...update, - fields: _.mapValues(update.fields, (v, k) => (v === undefined ? { $undefined: true } : v)), + fields: _.mapValues(update.fields, (v, k) => (v === undefined ? { $$undefined$$: true } : v)), })), }, }); diff --git a/plugins/dbgate-plugin-mongo/src/backend/driver.js b/plugins/dbgate-plugin-mongo/src/backend/driver.js index 58f0a5b6d..d9f4c5c4f 100644 --- a/plugins/dbgate-plugin-mongo/src/backend/driver.js +++ b/plugins/dbgate-plugin-mongo/src/backend/driver.js @@ -330,10 +330,10 @@ const driver = { } } else { const resdoc = await collection.updateOne(convertObjectId(update.condition), { - $set: convertObjectId(_.pickBy(update.fields, (v, k) => !v?.$undefined)), + $set: convertObjectId(_.pickBy(update.fields, (v, k) => !v?.$$undefined$$)), $unset: _.fromPairs( Object.keys(update.fields) - .filter((k) => update.fields[k]?.$undefined) + .filter((k) => update.fields[k]?.$$undefined$$) .map((k) => [k, '']) ), From 62de736bce59fa88d93a23b173cb1ffca07da6b3 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Mon, 26 Aug 2024 14:44:59 +0200 Subject: [PATCH 10/12] refactor --- packages/datalib/src/GridDisplay.ts | 53 ++++++++++++++++++++--------- 1 file changed, 36 insertions(+), 17 deletions(-) diff --git a/packages/datalib/src/GridDisplay.ts b/packages/datalib/src/GridDisplay.ts index 3ea5c1ffd..3a62d9bf1 100644 --- a/packages/datalib/src/GridDisplay.ts +++ b/packages/datalib/src/GridDisplay.ts @@ -104,15 +104,21 @@ export abstract class GridDisplay { setColumnVisibility(uniquePath: string[], isVisible: boolean) { const uniqueName = uniquePath.join('.'); if (uniquePath.length == 1) { - this.includeInColumnSet('hiddenColumns', uniqueName, !isVisible); + this.includeInColumnSet([ + { field: 'hiddenColumns', uniqueName, isIncluded: !isVisible }, + isVisible == false && this.isDynamicStructure && { field: 'addedColumns', uniqueName, isIncluded: false }, + ]); } else { - this.includeInColumnSet('addedColumns', uniqueName, isVisible); + this.includeInColumnSet([{ field: 'addedColumns', uniqueName, isIncluded: isVisible }]); if (!this.isDynamicStructure) this.reload(); } } addDynamicColumn(name: string) { - this.includeInColumnSet('addedColumns', name, true); + this.includeInColumnSet([ + { field: 'addedColumns', uniqueName: name, isIncluded: true }, + { field: 'hiddenColumns', uniqueName: name, isIncluded: false }, + ]); } focusColumns(uniqueNames: string[]) { @@ -150,19 +156,30 @@ export abstract class GridDisplay { this.setCache(reloadDataCacheFunc); } - includeInColumnSet(field: keyof GridConfigColumns, uniqueName: string, isIncluded: boolean) { - // console.log('includeInColumnSet', field, uniqueName, isIncluded); - if (isIncluded) { - this.setConfig(cfg => ({ - ...cfg, - [field]: [...(cfg[field] || []), uniqueName], - })); - } else { - this.setConfig(cfg => ({ - ...cfg, - [field]: (cfg[field] || []).filter(x => x != uniqueName), - })); - } + includeInColumnSet( + modifications: ({ field: keyof GridConfigColumns; uniqueName: string; isIncluded: boolean } | null)[] + ) { + this.setConfig(cfg => { + let res = cfg; + for (const modification of modifications) { + if (!modification) { + continue; + } + const { field, uniqueName, isIncluded } = modification; + if (isIncluded) { + res = { + ...res, + [field]: [...(cfg[field] || []), uniqueName], + }; + } else { + res = { + ...res, + [field]: (cfg[field] || []).filter(x => x != uniqueName), + }; + } + } + return res; + }); } showAllColumns() { @@ -355,7 +372,9 @@ export abstract class GridDisplay { } toggleExpandedColumn(uniqueName: string, value?: boolean) { - this.includeInColumnSet('expandedColumns', uniqueName, value == null ? !this.isExpandedColumn(uniqueName) : value); + this.includeInColumnSet([ + { field: 'expandedColumns', uniqueName, isIncluded: value == null ? !this.isExpandedColumn(uniqueName) : value }, + ]); } getFilter(uniqueName: string) { From d54b47f7133686be1ea1ab313847756365f1ae50 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Mon, 26 Aug 2024 15:09:44 +0200 Subject: [PATCH 11/12] mogno export+import uses EJSON --- .../src/backend/createBulkInsertStream.js | 3 ++- .../dbgate-plugin-mongo/src/backend/driver.js | 17 ++++++++++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/plugins/dbgate-plugin-mongo/src/backend/createBulkInsertStream.js b/plugins/dbgate-plugin-mongo/src/backend/createBulkInsertStream.js index bfe19cb17..2259873cf 100644 --- a/plugins/dbgate-plugin-mongo/src/backend/createBulkInsertStream.js +++ b/plugins/dbgate-plugin-mongo/src/backend/createBulkInsertStream.js @@ -1,5 +1,6 @@ const ObjectId = require('mongodb').ObjectId; const { getLogger } = global.DBGATE_PACKAGES['dbgate-tools']; +const { EJSON } = require('bson'); const logger = getLogger('mongoBulkInsert'); @@ -26,7 +27,7 @@ function createBulkInsertStream(driver, stream, pool, name, options) { ...row, }; } - writable.buffer.push(row); + writable.buffer.push(EJSON.deserialize(row)); }; writable.checkStructure = async () => { diff --git a/plugins/dbgate-plugin-mongo/src/backend/driver.js b/plugins/dbgate-plugin-mongo/src/backend/driver.js index d9f4c5c4f..e7433cadf 100644 --- a/plugins/dbgate-plugin-mongo/src/backend/driver.js +++ b/plugins/dbgate-plugin-mongo/src/backend/driver.js @@ -245,8 +245,23 @@ const driver = { const db = await getScriptableDb(pool); exprValue = func(db, ObjectId.createFromHexString); + const pass = new stream.PassThrough({ + objectMode: true, + highWaterMark: 100, + }); + + exprValue + .forEach((row) => pass.write(transformMongoData(row))) + .then(() => { + pass.end(); + // pass.end(() => { + // pass.emit('end'); + // }) + }); + + return pass; // return directly stream without header row - return exprValue.stream(); + // return exprValue.stream(); // pass.write(structure || { __isDynamicStructure: true }); // exprValue.on('data', (row) => pass.write(row)); From 4436ff95a8f98b348c5b55fd5dc2dee63d828fb2 Mon Sep 17 00:00:00 2001 From: Jan Prochazka Date: Mon, 26 Aug 2024 15:39:57 +0200 Subject: [PATCH 12/12] nosql add new column GUI improved --- .../web/src/datagrid/ColumnManager.svelte | 7 +++++ packages/web/src/datagrid/DataGridCore.svelte | 30 +++++++++++++++++++ packages/web/src/icons/FontIcon.svelte | 1 + packages/web/src/tabs/ArchiveFileTab.svelte | 4 +++ .../web/src/tabs/CollectionDataTab.svelte | 1 + 5 files changed, 43 insertions(+) diff --git a/packages/web/src/datagrid/ColumnManager.svelte b/packages/web/src/datagrid/ColumnManager.svelte index b404084ce..d6ff99ecd 100644 --- a/packages/web/src/datagrid/ColumnManager.svelte +++ b/packages/web/src/datagrid/ColumnManager.svelte @@ -182,6 +182,13 @@ header: 'Add new column', onConfirm: name => { display.addDynamicColumn(name); + tick().then(() => { + selectedColumns = [name]; + currentColumnUniqueName = name; + if (!isJsonView) { + display.focusColumns(selectedColumns); + } + }); }, }); }}>Add getCurrentDataGrid().insertNewRow(), }); + registerCommand({ + id: 'dataGrid.addNewColumn', + category: 'Data grid', + name: 'Add new column', + toolbarName: 'New column', + icon: 'icon add-column', + testEnabled: () => getCurrentDataGrid()?.addNewColumnEnabled(), + onClick: () => getCurrentDataGrid().addNewColumn(), + }); + registerCommand({ id: 'dataGrid.cloneRows', category: 'Data grid', @@ -419,6 +429,7 @@ import { showSnackbarSuccess } from '../utility/snackbar'; import { openJsonLinesData } from '../utility/openJsonLinesData'; import contextMenuActivator from '../utility/contextMenuActivator'; + import InputTextModal from '../modals/InputTextModal.svelte'; export let onLoadNextData = undefined; export let grider = undefined; @@ -554,6 +565,24 @@ grider.endUpdate(); } + export function addNewColumnEnabled() { + return getGrider()?.editable && isDynamicStructure; + } + + export function addNewColumn() { + showModal(InputTextModal, { + value: '', + label: 'Column name', + header: 'Add new column', + onConfirm: name => { + display.addDynamicColumn(name); + tick().then(() => { + display.focusColumns([name]); + }); + }, + }); + } + export async function insertNewRow() { if (!grider.canInsert) return; const rowIndex = grider.insertRow(); @@ -1728,6 +1757,7 @@ { command: 'dataGrid.hideColumn' }, { command: 'dataGrid.filterSelected' }, { command: 'dataGrid.clearFilter' }, + { command: 'dataGrid.addNewColumn', hideDisabled: true }, { command: 'dataGrid.undo', hideDisabled: true }, { command: 'dataGrid.redo', hideDisabled: true }, { divider: true }, diff --git a/packages/web/src/icons/FontIcon.svelte b/packages/web/src/icons/FontIcon.svelte index 61be8eadd..4296d82f4 100644 --- a/packages/web/src/icons/FontIcon.svelte +++ b/packages/web/src/icons/FontIcon.svelte @@ -61,6 +61,7 @@ 'icon app': 'mdi mdi-layers-triple', 'icon open-in-new': 'mdi mdi-open-in-new', 'icon add-folder': 'mdi mdi-folder-plus-outline', + 'icon add-column': 'mdi mdi-table-column-plus-after', 'icon window-restore': 'mdi mdi-window-restore', 'icon window-maximize': 'mdi mdi-window-maximize', diff --git a/packages/web/src/tabs/ArchiveFileTab.svelte b/packages/web/src/tabs/ArchiveFileTab.svelte index f7391a580..74e6d1579 100644 --- a/packages/web/src/tabs/ArchiveFileTab.svelte +++ b/packages/web/src/tabs/ArchiveFileTab.svelte @@ -179,6 +179,10 @@ + + + + diff --git a/packages/web/src/tabs/CollectionDataTab.svelte b/packages/web/src/tabs/CollectionDataTab.svelte index e684aad29..03f9fa85d 100644 --- a/packages/web/src/tabs/CollectionDataTab.svelte +++ b/packages/web/src/tabs/CollectionDataTab.svelte @@ -212,6 +212,7 @@ +