diff --git a/packages/api/src/shell/modifyJsonLinesReader.js b/packages/api/src/shell/modifyJsonLinesReader.js index acffb32f9..7aa007369 100644 --- a/packages/api/src/shell/modifyJsonLinesReader.js +++ b/packages/api/src/shell/modifyJsonLinesReader.js @@ -61,10 +61,13 @@ class ParseStream extends stream.Transform { if (update.document) { obj = update.document; } else { - obj = { - ...obj, - ...update.fields, - }; + obj = _.omitBy( + { + ...obj, + ...update.fields, + }, + (v, k) => v.$$undefined$$ + ); } } diff --git a/packages/datalib/src/GridDisplay.ts b/packages/datalib/src/GridDisplay.ts index 3ea5c1ffd..3a62d9bf1 100644 --- a/packages/datalib/src/GridDisplay.ts +++ b/packages/datalib/src/GridDisplay.ts @@ -104,15 +104,21 @@ export abstract class GridDisplay { setColumnVisibility(uniquePath: string[], isVisible: boolean) { const uniqueName = uniquePath.join('.'); if (uniquePath.length == 1) { - this.includeInColumnSet('hiddenColumns', uniqueName, !isVisible); + this.includeInColumnSet([ + { field: 'hiddenColumns', uniqueName, isIncluded: !isVisible }, + isVisible == false && this.isDynamicStructure && { field: 'addedColumns', uniqueName, isIncluded: false }, + ]); } else { - this.includeInColumnSet('addedColumns', uniqueName, isVisible); + this.includeInColumnSet([{ field: 'addedColumns', uniqueName, isIncluded: isVisible }]); if (!this.isDynamicStructure) this.reload(); } } addDynamicColumn(name: string) { - this.includeInColumnSet('addedColumns', name, true); + this.includeInColumnSet([ + { field: 'addedColumns', uniqueName: name, isIncluded: true }, + { field: 'hiddenColumns', uniqueName: name, isIncluded: false }, + ]); } focusColumns(uniqueNames: string[]) { @@ -150,19 +156,30 @@ export abstract class GridDisplay { this.setCache(reloadDataCacheFunc); } - includeInColumnSet(field: keyof GridConfigColumns, uniqueName: string, isIncluded: boolean) { - // console.log('includeInColumnSet', field, uniqueName, isIncluded); - if (isIncluded) { - this.setConfig(cfg => ({ - ...cfg, - [field]: [...(cfg[field] || []), uniqueName], - })); - } else { - this.setConfig(cfg => ({ - ...cfg, - [field]: (cfg[field] || []).filter(x => x != uniqueName), - })); - } + includeInColumnSet( + modifications: ({ field: keyof GridConfigColumns; uniqueName: string; isIncluded: boolean } | null)[] + ) { + this.setConfig(cfg => { + let res = cfg; + for (const modification of modifications) { + if (!modification) { + continue; + } + const { field, uniqueName, isIncluded } = modification; + if (isIncluded) { + res = { + ...res, + [field]: [...(cfg[field] || []), uniqueName], + }; + } else { + res = { + ...res, + [field]: (cfg[field] || []).filter(x => x != uniqueName), + }; + } + } + return res; + }); } showAllColumns() { @@ -355,7 +372,9 @@ export abstract class GridDisplay { } toggleExpandedColumn(uniqueName: string, value?: boolean) { - this.includeInColumnSet('expandedColumns', uniqueName, value == null ? !this.isExpandedColumn(uniqueName) : value); + this.includeInColumnSet([ + { field: 'expandedColumns', uniqueName, isIncluded: value == null ? !this.isExpandedColumn(uniqueName) : value }, + ]); } getFilter(uniqueName: string) { diff --git a/packages/tools/src/driverBase.ts b/packages/tools/src/driverBase.ts index d76318b9d..3a304525e 100644 --- a/packages/tools/src/driverBase.ts +++ b/packages/tools/src/driverBase.ts @@ -161,4 +161,9 @@ export const driverBase = { getCollectionExportQueryJson(collection: string, condition: any, sort: any) { return null; }, + + dataEditorTypesBehaviour: { + parseSqlNull: true, + parseHexAsBuffer: true, + }, }; diff --git a/packages/tools/src/stringTools.ts b/packages/tools/src/stringTools.ts index 21be36385..05e4fa423 100644 --- a/packages/tools/src/stringTools.ts +++ b/packages/tools/src/stringTools.ts @@ -1,6 +1,25 @@ import _isString from 'lodash/isString'; import _isArray from 'lodash/isArray'; +import _isNumber from 'lodash/isNumber'; import _isPlainObject from 'lodash/isPlainObject'; +import _pad from 'lodash/pad'; +import { DataEditorTypesBehaviour } from 'dbgate-types'; + +export type EditorDataType = + | 'null' + | 'objectid' + | 'string' + | 'number' + | 'object' + | 'date' + | 'array' + | 'boolean' + | 'unknown'; + +const dateTimeStorageRegex = + /^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?(([Zz])|()|([\+|\-]([01][0-9]|2[0-3]):[0-5][0-9]))$/; + +const dateTimeParseRegex = /^(\d{4})-(\d{2})-(\d{2})[Tt ](\d{2}):(\d{2}):(\d{2})(\.[0-9]+)?(([Zz])|()|([\+|\-]([01][0-9]|2[0-3]):[0-5][0-9]))$/; export function arrayToHexString(byteArray) { return byteArray.reduce((output, elem) => output + ('0' + elem.toString(16)).slice(-2), '').toUpperCase(); @@ -15,36 +34,267 @@ export function hexStringToArray(inputString) { return res; } -export function parseCellValue(value) { +export function parseCellValue(value, editorTypes?: DataEditorTypesBehaviour) { if (!_isString(value)) return value; - if (value == '(NULL)') return null; - - const mHex = value.match(/^0x([0-9a-fA-F][0-9a-fA-F])+$/); - if (mHex) { - return { - type: 'Buffer', - data: hexStringToArray(value.substring(2)), - }; + if (editorTypes?.parseSqlNull) { + if (value == '(NULL)') return null; } - const mOid = value.match(/^ObjectId\("([0-9a-f]{24})"\)$/); - if (mOid) { - return { $oid: mOid[1] }; + if (editorTypes?.parseHexAsBuffer) { + const mHex = value.match(/^0x([0-9a-fA-F][0-9a-fA-F])+$/); + if (mHex) { + return { + type: 'Buffer', + data: hexStringToArray(value.substring(2)), + }; + } + } + + if (editorTypes?.parseObjectIdAsDollar) { + const mOid = value.match(/^ObjectId\("([0-9a-f]{24})"\)$/); + if (mOid) { + return { $oid: mOid[1] }; + } + } + + if (editorTypes?.parseDateAsDollar) { + const m = value.match(dateTimeParseRegex); + if (m) { + return { + $date: `${m[1]}-${m[2]}-${m[3]}T${m[4]}:${m[5]}:${m[6]}Z`, + }; + } + } + + if (editorTypes?.parseJsonNull) { + if (value == 'null') return null; + } + + if (editorTypes?.parseJsonBoolean) { + if (value == 'true') return true; + if (value == 'false') return false; + } + + if (editorTypes?.parseNumber) { + if (/^-?[0-9]+(?:\.[0-9]+)?$/.test(value)) { + return parseFloat(value); + } + } + + if (editorTypes?.parseJsonArray || editorTypes?.parseJsonObject) { + const jsonValue = safeJsonParse(value); + if (_isPlainObject(jsonValue) && editorTypes?.parseJsonObject) return jsonValue; + if (_isArray(jsonValue) && editorTypes?.parseJsonArray) return jsonValue; } return value; } -export function stringifyCellValue(value) { - if (value === null) return '(NULL)'; - if (value === undefined) return '(NoField)'; - if (value?.type == 'Buffer' && _isArray(value.data)) return '0x' + arrayToHexString(value.data); - if (value?.$oid) return `ObjectId("${value?.$oid}")`; - if (_isPlainObject(value) || _isArray(value)) return JSON.stringify(value); +function parseFunc_ObjectIdAsDollar(value) { + if (value?.$oid) return value; + if (_isString(value)) { + if (value.match(/^[0-9a-f]{24}$/)) return { $oid: value }; + const mOid = value.match(/^ObjectId\("([0-9a-f]{24})"\)$/); + if (mOid) { + return { $oid: mOid[1] }; + } + } return value; } +function parseFunc_DateAsDollar(value) { + if (value?.$date) return value; + if (_isString(value)) { + const m = value.match(dateTimeParseRegex); + if (m) { + return { $date: `${m[1]}-${m[2]}-${m[3]}T${m[4]}:${m[5]}:${m[6]}Z` }; + } + } + return value; +} + +function makeBulletString(value) { + return _pad('', value.length, '•'); +} + +function highlightSpecialCharacters(value) { + value = value.replace(/\n/g, '↲'); + value = value.replace(/\r/g, ''); + value = value.replace(/^(\s+)/, makeBulletString); + value = value.replace(/(\s+)$/, makeBulletString); + value = value.replace(/(\s\s+)/g, makeBulletString); + return value; +} + +function stringifyJsonToGrid(value): ReturnType { + if (_isPlainObject(value)) { + const svalue = JSON.stringify(value, undefined, 2); + if (svalue.length < 100) { + return { value: svalue, gridStyle: 'nullCellStyle' }; + } else { + return { value: '(JSON)', gridStyle: 'nullCellStyle', gridTitle: svalue }; + } + } + if (_isArray(value)) { + return { + value: `[${value.length} items]`, + gridStyle: 'nullCellStyle', + gridTitle: value.map(x => JSON.stringify(x)).join('\n'), + }; + } + return { value: '(JSON)', gridStyle: 'nullCellStyle' }; +} + +export function stringifyCellValue( + value, + intent: 'gridCellIntent' | 'inlineEditorIntent' | 'multilineEditorIntent' | 'stringConversionIntent' | 'exportIntent', + editorTypes?: DataEditorTypesBehaviour, + gridFormattingOptions?: { useThousandsSeparator?: boolean }, + jsonParsedValue?: any +): { + value: string; + gridStyle?: 'textCellStyle' | 'valueCellStyle' | 'nullCellStyle'; // only for gridCellIntent + gridTitle?: string; // only for gridCellIntent +} { + if (editorTypes?.parseSqlNull) { + if (value === null) { + switch (intent) { + case 'exportIntent': + return { value: '' }; + default: + return { value: '(NULL)', gridStyle: 'nullCellStyle' }; + } + } + } + if (value === undefined) { + switch (intent) { + case 'gridCellIntent': + return { value: '(No Field)', gridStyle: 'nullCellStyle' }; + default: + return { value: '' }; + } + } + if (editorTypes?.parseJsonNull) { + if (value === null) { + return { value: 'null', gridStyle: 'valueCellStyle' }; + } + } + + if (value === true) return { value: 'true', gridStyle: 'valueCellStyle' }; + if (value === false) return { value: 'false', gridStyle: 'valueCellStyle' }; + + if (editorTypes?.parseHexAsBuffer) { + if (value?.type == 'Buffer' && _isArray(value.data)) { + return { value: '0x' + arrayToHexString(value.data), gridStyle: 'valueCellStyle' }; + } + } + if (editorTypes?.parseObjectIdAsDollar) { + if (value?.$oid) { + switch (intent) { + case 'exportIntent': + case 'stringConversionIntent': + return { value: value.$oid }; + default: + return { value: `ObjectId("${value.$oid}")`, gridStyle: 'valueCellStyle' }; + } + } + } + + if (editorTypes?.parseDateAsDollar) { + if (value?.$date) { + switch (intent) { + case 'exportIntent': + case 'stringConversionIntent': + return { value: value.$date }; + default: + const m = value.$date.match(dateTimeStorageRegex); + if (m) { + return { value: `${m[1]}-${m[2]}-${m[3]} ${m[4]}:${m[5]}:${m[6]}`, gridStyle: 'valueCellStyle' }; + } else { + return { value: value.$date.replaCE('T', ' '), gridStyle: 'valueCellStyle' }; + } + } + } + } + + if (_isArray(value)) { + switch (intent) { + case 'gridCellIntent': + return stringifyJsonToGrid(value); + case 'multilineEditorIntent': + return { value: JSON.stringify(value, null, 2) }; + default: + return { value: JSON.stringify(value), gridStyle: 'valueCellStyle' }; + } + } + + if (_isPlainObject(value)) { + switch (intent) { + case 'gridCellIntent': + return stringifyJsonToGrid(value); + case 'multilineEditorIntent': + return { value: JSON.stringify(value, null, 2) }; + default: + return { value: JSON.stringify(value), gridStyle: 'valueCellStyle' }; + } + } + + if (_isNumber(value)) { + switch (intent) { + case 'gridCellIntent': + return { + value: + gridFormattingOptions?.useThousandsSeparator && (value >= 10000 || value <= -10000) + ? value.toLocaleString() + : value.toString(), + gridStyle: 'valueCellStyle', + }; + default: + return { value: value.toString() }; + } + } + + if (_isString(value)) { + switch (intent) { + case 'gridCellIntent': + if (jsonParsedValue && !editorTypes?.explicitDataType) { + return stringifyJsonToGrid(jsonParsedValue); + } else { + if (!editorTypes?.explicitDataType) { + // reformat datetime for implicit date types + const m = value.match(dateTimeStorageRegex); + if (m) { + return { + value: `${m[1]}-${m[2]}-${m[3]} ${m[4]}:${m[5]}:${m[6]}`, + gridStyle: 'valueCellStyle', + }; + } + } + return { value: highlightSpecialCharacters(value), gridStyle: 'textCellStyle' }; + } + default: + return { value: value }; + } + } + + if (value === null || value === undefined) { + switch (intent) { + case 'gridCellIntent': + return { value: '(n/a)', gridStyle: 'nullCellStyle' }; + default: + return { value: '' }; + } + } + + switch (intent) { + case 'gridCellIntent': + return { value: '(Unknown)', gridStyle: 'nullCellStyle' }; + default: + return { value: '' }; + } +} + export function safeJsonParse(json, defaultValue?, logError = false) { if (_isArray(json) || _isPlainObject(json)) { return json; @@ -59,6 +309,28 @@ export function safeJsonParse(json, defaultValue?, logError = false) { } } +export function shouldOpenMultilineDialog(value) { + if (_isString(value)) { + if (value.includes('\n')) { + return true; + } + const parsed = safeJsonParse(value); + if (parsed && (_isPlainObject(parsed) || _isArray(parsed))) { + return true; + } + } + if (value?.$oid) { + return false; + } + if (value?.$date) { + return false; + } + if (_isPlainObject(value) || _isArray(value)) { + return true; + } + return false; +} + export function isJsonLikeLongString(value) { return _isString(value) && value.length > 100 && value.match(/^\s*\{.*\}\s*$|^\s*\[.*\]\s*$/); } @@ -127,3 +399,67 @@ export function parseSqlDefaultValue(value: string) { } return undefined; } + +export function detectCellDataType(value): EditorDataType { + if (value === null) return 'null'; + if (value?.$oid) return 'objectid'; + if (value?.$date) return 'date'; + if (_isString(value)) return 'string'; + if (_isNumber(value)) return 'number'; + if (_isPlainObject(value)) return 'object'; + if (_isArray(value)) return 'array'; + if (value === true || value === false) return 'boolean'; + return 'unknown'; +} + +export function detectTypeIcon(value) { + switch (detectCellDataType(value)) { + case 'null': + return 'icon type-null'; + case 'objectid': + return 'icon type-objectid'; + case 'date': + return 'icon type-date'; + case 'string': + return 'icon type-string'; + case 'number': + return 'icon type-number'; + case 'object': + return 'icon type-object'; + case 'array': + return 'icon type-array'; + case 'boolean': + return 'icon type-boolean'; + default: + return 'icon type-unknown'; + } +} + +export function getConvertValueMenu(value, onSetValue, editorTypes?: DataEditorTypesBehaviour) { + return [ + editorTypes?.supportStringType && { + text: 'String', + onClick: () => onSetValue(stringifyCellValue(value, 'stringConversionIntent', editorTypes).value), + }, + editorTypes?.supportNumberType && { text: 'Number', onClick: () => onSetValue(parseFloat(value)) }, + editorTypes?.supportNullType && { text: 'Null', onClick: () => onSetValue(null) }, + editorTypes?.supportBooleanType && { + text: 'Boolean', + onClick: () => onSetValue(value?.toString()?.toLowerCase() == 'true' || value == '1'), + }, + editorTypes?.supportObjectIdType && { + text: 'ObjectId', + onClick: () => onSetValue(parseFunc_ObjectIdAsDollar(value)), + }, + editorTypes?.supportDateType && { text: 'Date', onClick: () => onSetValue(parseFunc_DateAsDollar(value)) }, + editorTypes?.supportJsonType && { + text: 'JSON', + onClick: () => { + const jsonValue = safeJsonParse(value); + if (jsonValue != null) { + onSetValue(jsonValue); + } + }, + }, + ]; +} diff --git a/packages/types/engines.d.ts b/packages/types/engines.d.ts index c828a4b9e..dea2c0409 100644 --- a/packages/types/engines.d.ts +++ b/packages/types/engines.d.ts @@ -93,6 +93,29 @@ export interface CollectionSortDefinitionItem { export type CollectionSortDefinition = CollectionSortDefinitionItem[]; +export interface DataEditorTypesBehaviour { + parseSqlNull?: boolean; + parseJsonNull?: boolean; + parseJsonBoolean?: boolean; + parseNumber?: boolean; + parseJsonArray?: boolean; + parseJsonObject?: boolean; + parseHexAsBuffer?: boolean; + parseObjectIdAsDollar?: boolean; + parseDateAsDollar?: boolean; + + explicitDataType?: boolean; + supportNumberType?: boolean; + supportStringType?: boolean; + supportBooleanType?: boolean; + supportDateType?: boolean; + supportNullType?: boolean; + supportJsonType?: boolean; + supportObjectIdType?: boolean; + + supportFieldRemoval?: boolean; +} + export interface FilterBehaviourProvider { getFilterBehaviour(dataType: string, standardFilterBehaviours: { [id: string]: FilterBehaviour }): FilterBehaviour; } @@ -105,6 +128,7 @@ export interface EngineDriver extends FilterBehaviourProvider { editorMode?: string; readOnlySessions: boolean; supportedKeyTypes: SupportedDbKeyType[]; + dataEditorTypesBehaviour: DataEditorTypesBehaviour; supportsDatabaseUrl?: boolean; supportsDatabaseDump?: boolean; supportsServerSummary?: boolean; diff --git a/packages/web/src/datagrid/CellValue.svelte b/packages/web/src/datagrid/CellValue.svelte index e2e5eb1f2..7b6bcb7a1 100644 --- a/packages/web/src/datagrid/CellValue.svelte +++ b/packages/web/src/datagrid/CellValue.svelte @@ -1,105 +1,34 @@ - - {#if rowData == null} (No row) -{:else if value === null} - (NULL) -{:else if value === undefined} - (No field) -{:else if _.isDate(value)} - {value.toString()} -{:else if value === true} - true -{:else if value === false} - false -{:else if _.isNumber(value)} - {formatNumber(value)} -{:else if _.isString(value) && !jsonParsedValue} - {#if dateTimeRegex.test(value)} - - {formatDateTime(value)} - - {:else} - {highlightSpecialCharacters(value)} - {/if} -{:else if value?.type == 'Buffer' && _.isArray(value.data)} - {#if value.data.length <= 16} - {'0x' + arrayToHexString(value.data)} - {:else} - ({value.data.length} bytes) - {/if} -{:else if value.$oid} - ObjectId("{value.$oid}") -{:else if _.isPlainObject(value)} - {@const svalue = JSON.stringify(value, undefined, 2)} - {#if svalue.length < 100}{JSON.stringify(value)}{:else}(JSON){/if} -{:else if _.isArray(value)} - JSON.stringify(x)).join('\n')}>[{value.length} items] -{:else if _.isPlainObject(jsonParsedValue)} - {@const svalue = JSON.stringify(jsonParsedValue, undefined, 2)} - {#if svalue.length < 100}{JSON.stringify(jsonParsedValue)}{:else}(JSON){/if} -{:else if _.isArray(jsonParsedValue)} - JSON.stringify(x)).join('\n')} - >[{jsonParsedValue.length} items] {:else} - {value.toString()} + {stringified.value} {/if} diff --git a/packages/web/src/datagrid/ColumnManager.svelte b/packages/web/src/datagrid/ColumnManager.svelte index b404084ce..d6ff99ecd 100644 --- a/packages/web/src/datagrid/ColumnManager.svelte +++ b/packages/web/src/datagrid/ColumnManager.svelte @@ -182,6 +182,13 @@ header: 'Add new column', onConfirm: name => { display.addDynamicColumn(name); + tick().then(() => { + selectedColumns = [name]; + currentColumnUniqueName = name; + if (!isJsonView) { + display.focusColumns(selectedColumns); + } + }); }, }); }}>Add - import _ from 'lodash'; + import _, { isPlainObject } from 'lodash'; import ShowFormButton from '../formview/ShowFormButton.svelte'; - import { isJsonLikeLongString, safeJsonParse } from 'dbgate-tools'; + import { detectTypeIcon, getConvertValueMenu, isJsonLikeLongString, safeJsonParse } from 'dbgate-tools'; import { openJsonDocument } from '../tabs/JsonTab.svelte'; - import openNewTab from '../utility/openNewTab'; import CellValue from './CellValue.svelte'; - import { showModal } from '../modals/modalTools'; - import EditCellDataModal from '../modals/EditCellDataModal.svelte'; import { openJsonLinesData } from '../utility/openJsonLinesData'; + import ShowFormDropDownButton from '../formview/ShowFormDropDownButton.svelte'; export let rowIndex; export let col; @@ -33,6 +31,7 @@ export let isCurrentCell = false; export let onDictionaryLookup = null; export let onSetValue; + export let editorTypes = null; $: value = col.isStructured ? _.get(rowData || {}, col.uniquePath) : (rowData || {})[col.uniqueName]; @@ -51,7 +50,9 @@ $: style = computeStyle(maxWidth, col); $: isJson = _.isPlainObject(value) && !(value?.type == 'Buffer' && _.isArray(value.data)) && !value.$oid; - $: jsonParsedValue = isJsonLikeLongString(value) ? safeJsonParse(value) : null; + + // don't parse JSON for explicit data types + $: jsonParsedValue = !editorTypes?.explicitDataType && isJsonLikeLongString(value) ? safeJsonParse(value) : null; - + {#if allowHintField && rowData && _.some(col.hintColumnNames, hintColumnName => rowData[hintColumnName])} {/if} - {#if col.foreignKey && rowData && rowData[col.uniqueName] && !isCurrentCell} + {#if editorTypes?.explicitDataType} + {#if value !== undefined} + getConvertValueMenu(value, onSetValue, editorTypes)} + /> + {/if} + {#if _.isPlainObject(value)} + openJsonDocument(value, undefined, true)} /> + {/if} + {#if _.isArray(value)} + { + if (_.every(value, x => _.isPlainObject(x))) { + openJsonLinesData(value); + } else { + openJsonDocument(value, undefined, true); + } + }} + /> + {/if} + {:else if col.foreignKey && rowData && rowData[col.uniqueName] && !isCurrentCell} onSetFormView(rowData, col)} /> - {/if} - - {#if col.foreignKey && isCurrentCell && onDictionaryLookup} + {:else if col.foreignKey && isCurrentCell && onDictionaryLookup} - {/if} - - {#if isJson} + {:else if isJson} openJsonDocument(value, undefined, true)} /> - {/if} - - {#if jsonParsedValue && _.isPlainObject(jsonParsedValue)} + {:else if jsonParsedValue && _.isPlainObject(jsonParsedValue)} openJsonDocument(jsonParsedValue, undefined, true)} /> - {/if} - - {#if _.isArray(jsonParsedValue || value)} + {:else if _.isArray(jsonParsedValue || value)} { diff --git a/packages/web/src/datagrid/DataGridCore.svelte b/packages/web/src/datagrid/DataGridCore.svelte index ec51a266f..eeb0f8e5c 100644 --- a/packages/web/src/datagrid/DataGridCore.svelte +++ b/packages/web/src/datagrid/DataGridCore.svelte @@ -66,6 +66,16 @@ onClick: () => getCurrentDataGrid().insertNewRow(), }); + registerCommand({ + id: 'dataGrid.addNewColumn', + category: 'Data grid', + name: 'Add new column', + toolbarName: 'New column', + icon: 'icon add-column', + testEnabled: () => getCurrentDataGrid()?.addNewColumnEnabled(), + onClick: () => getCurrentDataGrid().addNewColumn(), + }); + registerCommand({ id: 'dataGrid.cloneRows', category: 'Data grid', @@ -81,10 +91,21 @@ category: 'Data grid', name: 'Set NULL', keyText: 'CtrlOrCommand+0', - testEnabled: () => getCurrentDataGrid()?.getGrider()?.editable, + testEnabled: () => + getCurrentDataGrid()?.getGrider()?.editable && !getCurrentDataGrid()?.getEditorTypes()?.supportFieldRemoval, onClick: () => getCurrentDataGrid().setFixedValue(null), }); + registerCommand({ + id: 'dataGrid.removeField', + category: 'Data grid', + name: 'Remove field', + keyText: 'CtrlOrCommand+0', + testEnabled: () => + getCurrentDataGrid()?.getGrider()?.editable && getCurrentDataGrid()?.getEditorTypes()?.supportFieldRemoval, + onClick: () => getCurrentDataGrid().setFixedValue(undefined), + }); + registerCommand({ id: 'dataGrid.undo', category: 'Data grid', @@ -343,7 +364,13 @@ @@ -21,6 +23,8 @@ {onSetValue} {options} {canSelectMultipleOptions} + {driver} + {dataEditorTypesBehaviourOverride} /> {:else} {/if} diff --git a/packages/web/src/datagrid/InplaceInput.svelte b/packages/web/src/datagrid/InplaceInput.svelte index aec5d7b61..de2ab375d 100644 --- a/packages/web/src/datagrid/InplaceInput.svelte +++ b/packages/web/src/datagrid/InplaceInput.svelte @@ -14,6 +14,9 @@ export let onSetValue; export let width; export let cellValue; + export let driver; + + export let dataEditorTypesBehaviourOverride = null; let domEditor; let showEditorButton = true; @@ -22,6 +25,8 @@ const isChangedRef = createRef(!!inplaceEditorState.text); + $: editorTypes = dataEditorTypesBehaviourOverride ?? driver?.dataEditorTypesBehaviour; + function handleKeyDown(event) { showEditorButton = false; @@ -32,7 +37,7 @@ break; case keycodes.enter: if (isChangedRef.get()) { - onSetValue(parseCellValue(domEditor.value)); + onSetValue(parseCellValue(domEditor.value, editorTypes)); isChangedRef.set(false); } domEditor.blur(); @@ -41,7 +46,7 @@ break; case keycodes.tab: if (isChangedRef.get()) { - onSetValue(parseCellValue(domEditor.value)); + onSetValue(parseCellValue(domEditor.value, editorTypes)); isChangedRef.set(false); } domEditor.blur(); @@ -51,7 +56,7 @@ case keycodes.s: if (isCtrlOrCommandKey(event)) { if (isChangedRef.get()) { - onSetValue(parseCellValue(domEditor.value)); + onSetValue(parseCellValue(domEditor.value, editorTypes)); isChangedRef.set(false); } event.preventDefault(); @@ -63,7 +68,7 @@ function handleBlur() { if (isChangedRef.get()) { - onSetValue(parseCellValue(domEditor.value)); + onSetValue(parseCellValue(domEditor.value, editorTypes)); // grider.setCellValue(rowIndex, uniqueName, editor.value); isChangedRef.set(false); } @@ -71,7 +76,7 @@ } onMount(() => { - domEditor.value = inplaceEditorState.text || stringifyCellValue(cellValue); + domEditor.value = inplaceEditorState.text || stringifyCellValue(cellValue, 'inlineEditorIntent', editorTypes).value; domEditor.focus(); if (inplaceEditorState.selectAll) { domEditor.select(); @@ -102,7 +107,8 @@ dispatchInsplaceEditor({ type: 'close' }); showModal(EditCellDataModal, { - value: stringifyCellValue(cellValue), + value: cellValue, + dataEditorTypesBehaviour: editorTypes, onSave: onSetValue, }); }} diff --git a/packages/web/src/datagrid/InplaceSelect.svelte b/packages/web/src/datagrid/InplaceSelect.svelte index 7148c2837..dbc3a0241 100644 --- a/packages/web/src/datagrid/InplaceSelect.svelte +++ b/packages/web/src/datagrid/InplaceSelect.svelte @@ -11,6 +11,9 @@ export let cellValue; export let options; export let canSelectMultipleOptions; + export let driver; + + export let dataEditorTypesBehaviourOverride = null; let value; let valueInit; @@ -18,22 +21,27 @@ let isOptionsHidden = false; onMount(() => { - value = inplaceEditorState.text || stringifyCellValue(cellValue); + value = + inplaceEditorState.text || + stringifyCellValue( + cellValue, + 'inlineEditorIntent', + dataEditorTypesBehaviourOverride ?? driver?.dataEditorTypesBehaviour + ).value; valueInit = value; const optionsSelected = value.split(','); - optionsData = options - .map(function(option) { - return { - value: option, - isSelected: optionsSelected.includes(option) - }; - }); + optionsData = options.map(function (option) { + return { + value: option, + isSelected: optionsSelected.includes(option), + }; + }); }); function handleCheckboxChanged(e, option) { if (!canSelectMultipleOptions) { - optionsData.forEach(option => option.isSelected = false); + optionsData.forEach(option => (option.isSelected = false)); option.isSelected = true; } else { option.isSelected = e.target.checked; @@ -44,8 +52,7 @@ .map(option => option.value) .join(','); - if(!canSelectMultipleOptions) - handleConfirm(); + if (!canSelectMultipleOptions) handleConfirm(); } function handleConfirm() { @@ -69,13 +76,8 @@ } -
-
isOptionsHidden = !isOptionsHidden} class="value"> +
+
(isOptionsHidden = !isOptionsHidden)} class="value"> {value}
@@ -88,11 +90,12 @@
{#each optionsData ?? [] as option} {/each} @@ -112,7 +115,7 @@ background-color: var(--theme-bg-alt); max-height: 150px; overflow: auto; - box-shadow: 0 1px 10px 1px var(--theme-bg-inv-3);; + box-shadow: 0 1px 10px 1px var(--theme-bg-inv-3); } .value { diff --git a/packages/web/src/datagrid/JslDataGrid.svelte b/packages/web/src/datagrid/JslDataGrid.svelte index 72e62a8d0..da9acad6a 100644 --- a/packages/web/src/datagrid/JslDataGrid.svelte +++ b/packages/web/src/datagrid/JslDataGrid.svelte @@ -99,5 +99,22 @@ preprocessLoadedRow={changeSetState?.value?.dataUpdateCommands ? row => processJsonDataUpdateCommands(row, changeSetState?.value?.dataUpdateCommands) : null} + dataEditorTypesBehaviourOverride={{ + parseJsonNull: true, + parseJsonBoolean: true, + parseNumber: true, + parseJsonArray: true, + parseJsonObject: true, + + explicitDataType: true, + + supportNumberType: true, + supportStringType: true, + supportBooleanType: true, + supportNullType: true, + supportJsonType: true, + + supportFieldRemoval: true, + }} /> {/key} diff --git a/packages/web/src/datagrid/JslDataGridCore.svelte b/packages/web/src/datagrid/JslDataGridCore.svelte index f089fe0b8..c25164e6b 100644 --- a/packages/web/src/datagrid/JslDataGridCore.svelte +++ b/packages/web/src/datagrid/JslDataGridCore.svelte @@ -69,7 +69,7 @@ export let macroPreview; export let macroValues; - export let onPublishedCellsChanged + export let onPublishedCellsChanged; export const activator = createActivator('JslDataGridCore', false); export let setLoadedRows; @@ -201,7 +201,7 @@ bind:this={domGrid} {...$$props} setLoadedRows={handleSetLoadedRows} - onPublishedCellsChanged={value => { + onPublishedCellsChanged={value => { publishedCells = value; if (onPublishedCellsChanged) { onPublishedCellsChanged(value); diff --git a/packages/web/src/elements/SelectionMapView.svelte b/packages/web/src/elements/SelectionMapView.svelte index ec64fde5c..60ca253b0 100644 --- a/packages/web/src/elements/SelectionMapView.svelte +++ b/packages/web/src/elements/SelectionMapView.svelte @@ -37,7 +37,7 @@ function createColumnsTable(cells) { if (cells.length == 0) return ''; return `${cells - .map(cell => ``) + .map(cell => ``) .join('\n')}
${cell.column}${stringifyCellValue(cell.value)}
${cell.column}${stringifyCellValue(cell.value, 'exportIntent').value}
`; } diff --git a/packages/web/src/formview/FormView.svelte b/packages/web/src/formview/FormView.svelte index e24d90216..2493db23c 100644 --- a/packages/web/src/formview/FormView.svelte +++ b/packages/web/src/formview/FormView.svelte @@ -48,10 +48,19 @@ category: 'Data form', name: 'Set NULL', keyText: 'CtrlOrCommand+0', - testEnabled: () => getCurrentDataForm() != null, + testEnabled: () => getCurrentDataForm() != null && !getCurrentDataForm()?.getEditorTypes()?.supportFieldRemoval, onClick: () => getCurrentDataForm().setFixedValue(null), }); + registerCommand({ + id: 'dataForm.removeField', + category: 'Data form', + name: 'Remove field', + keyText: 'CtrlOrCommand+0', + testEnabled: () => getCurrentDataForm() != null && getCurrentDataForm()?.getEditorTypes()?.supportFieldRemoval, + onClick: () => getCurrentDataForm().setFixedValue(undefined), + }); + registerCommand({ id: 'dataForm.undo', category: 'Data form', @@ -157,7 +166,7 @@
@@ -23,6 +25,10 @@ border: 1px solid var(--theme-bg-1); } + .secondary { + margin-right: 20px; + } + div:hover { color: var(--theme-font-hover); border: var(--theme-border); diff --git a/packages/web/src/formview/ShowFormDropDownButton.svelte b/packages/web/src/formview/ShowFormDropDownButton.svelte new file mode 100644 index 000000000..d0977aaf3 --- /dev/null +++ b/packages/web/src/formview/ShowFormDropDownButton.svelte @@ -0,0 +1,44 @@ + + +
+ +
+ + diff --git a/packages/web/src/icons/FontIcon.svelte b/packages/web/src/icons/FontIcon.svelte index c64332de1..4296d82f4 100644 --- a/packages/web/src/icons/FontIcon.svelte +++ b/packages/web/src/icons/FontIcon.svelte @@ -61,6 +61,7 @@ 'icon app': 'mdi mdi-layers-triple', 'icon open-in-new': 'mdi mdi-open-in-new', 'icon add-folder': 'mdi mdi-folder-plus-outline', + 'icon add-column': 'mdi mdi-table-column-plus-after', 'icon window-restore': 'mdi mdi-window-restore', 'icon window-maximize': 'mdi mdi-window-maximize', @@ -186,6 +187,16 @@ 'icon num-9-outline': 'mdi mdi-numeric-9-circle-outline', 'icon num-9-plus-outline': 'mdi mdi-numeric-9-plus-circle-outline', + 'icon type-string': 'mdi mdi-code-string', + 'icon type-object': 'mdi mdi-code-braces-box', + 'icon type-array': 'mdi mdi-code-array', + 'icon type-number': 'mdi mdi-pound-box', + 'icon type-boolean': 'mdi mdi-code-equal', + 'icon type-date': 'mdi mdi-alpha-d-box', + 'icon type-objectid': 'mdi mdi-alpha-i-box', + 'icon type-null': 'mdi mdi-code-equal', + 'icon type-unknown': 'mdi mdi-help-box', + 'img ok': 'mdi mdi-check-circle color-icon-green', 'img ok-inv': 'mdi mdi-check-circle color-icon-inv-green', 'img alert': 'mdi mdi-alert-circle color-icon-blue', diff --git a/packages/web/src/modals/EditCellDataModal.svelte b/packages/web/src/modals/EditCellDataModal.svelte index 9b6918ce3..39fa738c9 100644 --- a/packages/web/src/modals/EditCellDataModal.svelte +++ b/packages/web/src/modals/EditCellDataModal.svelte @@ -1,18 +1,3 @@ - - diff --git a/packages/web/src/tabs/ArchiveFileTab.svelte b/packages/web/src/tabs/ArchiveFileTab.svelte index 1756d7484..74e6d1579 100644 --- a/packages/web/src/tabs/ArchiveFileTab.svelte +++ b/packages/web/src/tabs/ArchiveFileTab.svelte @@ -30,6 +30,7 @@ import { changeSetContainsChanges, createChangeSet } from 'dbgate-datalib'; import localforage from 'localforage'; import { onMount, tick } from 'svelte'; + import _ from 'lodash'; import ToolStripCommandButton from '../buttons/ToolStripCommandButton.svelte'; import ToolStripCommandSplitButton from '../buttons/ToolStripCommandSplitButton.svelte'; @@ -129,7 +130,13 @@ await apiCall('archive/modify-file', { folder: archiveFolder, file: archiveFile, - changeSet: $changeSetStore.value, + changeSet: { + ...$changeSetStore.value, + updates: $changeSetStore.value.updates.map(update => ({ + ...update, + fields: _.mapValues(update.fields, (v, k) => (v === undefined ? { $$undefined$$: true } : v)), + })), + }, }); await afterSaveChangeSet(); } @@ -172,6 +179,10 @@ + + + + diff --git a/packages/web/src/tabs/CollectionDataTab.svelte b/packages/web/src/tabs/CollectionDataTab.svelte index 2dd8fb6d6..03f9fa85d 100644 --- a/packages/web/src/tabs/CollectionDataTab.svelte +++ b/packages/web/src/tabs/CollectionDataTab.svelte @@ -121,7 +121,13 @@ const resp = await apiCall('database-connections/update-collection', { conid, database, - changeSet, + changeSet: { + ...changeSet, + updates: changeSet.updates.map(update => ({ + ...update, + fields: _.mapValues(update.fields, (v, k) => (v === undefined ? { $$undefined$$: true } : v)), + })), + }, }); const { errorMessage } = resp || {}; if (errorMessage) { @@ -206,6 +212,7 @@ + diff --git a/packages/web/src/utility/clipboard.ts b/packages/web/src/utility/clipboard.ts index f1885763d..7adf11eae 100644 --- a/packages/web/src/utility/clipboard.ts +++ b/packages/web/src/utility/clipboard.ts @@ -1,6 +1,7 @@ import _ from 'lodash'; import { arrayToHexString, stringifyCellValue } from 'dbgate-tools'; import yaml from 'js-yaml'; +import { DataEditorTypesBehaviour } from 'dbgate-types'; export function copyTextToClipboard(text) { const oldFocus = document.activeElement; @@ -71,13 +72,13 @@ export async function getClipboardText() { return await navigator.clipboard.readText(); } -export function extractRowCopiedValue(row, col) { +export function extractRowCopiedValue(row, col, editorTypes?: DataEditorTypesBehaviour) { let value = row[col]; if (value === undefined) value = _.get(row, col); - return stringifyCellValue(value); + return stringifyCellValue(value, 'exportIntent', editorTypes).value; } -const clipboardHeadersFormatter = (delimiter) => (columns) => { +const clipboardHeadersFormatter = delimiter => columns => { return columns.join(delimiter); }; diff --git a/plugins/dbgate-plugin-mongo/package.json b/plugins/dbgate-plugin-mongo/package.json index 1a0b80879..3d28ffd57 100644 --- a/plugins/dbgate-plugin-mongo/package.json +++ b/plugins/dbgate-plugin-mongo/package.json @@ -31,14 +31,15 @@ "prepublishOnly": "yarn build" }, "devDependencies": { + "bson": "^6.8.0", "dbgate-plugin-tools": "^1.0.7", "dbgate-query-splitter": "^4.10.1", - "lodash": "^4.17.21", - "webpack": "^5.91.0", - "webpack-cli": "^5.1.4", "dbgate-tools": "^5.0.0-alpha.1", "is-promise": "^4.0.0", + "lodash": "^4.17.21", "mongodb": "^6.3.0", - "mongodb-client-encryption": "^6.0.0" + "mongodb-client-encryption": "^6.0.0", + "webpack": "^5.91.0", + "webpack-cli": "^5.1.4" } -} \ No newline at end of file +} diff --git a/plugins/dbgate-plugin-mongo/src/backend/createBulkInsertStream.js b/plugins/dbgate-plugin-mongo/src/backend/createBulkInsertStream.js index bfe19cb17..2259873cf 100644 --- a/plugins/dbgate-plugin-mongo/src/backend/createBulkInsertStream.js +++ b/plugins/dbgate-plugin-mongo/src/backend/createBulkInsertStream.js @@ -1,5 +1,6 @@ const ObjectId = require('mongodb').ObjectId; const { getLogger } = global.DBGATE_PACKAGES['dbgate-tools']; +const { EJSON } = require('bson'); const logger = getLogger('mongoBulkInsert'); @@ -26,7 +27,7 @@ function createBulkInsertStream(driver, stream, pool, name, options) { ...row, }; } - writable.buffer.push(row); + writable.buffer.push(EJSON.deserialize(row)); }; writable.checkStructure = async () => { diff --git a/plugins/dbgate-plugin-mongo/src/backend/driver.js b/plugins/dbgate-plugin-mongo/src/backend/driver.js index f7b1aec2e..e7433cadf 100644 --- a/plugins/dbgate-plugin-mongo/src/backend/driver.js +++ b/plugins/dbgate-plugin-mongo/src/backend/driver.js @@ -3,9 +3,8 @@ const stream = require('stream'); const isPromise = require('is-promise'); const driverBase = require('../frontend/driver'); const Analyser = require('./Analyser'); -const MongoClient = require('mongodb').MongoClient; -const ObjectId = require('mongodb').ObjectId; -const AbstractCursor = require('mongodb').AbstractCursor; +const { MongoClient, ObjectId, AbstractCursor } = require('mongodb'); +const { EJSON } = require('bson'); const createBulkInsertStream = require('./createBulkInsertStream'); const { convertToMongoCondition, @@ -14,9 +13,7 @@ const { } = require('../frontend/convertToMongoCondition'); function transformMongoData(row) { - return _.cloneDeepWith(row, (x) => { - if (x && x.constructor == ObjectId) return { $oid: x.toString() }; - }); + return EJSON.serialize(row); } async function readCursor(cursor, options) { @@ -27,11 +24,7 @@ async function readCursor(cursor, options) { } function convertObjectId(condition) { - return _.cloneDeepWith(condition, (x) => { - if (x && x.$oid) { - return ObjectId.createFromHexString(x.$oid); - } - }); + return EJSON.deserialize(condition); } function findArrayResult(resValue) { @@ -252,8 +245,23 @@ const driver = { const db = await getScriptableDb(pool); exprValue = func(db, ObjectId.createFromHexString); + const pass = new stream.PassThrough({ + objectMode: true, + highWaterMark: 100, + }); + + exprValue + .forEach((row) => pass.write(transformMongoData(row))) + .then(() => { + pass.end(); + // pass.end(() => { + // pass.emit('end'); + // }) + }); + + return pass; // return directly stream without header row - return exprValue.stream(); + // return exprValue.stream(); // pass.write(structure || { __isDynamicStructure: true }); // exprValue.on('data', (row) => pass.write(row)); @@ -337,7 +345,14 @@ const driver = { } } else { const resdoc = await collection.updateOne(convertObjectId(update.condition), { - $set: convertObjectId(update.fields), + $set: convertObjectId(_.pickBy(update.fields, (v, k) => !v?.$$undefined$$)), + $unset: _.fromPairs( + Object.keys(update.fields) + .filter((k) => update.fields[k]?.$$undefined$$) + .map((k) => [k, '']) + ), + + // $set: convertObjectId(update.fields), }); res.updated.push(resdoc._id); } diff --git a/plugins/dbgate-plugin-mongo/src/frontend/driver.js b/plugins/dbgate-plugin-mongo/src/frontend/driver.js index b24d07d14..4f3f1bfea 100644 --- a/plugins/dbgate-plugin-mongo/src/frontend/driver.js +++ b/plugins/dbgate-plugin-mongo/src/frontend/driver.js @@ -2,6 +2,8 @@ const { driverBase } = global.DBGATE_PACKAGES['dbgate-tools']; const { convertToMongoCondition, convertToMongoSort } = require('./convertToMongoCondition'); const Dumper = require('./Dumper'); const { mongoSplitterOptions } = require('dbgate-query-splitter/lib/options'); +const _pickBy = require('lodash/pickBy'); +const _fromPairs = require('lodash/fromPairs'); function jsonStringifyWithObjectId(obj) { return JSON.stringify(obj, undefined, 2).replace( @@ -96,7 +98,12 @@ const driver = { res += `db.${update.pureName}.updateOne(${jsonStringifyWithObjectId( update.condition )}, ${jsonStringifyWithObjectId({ - $set: update.fields, + $set: _pickBy(update.fields, (v, k) => v !== undefined), + $unset: _fromPairs( + Object.keys(update.fields) + .filter((k) => update.fields[k] === undefined) + .map((k) => [k, '']) + ), })});\n`; } } @@ -122,6 +129,27 @@ const driver = { sort: convertToMongoSort(sort) || {}, }; }, + + dataEditorTypesBehaviour: { + parseJsonNull: true, + parseJsonBoolean: true, + parseNumber: true, + parseJsonArray: true, + parseJsonObject: true, + parseObjectIdAsDollar: true, + parseDateAsDollar: true, + + explicitDataType: true, + supportNumberType: true, + supportStringType: true, + supportBooleanType: true, + supportDateType: true, + supportJsonType: true, + supportObjectIdType: true, + supportNullType: true, + + supportFieldRemoval: true, + }, }; module.exports = driver; diff --git a/yarn.lock b/yarn.lock index 817e109d5..84be0550e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2458,6 +2458,11 @@ bson@^6.7.0: resolved "https://registry.yarnpkg.com/bson/-/bson-6.7.0.tgz#51973b132cdc424c8372fda3cb43e3e3e2ae2227" integrity sha512-w2IquM5mYzYZv6rs3uN2DZTOBe2a0zXLj53TGDqwF4l6Sz/XsISrisXOJihArF9+BZ6Cq/GjVht7Sjfmri7ytQ== +bson@^6.8.0: + version "6.8.0" + resolved "https://registry.yarnpkg.com/bson/-/bson-6.8.0.tgz#5063c41ba2437c2b8ff851b50d9e36cb7aaa7525" + integrity sha512-iOJg8pr7wq2tg/zSlCCHMi3hMm5JTOxLTagf3zxhcenHsFp+c6uOs6K7W5UE7A4QIJGtqh/ZovFNMP4mOPJynQ== + buffer-crc32@^0.2.5: version "0.2.13" resolved "https://registry.yarnpkg.com/buffer-crc32/-/buffer-crc32-0.2.13.tgz#0d333e3f00eac50aa1454abd30ef8c2a5d9a7242"