Merge branch 'grid-data-types'

packages/api/src/shell/modifyJsonLinesReader.js

@@ -61,10 +61,13 @@ class ParseStream extends stream.Transform {
           if (update.document) {
             obj = update.document;
           } else {
-            obj = {
+            obj = _.omitBy(
+              {
                 ...obj,
                 ...update.fields,
-            };
+              },
+              (v, k) => v.$$undefined$$
+            );
           }
         }
 

packages/datalib/src/GridDisplay.ts

@@ -104,15 +104,21 @@ export abstract class GridDisplay {
   setColumnVisibility(uniquePath: string[], isVisible: boolean) {
     const uniqueName = uniquePath.join('.');
     if (uniquePath.length == 1) {
-      this.includeInColumnSet('hiddenColumns', uniqueName, !isVisible);
+      this.includeInColumnSet([
+        { field: 'hiddenColumns', uniqueName, isIncluded: !isVisible },
+        isVisible == false && this.isDynamicStructure && { field: 'addedColumns', uniqueName, isIncluded: false },
+      ]);
     } else {
-      this.includeInColumnSet('addedColumns', uniqueName, isVisible);
+      this.includeInColumnSet([{ field: 'addedColumns', uniqueName, isIncluded: isVisible }]);
       if (!this.isDynamicStructure) this.reload();
     }
   }
 
   addDynamicColumn(name: string) {
-    this.includeInColumnSet('addedColumns', name, true);
+    this.includeInColumnSet([
+      { field: 'addedColumns', uniqueName: name, isIncluded: true },
+      { field: 'hiddenColumns', uniqueName: name, isIncluded: false },
+    ]);
   }
 
   focusColumns(uniqueNames: string[]) {
@@ -150,19 +156,30 @@ export abstract class GridDisplay {
     this.setCache(reloadDataCacheFunc);
   }
 
-  includeInColumnSet(field: keyof GridConfigColumns, uniqueName: string, isIncluded: boolean) {
+  includeInColumnSet(
-    // console.log('includeInColumnSet', field, uniqueName, isIncluded);
+    modifications: ({ field: keyof GridConfigColumns; uniqueName: string; isIncluded: boolean } | null)[]
-    if (isIncluded) {
+  ) {
-      this.setConfig(cfg => ({
+    this.setConfig(cfg => {
-        ...cfg,
+      let res = cfg;
-        [field]: [...(cfg[field] || []), uniqueName],
+      for (const modification of modifications) {
-      }));
+        if (!modification) {
-    } else {
+          continue;
-      this.setConfig(cfg => ({
-        ...cfg,
-        [field]: (cfg[field] || []).filter(x => x != uniqueName),
-      }));
         }
+        const { field, uniqueName, isIncluded } = modification;
+        if (isIncluded) {
+          res = {
+            ...res,
+            [field]: [...(cfg[field] || []), uniqueName],
+          };
+        } else {
+          res = {
+            ...res,
+            [field]: (cfg[field] || []).filter(x => x != uniqueName),
+          };
+        }
+      }
+      return res;
+    });
   }
 
   showAllColumns() {
@@ -355,7 +372,9 @@ export abstract class GridDisplay {
   }
 
   toggleExpandedColumn(uniqueName: string, value?: boolean) {
-    this.includeInColumnSet('expandedColumns', uniqueName, value == null ? !this.isExpandedColumn(uniqueName) : value);
+    this.includeInColumnSet([
+      { field: 'expandedColumns', uniqueName, isIncluded: value == null ? !this.isExpandedColumn(uniqueName) : value },
+    ]);
   }
 
   getFilter(uniqueName: string) {

packages/tools/src/driverBase.ts

@@ -161,4 +161,9 @@ export const driverBase = {
   getCollectionExportQueryJson(collection: string, condition: any, sort: any) {
     return null;
   },
+
+  dataEditorTypesBehaviour: {
+    parseSqlNull: true,
+    parseHexAsBuffer: true,
+  },
 };

packages/tools/src/stringTools.ts

@@ -1,6 +1,25 @@
 import _isString from 'lodash/isString';
 import _isArray from 'lodash/isArray';
+import _isNumber from 'lodash/isNumber';
 import _isPlainObject from 'lodash/isPlainObject';
+import _pad from 'lodash/pad';
+import { DataEditorTypesBehaviour } from 'dbgate-types';
+
+export type EditorDataType =
+  | 'null'
+  | 'objectid'
+  | 'string'
+  | 'number'
+  | 'object'
+  | 'date'
+  | 'array'
+  | 'boolean'
+  | 'unknown';
+
+const dateTimeStorageRegex =
+  /^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?(([Zz])|()|([\+|\-]([01][0-9]|2[0-3]):[0-5][0-9]))$/;
+
+const dateTimeParseRegex = /^(\d{4})-(\d{2})-(\d{2})[Tt ](\d{2}):(\d{2}):(\d{2})(\.[0-9]+)?(([Zz])|()|([\+|\-]([01][0-9]|2[0-3]):[0-5][0-9]))$/;
 
 export function arrayToHexString(byteArray) {
   return byteArray.reduce((output, elem) => output + ('0' + elem.toString(16)).slice(-2), '').toUpperCase();
@@ -15,11 +34,14 @@ export function hexStringToArray(inputString) {
   return res;
 }
 
-export function parseCellValue(value) {
+export function parseCellValue(value, editorTypes?: DataEditorTypesBehaviour) {
   if (!_isString(value)) return value;
 
+  if (editorTypes?.parseSqlNull) {
     if (value == '(NULL)') return null;
+  }
 
+  if (editorTypes?.parseHexAsBuffer) {
     const mHex = value.match(/^0x([0-9a-fA-F][0-9a-fA-F])+$/);
     if (mHex) {
       return {
@@ -27,24 +49,252 @@ export function parseCellValue(value) {
         data: hexStringToArray(value.substring(2)),
       };
     }
+  }
 
+  if (editorTypes?.parseObjectIdAsDollar) {
     const mOid = value.match(/^ObjectId\("([0-9a-f]{24})"\)$/);
     if (mOid) {
       return { $oid: mOid[1] };
     }
+  }
+
+  if (editorTypes?.parseDateAsDollar) {
+    const m = value.match(dateTimeParseRegex);
+    if (m) {
+      return {
+        $date: `${m[1]}-${m[2]}-${m[3]}T${m[4]}:${m[5]}:${m[6]}Z`,
+      };
+    }
+  }
+
+  if (editorTypes?.parseJsonNull) {
+    if (value == 'null') return null;
+  }
+
+  if (editorTypes?.parseJsonBoolean) {
+    if (value == 'true') return true;
+    if (value == 'false') return false;
+  }
+
+  if (editorTypes?.parseNumber) {
+    if (/^-?[0-9]+(?:\.[0-9]+)?$/.test(value)) {
+      return parseFloat(value);
+    }
+  }
+
+  if (editorTypes?.parseJsonArray || editorTypes?.parseJsonObject) {
+    const jsonValue = safeJsonParse(value);
+    if (_isPlainObject(jsonValue) && editorTypes?.parseJsonObject) return jsonValue;
+    if (_isArray(jsonValue) && editorTypes?.parseJsonArray) return jsonValue;
+  }
 
   return value;
 }
 
-export function stringifyCellValue(value) {
+function parseFunc_ObjectIdAsDollar(value) {
-  if (value === null) return '(NULL)';
+  if (value?.$oid) return value;
-  if (value === undefined) return '(NoField)';
+  if (_isString(value)) {
-  if (value?.type == 'Buffer' && _isArray(value.data)) return '0x' + arrayToHexString(value.data);
+    if (value.match(/^[0-9a-f]{24}$/)) return { $oid: value };
-  if (value?.$oid) return `ObjectId("${value?.$oid}")`;
+    const mOid = value.match(/^ObjectId\("([0-9a-f]{24})"\)$/);
-  if (_isPlainObject(value) || _isArray(value)) return JSON.stringify(value);
+    if (mOid) {
+      return { $oid: mOid[1] };
+    }
+  }
   return value;
 }
 
+function parseFunc_DateAsDollar(value) {
+  if (value?.$date) return value;
+  if (_isString(value)) {
+    const m = value.match(dateTimeParseRegex);
+    if (m) {
+      return { $date: `${m[1]}-${m[2]}-${m[3]}T${m[4]}:${m[5]}:${m[6]}Z` };
+    }
+  }
+  return value;
+}
+
+function makeBulletString(value) {
+  return _pad('', value.length, '•');
+}
+
+function highlightSpecialCharacters(value) {
+  value = value.replace(/\n/g, '↲');
+  value = value.replace(/\r/g, '');
+  value = value.replace(/^(\s+)/, makeBulletString);
+  value = value.replace(/(\s+)$/, makeBulletString);
+  value = value.replace(/(\s\s+)/g, makeBulletString);
+  return value;
+}
+
+function stringifyJsonToGrid(value): ReturnType<typeof stringifyCellValue> {
+  if (_isPlainObject(value)) {
+    const svalue = JSON.stringify(value, undefined, 2);
+    if (svalue.length < 100) {
+      return { value: svalue, gridStyle: 'nullCellStyle' };
+    } else {
+      return { value: '(JSON)', gridStyle: 'nullCellStyle', gridTitle: svalue };
+    }
+  }
+  if (_isArray(value)) {
+    return {
+      value: `[${value.length} items]`,
+      gridStyle: 'nullCellStyle',
+      gridTitle: value.map(x => JSON.stringify(x)).join('\n'),
+    };
+  }
+  return { value: '(JSON)', gridStyle: 'nullCellStyle' };
+}
+
+export function stringifyCellValue(
+  value,
+  intent: 'gridCellIntent' | 'inlineEditorIntent' | 'multilineEditorIntent' | 'stringConversionIntent' | 'exportIntent',
+  editorTypes?: DataEditorTypesBehaviour,
+  gridFormattingOptions?: { useThousandsSeparator?: boolean },
+  jsonParsedValue?: any
+): {
+  value: string;
+  gridStyle?: 'textCellStyle' | 'valueCellStyle' | 'nullCellStyle'; // only for gridCellIntent
+  gridTitle?: string; // only for gridCellIntent
+} {
+  if (editorTypes?.parseSqlNull) {
+    if (value === null) {
+      switch (intent) {
+        case 'exportIntent':
+          return { value: '' };
+        default:
+          return { value: '(NULL)', gridStyle: 'nullCellStyle' };
+      }
+    }
+  }
+  if (value === undefined) {
+    switch (intent) {
+      case 'gridCellIntent':
+        return { value: '(No Field)', gridStyle: 'nullCellStyle' };
+      default:
+        return { value: '' };
+    }
+  }
+  if (editorTypes?.parseJsonNull) {
+    if (value === null) {
+      return { value: 'null', gridStyle: 'valueCellStyle' };
+    }
+  }
+
+  if (value === true) return { value: 'true', gridStyle: 'valueCellStyle' };
+  if (value === false) return { value: 'false', gridStyle: 'valueCellStyle' };
+
+  if (editorTypes?.parseHexAsBuffer) {
+    if (value?.type == 'Buffer' && _isArray(value.data)) {
+      return { value: '0x' + arrayToHexString(value.data), gridStyle: 'valueCellStyle' };
+    }
+  }
+  if (editorTypes?.parseObjectIdAsDollar) {
+    if (value?.$oid) {
+      switch (intent) {
+        case 'exportIntent':
+        case 'stringConversionIntent':
+          return { value: value.$oid };
+        default:
+          return { value: `ObjectId("${value.$oid}")`, gridStyle: 'valueCellStyle' };
+      }
+    }
+  }
+
+  if (editorTypes?.parseDateAsDollar) {
+    if (value?.$date) {
+      switch (intent) {
+        case 'exportIntent':
+        case 'stringConversionIntent':
+          return { value: value.$date };
+        default:
+          const m = value.$date.match(dateTimeStorageRegex);
+          if (m) {
+            return { value: `${m[1]}-${m[2]}-${m[3]} ${m[4]}:${m[5]}:${m[6]}`, gridStyle: 'valueCellStyle' };
+          } else {
+            return { value: value.$date.replaCE('T', ' '), gridStyle: 'valueCellStyle' };
+          }
+      }
+    }
+  }
+
+  if (_isArray(value)) {
+    switch (intent) {
+      case 'gridCellIntent':
+        return stringifyJsonToGrid(value);
+      case 'multilineEditorIntent':
+        return { value: JSON.stringify(value, null, 2) };
+      default:
+        return { value: JSON.stringify(value), gridStyle: 'valueCellStyle' };
+    }
+  }
+
+  if (_isPlainObject(value)) {
+    switch (intent) {
+      case 'gridCellIntent':
+        return stringifyJsonToGrid(value);
+      case 'multilineEditorIntent':
+        return { value: JSON.stringify(value, null, 2) };
+      default:
+        return { value: JSON.stringify(value), gridStyle: 'valueCellStyle' };
+    }
+  }
+
+  if (_isNumber(value)) {
+    switch (intent) {
+      case 'gridCellIntent':
+        return {
+          value:
+            gridFormattingOptions?.useThousandsSeparator && (value >= 10000 || value <= -10000)
+              ? value.toLocaleString()
+              : value.toString(),
+          gridStyle: 'valueCellStyle',
+        };
+      default:
+        return { value: value.toString() };
+    }
+  }
+
+  if (_isString(value)) {
+    switch (intent) {
+      case 'gridCellIntent':
+        if (jsonParsedValue && !editorTypes?.explicitDataType) {
+          return stringifyJsonToGrid(jsonParsedValue);
+        } else {
+          if (!editorTypes?.explicitDataType) {
+            // reformat datetime for implicit date types
+            const m = value.match(dateTimeStorageRegex);
+            if (m) {
+              return {
+                value: `${m[1]}-${m[2]}-${m[3]} ${m[4]}:${m[5]}:${m[6]}`,
+                gridStyle: 'valueCellStyle',
+              };
+            }
+          }
+          return { value: highlightSpecialCharacters(value), gridStyle: 'textCellStyle' };
+        }
+      default:
+        return { value: value };
+    }
+  }
+
+  if (value === null || value === undefined) {
+    switch (intent) {
+      case 'gridCellIntent':
+        return { value: '(n/a)', gridStyle: 'nullCellStyle' };
+      default:
+        return { value: '' };
+    }
+  }
+
+  switch (intent) {
+    case 'gridCellIntent':
+      return { value: '(Unknown)', gridStyle: 'nullCellStyle' };
+    default:
+      return { value: '' };
+  }
+}
+
 export function safeJsonParse(json, defaultValue?, logError = false) {
   if (_isArray(json) || _isPlainObject(json)) {
     return json;
@@ -59,6 +309,28 @@ export function safeJsonParse(json, defaultValue?, logError = false) {
   }
 }
 
+export function shouldOpenMultilineDialog(value) {
+  if (_isString(value)) {
+    if (value.includes('\n')) {
+      return true;
+    }
+    const parsed = safeJsonParse(value);
+    if (parsed && (_isPlainObject(parsed) || _isArray(parsed))) {
+      return true;
+    }
+  }
+  if (value?.$oid) {
+    return false;
+  }
+  if (value?.$date) {
+    return false;
+  }
+  if (_isPlainObject(value) || _isArray(value)) {
+    return true;
+  }
+  return false;
+}
+
 export function isJsonLikeLongString(value) {
   return _isString(value) && value.length > 100 && value.match(/^\s*\{.*\}\s*$|^\s*\[.*\]\s*$/);
 }
@@ -127,3 +399,67 @@ export function parseSqlDefaultValue(value: string) {
   }
   return undefined;
 }
+
+export function detectCellDataType(value): EditorDataType {
+  if (value === null) return 'null';
+  if (value?.$oid) return 'objectid';
+  if (value?.$date) return 'date';
+  if (_isString(value)) return 'string';
+  if (_isNumber(value)) return 'number';
+  if (_isPlainObject(value)) return 'object';
+  if (_isArray(value)) return 'array';
+  if (value === true || value === false) return 'boolean';
+  return 'unknown';
+}
+
+export function detectTypeIcon(value) {
+  switch (detectCellDataType(value)) {
+    case 'null':
+      return 'icon type-null';
+    case 'objectid':
+      return 'icon type-objectid';
+    case 'date':
+      return 'icon type-date';
+    case 'string':
+      return 'icon type-string';
+    case 'number':
+      return 'icon type-number';
+    case 'object':
+      return 'icon type-object';
+    case 'array':
+      return 'icon type-array';
+    case 'boolean':
+      return 'icon type-boolean';
+    default:
+      return 'icon type-unknown';
+  }
+}
+
+export function getConvertValueMenu(value, onSetValue, editorTypes?: DataEditorTypesBehaviour) {
+  return [
+    editorTypes?.supportStringType && {
+      text: 'String',
+      onClick: () => onSetValue(stringifyCellValue(value, 'stringConversionIntent', editorTypes).value),
+    },
+    editorTypes?.supportNumberType && { text: 'Number', onClick: () => onSetValue(parseFloat(value)) },
+    editorTypes?.supportNullType && { text: 'Null', onClick: () => onSetValue(null) },
+    editorTypes?.supportBooleanType && {
+      text: 'Boolean',
+      onClick: () => onSetValue(value?.toString()?.toLowerCase() == 'true' || value == '1'),
+    },
+    editorTypes?.supportObjectIdType && {
+      text: 'ObjectId',
+      onClick: () => onSetValue(parseFunc_ObjectIdAsDollar(value)),
+    },
+    editorTypes?.supportDateType && { text: 'Date', onClick: () => onSetValue(parseFunc_DateAsDollar(value)) },
+    editorTypes?.supportJsonType && {
+      text: 'JSON',
+      onClick: () => {
+        const jsonValue = safeJsonParse(value);
+        if (jsonValue != null) {
+          onSetValue(jsonValue);
+        }
+      },
+    },
+  ];
+}

packages/types/engines.d.ts

@@ -93,6 +93,29 @@ export interface CollectionSortDefinitionItem {
 
 export type CollectionSortDefinition = CollectionSortDefinitionItem[];
 
+export interface DataEditorTypesBehaviour {
+  parseSqlNull?: boolean;
+  parseJsonNull?: boolean;
+  parseJsonBoolean?: boolean;
+  parseNumber?: boolean;
+  parseJsonArray?: boolean;
+  parseJsonObject?: boolean;
+  parseHexAsBuffer?: boolean;
+  parseObjectIdAsDollar?: boolean;
+  parseDateAsDollar?: boolean;
+
+  explicitDataType?: boolean;
+  supportNumberType?: boolean;
+  supportStringType?: boolean;
+  supportBooleanType?: boolean;
+  supportDateType?: boolean;
+  supportNullType?: boolean;
+  supportJsonType?: boolean;
+  supportObjectIdType?: boolean;
+
+  supportFieldRemoval?: boolean;
+}
+
 export interface FilterBehaviourProvider {
   getFilterBehaviour(dataType: string, standardFilterBehaviours: { [id: string]: FilterBehaviour }): FilterBehaviour;
 }
@@ -105,6 +128,7 @@ export interface EngineDriver extends FilterBehaviourProvider {
   editorMode?: string;
   readOnlySessions: boolean;
   supportedKeyTypes: SupportedDbKeyType[];
+  dataEditorTypesBehaviour: DataEditorTypesBehaviour;
   supportsDatabaseUrl?: boolean;
   supportsDatabaseDump?: boolean;
   supportsServerSummary?: boolean;

packages/web/src/datagrid/CellValue.svelte

@@ -1,105 +1,34 @@
-<script context="module">
-  function makeBulletString(value) {
-    return _.pad('', value.length, '•');
-  }
-
-  function highlightSpecialCharacters(value) {
-    value = value.replace(/\n/g, '↲');
-    value = value.replace(/\r/g, '');
-    value = value.replace(/^(\s+)/, makeBulletString);
-    value = value.replace(/(\s+)$/, makeBulletString);
-    value = value.replace(/(\s\s+)/g, makeBulletString);
-    return value;
-  }
-
-  // const dateTimeRegex = /^\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d(\.\d\d\d)?Z?$/;
-  const dateTimeRegex =
-    /^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?(([Zz])|()|([\+|\-]([01][0-9]|2[0-3]):[0-5][0-9]))$/;
-
-  function formatNumber(value) {
-    if (value >= 10000 || value <= -10000) {
-      if (getBoolSettingsValue('dataGrid.thousandsSeparator', false)) {
-        return value.toLocaleString();
-      } else {
-        return value.toString();
-      }
-    }
-
-    return value.toString();
-  }
-
-  function formatDateTime(testedString) {
-    const m = testedString.match(dateTimeRegex);
-    return `${m[1]}-${m[2]}-${m[3]} ${m[4]}:${m[5]}:${m[6]}`;
-  }
-</script>
-
 <script lang="ts">
   import _ from 'lodash';
   import { getBoolSettingsValue } from '../settings/settingsTools';
-  import { arrayToHexString } from 'dbgate-tools';
+  import { stringifyCellValue } from 'dbgate-tools';
 
   export let rowData;
   export let value;
   export let jsonParsedValue = undefined;
+  export let editorTypes;
+
+  $: stringified = stringifyCellValue(
+    value,
+    'gridCellIntent',
+    editorTypes,
+    { useThousandsSeparator: getBoolSettingsValue('dataGrid.thousandsSeparator', false) },
+    jsonParsedValue
+  );
 </script>
 
 {#if rowData == null}
   <span class="null">(No row)</span>
-{:else if value === null}
-  <span class="null">(NULL)</span>
-{:else if value === undefined}
-  <span class="null">(No field)</span>
-{:else if _.isDate(value)}
-  {value.toString()}
-{:else if value === true}
-  <span class="value">true</span>
-{:else if value === false}
-  <span class="value">false</span>
-{:else if _.isNumber(value)}
-  <span class="value">{formatNumber(value)}</span>
-{:else if _.isString(value) && !jsonParsedValue}
-  {#if dateTimeRegex.test(value)}
-    <span class="value">
-      {formatDateTime(value)}
-    </span>
-  {:else}
-    {highlightSpecialCharacters(value)}
-  {/if}
-{:else if value?.type == 'Buffer' && _.isArray(value.data)}
-  {#if value.data.length <= 16}
-    <span class="value">{'0x' + arrayToHexString(value.data)}</span>
-  {:else}
-    <span class="null">({value.data.length} bytes)</span>
-  {/if}
-{:else if value.$oid}
-  <span class="value">ObjectId("{value.$oid}")</span>
-{:else if _.isPlainObject(value)}
-  {@const svalue = JSON.stringify(value, undefined, 2)}
-  <span class="null" title={svalue}
-    >{#if svalue.length < 100}{JSON.stringify(value)}{:else}(JSON){/if}</span
-  >
-{:else if _.isArray(value)}
-  <span class="null" title={value.map(x => JSON.stringify(x)).join('\n')}>[{value.length} items]</span>
-{:else if _.isPlainObject(jsonParsedValue)}
-  {@const svalue = JSON.stringify(jsonParsedValue, undefined, 2)}
-  <span class="null" title={svalue}
-    >{#if svalue.length < 100}{JSON.stringify(jsonParsedValue)}{:else}(JSON){/if}</span
-  >
-{:else if _.isArray(jsonParsedValue)}
-  <span class="null" title={jsonParsedValue.map(x => JSON.stringify(x)).join('\n')}
-    >[{jsonParsedValue.length} items]</span
-  >
 {:else}
-  {value.toString()}
+  <span class={stringified.gridStyle} title={stringified.gridTitle}>{stringified.value}</span>
 {/if}
 
 <style>
-  .null {
+  .nullCellStyle {
     color: var(--theme-font-3);
     font-style: italic;
   }
-  .value {
+  .valueCellStyle {
     color: var(--theme-icon-green);
   }
 </style>

packages/web/src/datagrid/ColumnManager.svelte

@@ -182,6 +182,13 @@
           header: 'Add new column',
           onConfirm: name => {
             display.addDynamicColumn(name);
+            tick().then(() => {
+              selectedColumns = [name];
+              currentColumnUniqueName = name;
+              if (!isJsonView) {
+                display.focusColumns(selectedColumns);
+              }
+            });
           },
         });
       }}>Add</InlineButton

packages/web/src/datagrid/DataGridCell.svelte

@@ -1,13 +1,11 @@
 <script lang="ts">
-  import _ from 'lodash';
+  import _, { isPlainObject } from 'lodash';
   import ShowFormButton from '../formview/ShowFormButton.svelte';
-  import { isJsonLikeLongString, safeJsonParse } from 'dbgate-tools';
+  import { detectTypeIcon, getConvertValueMenu, isJsonLikeLongString, safeJsonParse } from 'dbgate-tools';
   import { openJsonDocument } from '../tabs/JsonTab.svelte';
-  import openNewTab from '../utility/openNewTab';
   import CellValue from './CellValue.svelte';
-  import { showModal } from '../modals/modalTools';
-  import EditCellDataModal from '../modals/EditCellDataModal.svelte';
   import { openJsonLinesData } from '../utility/openJsonLinesData';
+  import ShowFormDropDownButton from '../formview/ShowFormDropDownButton.svelte';
 
   export let rowIndex;
   export let col;
@@ -33,6 +31,7 @@
   export let isCurrentCell = false;
   export let onDictionaryLookup = null;
   export let onSetValue;
+  export let editorTypes = null;
 
   $: value = col.isStructured ? _.get(rowData || {}, col.uniquePath) : (rowData || {})[col.uniqueName];
 
@@ -51,7 +50,9 @@
   $: style = computeStyle(maxWidth, col);
 
   $: isJson = _.isPlainObject(value) && !(value?.type == 'Buffer' && _.isArray(value.data)) && !value.$oid;
-  $: jsonParsedValue = isJsonLikeLongString(value) ? safeJsonParse(value) : null;
+
+  // don't parse JSON for explicit data types
+  $: jsonParsedValue = !editorTypes?.explicitDataType && isJsonLikeLongString(value) ? safeJsonParse(value) : null;
 </script>
 
 <td
@@ -68,7 +69,7 @@
   class:isFocusedColumn
   {style}
 >
-  <CellValue {rowData} {value} {jsonParsedValue} />
+  <CellValue {rowData} {value} {jsonParsedValue} {editorTypes} />
 
   {#if allowHintField && rowData && _.some(col.hintColumnNames, hintColumnName => rowData[hintColumnName])}
     <span class="hint"
@@ -76,23 +77,38 @@
     >
   {/if}
 
-  {#if col.foreignKey && rowData && rowData[col.uniqueName] && !isCurrentCell}
+  {#if editorTypes?.explicitDataType}
+    {#if value !== undefined}
+      <ShowFormDropDownButton
+        icon={detectTypeIcon(value)}
+        menu={() => getConvertValueMenu(value, onSetValue, editorTypes)}
+      />
+    {/if}
+    {#if _.isPlainObject(value)}
+      <ShowFormButton secondary icon="icon open-in-new" on:click={() => openJsonDocument(value, undefined, true)} />
+    {/if}
+    {#if _.isArray(value)}
+      <ShowFormButton
+        secondary
+        icon="icon open-in-new"
+        on:click={() => {
+          if (_.every(value, x => _.isPlainObject(x))) {
+            openJsonLinesData(value);
+          } else {
+            openJsonDocument(value, undefined, true);
+          }
+        }}
+      />
+    {/if}
+  {:else if col.foreignKey && rowData && rowData[col.uniqueName] && !isCurrentCell}
     <ShowFormButton on:click={() => onSetFormView(rowData, col)} />
-  {/if}
+  {:else if col.foreignKey && isCurrentCell && onDictionaryLookup}
-
-  {#if col.foreignKey && isCurrentCell && onDictionaryLookup}
     <ShowFormButton icon="icon dots-horizontal" on:click={onDictionaryLookup} />
-  {/if}
+  {:else if isJson}
-
-  {#if isJson}
     <ShowFormButton icon="icon open-in-new" on:click={() => openJsonDocument(value, undefined, true)} />
-  {/if}
+  {:else if jsonParsedValue && _.isPlainObject(jsonParsedValue)}
-
-  {#if jsonParsedValue && _.isPlainObject(jsonParsedValue)}
     <ShowFormButton icon="icon open-in-new" on:click={() => openJsonDocument(jsonParsedValue, undefined, true)} />
-  {/if}
+  {:else if _.isArray(jsonParsedValue || value)}
-
-  {#if _.isArray(jsonParsedValue || value)}
     <ShowFormButton
       icon="icon open-in-new"
       on:click={() => {

packages/web/src/datagrid/DataGridCore.svelte

@@ -66,6 +66,16 @@
     onClick: () => getCurrentDataGrid().insertNewRow(),
   });
 
+  registerCommand({
+    id: 'dataGrid.addNewColumn',
+    category: 'Data grid',
+    name: 'Add new column',
+    toolbarName: 'New column',
+    icon: 'icon add-column',
+    testEnabled: () => getCurrentDataGrid()?.addNewColumnEnabled(),
+    onClick: () => getCurrentDataGrid().addNewColumn(),
+  });
+
   registerCommand({
     id: 'dataGrid.cloneRows',
     category: 'Data grid',
@@ -81,10 +91,21 @@
     category: 'Data grid',
     name: 'Set NULL',
     keyText: 'CtrlOrCommand+0',
-    testEnabled: () => getCurrentDataGrid()?.getGrider()?.editable,
+    testEnabled: () =>
+      getCurrentDataGrid()?.getGrider()?.editable && !getCurrentDataGrid()?.getEditorTypes()?.supportFieldRemoval,
     onClick: () => getCurrentDataGrid().setFixedValue(null),
   });
 
+  registerCommand({
+    id: 'dataGrid.removeField',
+    category: 'Data grid',
+    name: 'Remove field',
+    keyText: 'CtrlOrCommand+0',
+    testEnabled: () =>
+      getCurrentDataGrid()?.getGrider()?.editable && getCurrentDataGrid()?.getEditorTypes()?.supportFieldRemoval,
+    onClick: () => getCurrentDataGrid().setFixedValue(undefined),
+  });
+
   registerCommand({
     id: 'dataGrid.undo',
     category: 'Data grid',
@@ -343,7 +364,13 @@
 
 <script lang="ts">
   import { GridDisplay } from 'dbgate-datalib';
-  import { driverBase, parseCellValue, detectSqlFilterBehaviour } from 'dbgate-tools';
+  import {
+    driverBase,
+    parseCellValue,
+    detectSqlFilterBehaviour,
+    stringifyCellValue,
+    shouldOpenMultilineDialog,
+  } from 'dbgate-tools';
   import { getContext, onDestroy } from 'svelte';
   import _, { map } from 'lodash';
   import registerCommand from '../commands/registerCommand';
@@ -397,11 +424,12 @@
   import { isCtrlOrCommandKey, isMac } from '../utility/common';
   import { createGeoJsonFromSelection, selectionCouldBeShownOnMap } from '../elements/SelectionMapView.svelte';
   import ErrorMessageModal from '../modals/ErrorMessageModal.svelte';
-  import EditCellDataModal, { shouldOpenMultilineDialog } from '../modals/EditCellDataModal.svelte';
+  import EditCellDataModal from '../modals/EditCellDataModal.svelte';
   import { getDatabaseInfo, useDatabaseStatus } from '../utility/metadataLoaders';
   import { showSnackbarSuccess } from '../utility/snackbar';
   import { openJsonLinesData } from '../utility/openJsonLinesData';
   import contextMenuActivator from '../utility/contextMenuActivator';
+  import InputTextModal from '../modals/InputTextModal.svelte';
 
   export let onLoadNextData = undefined;
   export let grider = undefined;
@@ -440,6 +468,8 @@
 
   export const activator = createActivator('DataGridCore', false);
 
+  export let dataEditorTypesBehaviourOverride = null;
+
   const wheelRowCount = 5;
   const tabVisible: any = getContext('tabVisible');
 
@@ -535,6 +565,24 @@
     grider.endUpdate();
   }
 
+  export function addNewColumnEnabled() {
+    return getGrider()?.editable && isDynamicStructure;
+  }
+
+  export function addNewColumn() {
+    showModal(InputTextModal, {
+      value: '',
+      label: 'Column name',
+      header: 'Add new column',
+      onConfirm: name => {
+        display.addDynamicColumn(name);
+        tick().then(() => {
+          display.focusColumns([name]);
+        });
+      },
+    });
+  }
+
   export async function insertNewRow() {
     if (!grider.canInsert) return;
     const rowIndex = grider.insertRow();
@@ -811,11 +859,16 @@
     const cellData = rowData[realColumnUniqueNames[currentCell[1]]];
 
     showModal(EditCellDataModal, {
-      value: cellData?.toString() || '',
+      value: cellData,
+      dataEditorTypesBehaviour: getEditorTypes(),
       onSave: value => grider.setCellValue(currentCell[0], realColumnUniqueNames[currentCell[1]], value),
     });
   }
 
+  export function getEditorTypes() {
+    return dataEditorTypesBehaviourOverride ?? display?.driver?.dataEditorTypesBehaviour;
+  }
+
   export function addJsonDocumentEnabled() {
     return grider.editable;
   }
@@ -1246,6 +1299,7 @@
     const cellData = rowData[realColumnUniqueNames[cell[1]]];
     if (shouldOpenMultilineDialog(cellData)) {
       showModal(EditCellDataModal, {
+        dataEditorTypesBehaviour: getEditorTypes(),
         value: cellData,
         onSave: value => grider.setCellValue(cell[0], realColumnUniqueNames[cell[1]], value),
       });
@@ -1557,7 +1611,7 @@
           }
           let colIndex = startCol;
           for (const cell of rowData) {
-            setCellValue([rowIndex, colIndex], parseCellValue(cell));
+            setCellValue([rowIndex, colIndex], parseCellValue(cell, getEditorTypes()));
             colIndex += 1;
           }
           rowIndex += 1;
@@ -1695,13 +1749,15 @@
     { command: 'dataGrid.deleteSelectedRows' },
     { command: 'dataGrid.insertNewRow' },
     { command: 'dataGrid.cloneRows' },
-    { command: 'dataGrid.setNull' },
+    { command: 'dataGrid.setNull', hideDisabled: true },
+    { command: 'dataGrid.removeField', hideDisabled: true },
     { placeTag: 'edit' },
     { divider: true },
     { command: 'dataGrid.findColumn' },
     { command: 'dataGrid.hideColumn' },
     { command: 'dataGrid.filterSelected' },
     { command: 'dataGrid.clearFilter' },
+    { command: 'dataGrid.addNewColumn', hideDisabled: true },
     { command: 'dataGrid.undo', hideDisabled: true },
     { command: 'dataGrid.redo', hideDisabled: true },
     { divider: true },
@@ -1941,6 +1997,7 @@
             {dispatchInsplaceEditor}
             {frameSelection}
             onSetFormView={formViewAvailable && display?.baseTable?.primaryKey ? handleSetFormView : null}
+            {dataEditorTypesBehaviourOverride}
           />
         {/each}
       </tbody>

packages/web/src/datagrid/DataGridRow.svelte

@@ -27,6 +27,8 @@
   export let database;
   export let driver;
 
+  export let dataEditorTypesBehaviourOverride = null;
+
   $: rowData = grider.getRowData(rowIndex);
   $: rowStatus = grider.getRowStatus(rowIndex);
 
@@ -59,9 +61,11 @@
         {inplaceEditorState}
         {dispatchInsplaceEditor}
         cellValue={rowData[col.uniqueName]}
-        options="{col.options}"
+        options={col.options}
-        canSelectMultipleOptions="{col.canSelectMultipleOptions}"
+        canSelectMultipleOptions={col.canSelectMultipleOptions}
         onSetValue={value => grider.setCellValue(rowIndex, col.uniqueName, value)}
+        {driver}
+        {dataEditorTypesBehaviourOverride}
         />
     {:else}
       <DataGridCell
@@ -70,6 +74,7 @@
         {col}
         {conid}
         {database}
+        editorTypes={dataEditorTypesBehaviourOverride ?? driver?.dataEditorTypesBehaviour}
         allowHintField={hintFieldsAllowed?.includes(col.uniqueName)}
         isSelected={frameSelection ? false : cellIsSelected(rowIndex, col.colIndex, selectedCells)}
         isCurrentCell={col.colIndex == currentCellColumn}

packages/web/src/datagrid/InplaceEditor.svelte

@@ -9,6 +9,8 @@
   export let cellValue;
   export let options;
   export let canSelectMultipleOptions;
+  export let driver;
+  export let dataEditorTypesBehaviourOverride = null;
 </script>
 
 <td class="editor">
@@ -21,6 +23,8 @@
         {onSetValue}
         {options}
         {canSelectMultipleOptions}
+        {driver}
+        {dataEditorTypesBehaviourOverride}
       />
     {:else}
       <InplaceInput
@@ -29,6 +33,8 @@
         {dispatchInsplaceEditor}
         {cellValue}
         {onSetValue}
+        {driver}
+        {dataEditorTypesBehaviourOverride}
       />
     {/if}
   </div>

packages/web/src/datagrid/InplaceInput.svelte

@@ -14,6 +14,9 @@
   export let onSetValue;
   export let width;
   export let cellValue;
+  export let driver;
+
+  export let dataEditorTypesBehaviourOverride = null;
 
   let domEditor;
   let showEditorButton = true;
@@ -22,6 +25,8 @@
 
   const isChangedRef = createRef(!!inplaceEditorState.text);
 
+  $: editorTypes = dataEditorTypesBehaviourOverride ?? driver?.dataEditorTypesBehaviour;
+
   function handleKeyDown(event) {
     showEditorButton = false;
 
@@ -32,7 +37,7 @@
         break;
       case keycodes.enter:
         if (isChangedRef.get()) {
-          onSetValue(parseCellValue(domEditor.value));
+          onSetValue(parseCellValue(domEditor.value, editorTypes));
           isChangedRef.set(false);
         }
         domEditor.blur();
@@ -41,7 +46,7 @@
         break;
       case keycodes.tab:
         if (isChangedRef.get()) {
-          onSetValue(parseCellValue(domEditor.value));
+          onSetValue(parseCellValue(domEditor.value, editorTypes));
           isChangedRef.set(false);
         }
         domEditor.blur();
@@ -51,7 +56,7 @@
       case keycodes.s:
         if (isCtrlOrCommandKey(event)) {
           if (isChangedRef.get()) {
-            onSetValue(parseCellValue(domEditor.value));
+            onSetValue(parseCellValue(domEditor.value, editorTypes));
             isChangedRef.set(false);
           }
           event.preventDefault();
@@ -63,7 +68,7 @@
 
   function handleBlur() {
     if (isChangedRef.get()) {
-      onSetValue(parseCellValue(domEditor.value));
+      onSetValue(parseCellValue(domEditor.value, editorTypes));
       // grider.setCellValue(rowIndex, uniqueName, editor.value);
       isChangedRef.set(false);
     }
@@ -71,7 +76,7 @@
   }
 
   onMount(() => {
-    domEditor.value = inplaceEditorState.text || stringifyCellValue(cellValue);
+    domEditor.value = inplaceEditorState.text || stringifyCellValue(cellValue, 'inlineEditorIntent', editorTypes).value;
     domEditor.focus();
     if (inplaceEditorState.selectAll) {
       domEditor.select();
@@ -102,7 +107,8 @@
       dispatchInsplaceEditor({ type: 'close' });
 
       showModal(EditCellDataModal, {
-        value: stringifyCellValue(cellValue),
+        value: cellValue,
+        dataEditorTypesBehaviour: editorTypes,
         onSave: onSetValue,
       });
     }}

packages/web/src/datagrid/InplaceSelect.svelte

@@ -11,6 +11,9 @@
   export let cellValue;
   export let options;
   export let canSelectMultipleOptions;
+  export let driver;
+
+  export let dataEditorTypesBehaviourOverride = null;
 
   let value;
   let valueInit;
@@ -18,22 +21,27 @@
   let isOptionsHidden = false;
 
   onMount(() => {
-    value = inplaceEditorState.text || stringifyCellValue(cellValue);
+    value =
+      inplaceEditorState.text ||
+      stringifyCellValue(
+        cellValue,
+        'inlineEditorIntent',
+        dataEditorTypesBehaviourOverride ?? driver?.dataEditorTypesBehaviour
+      ).value;
     valueInit = value;
 
     const optionsSelected = value.split(',');
-    optionsData = options
+    optionsData = options.map(function (option) {
-      .map(function(option) {
       return {
         value: option,
-          isSelected: optionsSelected.includes(option)
+        isSelected: optionsSelected.includes(option),
       };
     });
   });
 
   function handleCheckboxChanged(e, option) {
     if (!canSelectMultipleOptions) {
-      optionsData.forEach(option => option.isSelected = false);
+      optionsData.forEach(option => (option.isSelected = false));
       option.isSelected = true;
     } else {
       option.isSelected = e.target.checked;
@@ -44,8 +52,7 @@
       .map(option => option.value)
       .join(',');
 
-    if(!canSelectMultipleOptions)
+    if (!canSelectMultipleOptions) handleConfirm();
-      handleConfirm();
   }
 
   function handleConfirm() {
@@ -69,13 +76,8 @@
   }
 </script>
 
-<div
+<div use:clickOutside on:clickOutside={handleClickOutside} on:keydown={handleKeyDown} class="inplaceselect">
-  use:clickOutside
+  <div on:click={() => (isOptionsHidden = !isOptionsHidden)} class="value">
-  on:clickOutside={handleClickOutside}
-  on:keydown={handleKeyDown}
-  class="inplaceselect"
->
-  <div on:click={() => isOptionsHidden = !isOptionsHidden} class="value">
     {value}
   </div>
 
@@ -88,11 +90,12 @@
   <div class="options" class:hidden={isOptionsHidden}>
     {#each optionsData ?? [] as option}
       <label>
-        <input type="checkbox"
+        <input
+          type="checkbox"
           on:change={e => handleCheckboxChanged(e, option)}
           bind:checked={option.isSelected}
           class:hidden={!canSelectMultipleOptions}
-        >
+        />
         {option.value}
       </label>
     {/each}
@@ -112,7 +115,7 @@
     background-color: var(--theme-bg-alt);
     max-height: 150px;
     overflow: auto;
-    box-shadow: 0 1px 10px 1px var(--theme-bg-inv-3);;
+    box-shadow: 0 1px 10px 1px var(--theme-bg-inv-3);
   }
 
   .value {

packages/web/src/datagrid/JslDataGrid.svelte

@@ -99,5 +99,22 @@
     preprocessLoadedRow={changeSetState?.value?.dataUpdateCommands
       ? row => processJsonDataUpdateCommands(row, changeSetState?.value?.dataUpdateCommands)
       : null}
+    dataEditorTypesBehaviourOverride={{
+      parseJsonNull: true,
+      parseJsonBoolean: true,
+      parseNumber: true,
+      parseJsonArray: true,
+      parseJsonObject: true,
+
+      explicitDataType: true,
+
+      supportNumberType: true,
+      supportStringType: true,
+      supportBooleanType: true,
+      supportNullType: true,
+      supportJsonType: true,
+
+      supportFieldRemoval: true,
+    }}
   />
 {/key}

packages/web/src/datagrid/JslDataGridCore.svelte

@@ -69,7 +69,7 @@
 
   export let macroPreview;
   export let macroValues;
-  export let onPublishedCellsChanged
+  export let onPublishedCellsChanged;
   export const activator = createActivator('JslDataGridCore', false);
 
   export let setLoadedRows;

packages/web/src/elements/SelectionMapView.svelte

@@ -37,7 +37,7 @@
   function createColumnsTable(cells) {
     if (cells.length == 0) return '';
     return `<table>${cells
-      .map(cell => `<tr><td>${cell.column}</td><td>${stringifyCellValue(cell.value)}</td></tr>`)
+      .map(cell => `<tr><td>${cell.column}</td><td>${stringifyCellValue(cell.value, 'exportIntent').value}</td></tr>`)
       .join('\n')}</table>`;
   }
 

packages/web/src/formview/FormView.svelte

@@ -48,10 +48,19 @@
     category: 'Data form',
     name: 'Set NULL',
     keyText: 'CtrlOrCommand+0',
-    testEnabled: () => getCurrentDataForm() != null,
+    testEnabled: () => getCurrentDataForm() != null && !getCurrentDataForm()?.getEditorTypes()?.supportFieldRemoval,
     onClick: () => getCurrentDataForm().setFixedValue(null),
   });
 
+  registerCommand({
+    id: 'dataForm.removeField',
+    category: 'Data form',
+    name: 'Remove field',
+    keyText: 'CtrlOrCommand+0',
+    testEnabled: () => getCurrentDataForm() != null && getCurrentDataForm()?.getEditorTypes()?.supportFieldRemoval,
+    onClick: () => getCurrentDataForm().setFixedValue(undefined),
+  });
+
   registerCommand({
     id: 'dataForm.undo',
     category: 'Data form',
@@ -157,7 +166,7 @@
 <script lang="ts">
   import { getFilterValueExpression } from 'dbgate-filterparser';
 
-  import { filterName } from 'dbgate-tools';
+  import { filterName, shouldOpenMultilineDialog, stringifyCellValue } from 'dbgate-tools';
 
   import _ from 'lodash';
 
@@ -175,7 +184,7 @@
   import { plusExpandIcon } from '../icons/expandIcons';
   import FontIcon from '../icons/FontIcon.svelte';
   import DictionaryLookupModal from '../modals/DictionaryLookupModal.svelte';
-  import EditCellDataModal, { shouldOpenMultilineDialog } from '../modals/EditCellDataModal.svelte';
+  import EditCellDataModal from '../modals/EditCellDataModal.svelte';
   import { showModal } from '../modals/modalTools';
   import { apiCall } from '../utility/api';
 
@@ -201,6 +210,7 @@
   export let rowCountNotAvailable;
   // export let formDisplay;
   export let onNavigate;
+  export let dataEditorTypesBehaviourOverride = null;
 
   let wrapperHeight = 1;
   let wrapperWidth = 1;
@@ -273,7 +283,10 @@
   export function copyToClipboard() {
     const column = getCellColumn(currentCell);
     if (!column) return;
-    const text = currentCell[1] % 2 == 1 ? extractRowCopiedValue(rowData, column.uniqueName) : column.columnName;
+    const text =
+      currentCell[1] % 2 == 1
+        ? extractRowCopiedValue(rowData, column.uniqueName, display?.driver?.dataEditorTypesBehaviour)
+        : column.columnName;
     copyTextToClipboard(text);
   }
 
@@ -318,6 +331,10 @@
 
   export const activator = createActivator('FormView', false);
 
+  export function getEditorTypes() {
+    return display?.driver?.dataEditorTypesBehaviour;
+  }
+
   const handleTableMouseDown = event => {
     if (event.target.closest('.buttonLike')) return;
     if (event.target.closest('.resizeHandleControl')) return;
@@ -408,10 +425,11 @@
     { divider: true },
     { placeTag: 'save' },
     { command: 'dataForm.revertRowChanges' },
-    { command: 'dataForm.setNull' },
+    { command: 'dataForm.setNull', hideDisabled: true },
+    { command: 'dataForm.removeField', hideDisabled: true },
     { divider: true },
-    { command: 'dataForm.undo' },
+    { command: 'dataForm.undo', hideDisabled: true },
-    { command: 'dataForm.redo' },
+    { command: 'dataForm.redo', hideDisabled: true },
     { divider: true },
     { command: 'dataForm.goToFirst' },
     { command: 'dataForm.goToPrevious' },
@@ -488,6 +506,7 @@
     if (shouldOpenMultilineDialog(cellData)) {
       showModal(EditCellDataModal, {
         value: cellData,
+        dataEditorTypesBehaviour: display?.driver?.dataEditorTypesBehaviour,
         onSave: value => grider.setCellValue(0, column.uniqueName, value),
       });
       return true;
@@ -631,11 +650,13 @@
             {#if rowData && $inplaceEditorState.cell && rowIndex == $inplaceEditorState.cell[0] && chunkIndex * 2 + 1 == $inplaceEditorState.cell[1]}
               <InplaceEditor
                 width={getCellWidth(rowIndex, chunkIndex * 2 + 1)}
+                driver={display?.driver}
                 inplaceEditorState={$inplaceEditorState}
                 {dispatchInsplaceEditor}
+                {dataEditorTypesBehaviourOverride}
                 cellValue={rowData[col.uniqueName]}
-                options="{col.options}"
+                options={col.options}
-                canSelectMultipleOptions="{col.canSelectMultipleOptions}"
+                canSelectMultipleOptions={col.canSelectMultipleOptions}
                 onSetValue={value => {
                   grider.setCellValue(0, col.uniqueName, value);
                 }}
@@ -644,6 +665,7 @@
               <DataGridCell
                 maxWidth={(wrapperWidth * 2) / 3}
                 minWidth={200}
+                editorTypes={display?.driver?.dataEditorTypesBehaviour}
                 {rowIndex}
                 {col}
                 {rowData}
@@ -659,6 +681,9 @@
                     chunkIndex * 2 + 1 == $inplaceEditorState.cell[1])}
                 isCurrentCell={currentCell[0] == rowIndex && currentCell[1] == chunkIndex * 2 + 1}
                 onDictionaryLookup={() => handleLookup(col)}
+                onSetValue={value => {
+                  grider.setCellValue(0, col.uniqueName, value);
+                }}
               />
             {/if}
           </tr>

packages/web/src/formview/ShowFormButton.svelte

@@ -2,6 +2,7 @@
   import FontIcon from '../icons/FontIcon.svelte';
 
   export let icon = 'icon form';
+  export let secondary = false;
 </script>
 
 <div
@@ -9,6 +10,7 @@
   on:mousedown|stopPropagation|preventDefault
   on:mouseup|stopPropagation|preventDefault
   class="showFormButtonMarker"
+  class:secondary
 >
   <FontIcon {icon} />
 </div>
@@ -23,6 +25,10 @@
     border: 1px solid var(--theme-bg-1);
   }
 
+  .secondary {
+    margin-right: 20px;
+  }
+
   div:hover {
     color: var(--theme-font-hover);
     border: var(--theme-border);

packages/web/src/formview/ShowFormDropDownButton.svelte

@@ -0,0 +1,44 @@
+<script lang="ts">
+  import FontIcon from '../icons/FontIcon.svelte';
+  import { currentDropDownMenu } from '../stores';
+
+  export let icon = 'icon form';
+  export let menu;
+
+  let domButton;
+
+  function handleClick() {
+    const rect = domButton.getBoundingClientRect();
+    const left = rect.left;
+    const top = rect.bottom;
+    currentDropDownMenu.set({ left, top, items: menu });
+  }
+</script>
+
+<div
+  on:click|stopPropagation|preventDefault={handleClick}
+  bind:this={domButton}
+  on:mousedown|stopPropagation|preventDefault
+  on:mouseup|stopPropagation|preventDefault
+  class="showFormButtonMarker"
+>
+  <FontIcon {icon} />
+</div>
+
+<style>
+  div {
+    position: absolute;
+    right: 0px;
+    top: 1px;
+    color: var(--theme-font-3);
+    background-color: var(--theme-bg-1);
+    border: 1px solid var(--theme-bg-1);
+  }
+
+  div:hover {
+    color: var(--theme-font-hover);
+    border: var(--theme-border);
+    top: 1px;
+    right: 0px;
+  }
+</style>

packages/web/src/icons/FontIcon.svelte

@@ -61,6 +61,7 @@
     'icon app': 'mdi mdi-layers-triple',
     'icon open-in-new': 'mdi mdi-open-in-new',
     'icon add-folder': 'mdi mdi-folder-plus-outline',
+    'icon add-column': 'mdi mdi-table-column-plus-after',
 
     'icon window-restore': 'mdi mdi-window-restore',
     'icon window-maximize': 'mdi mdi-window-maximize',
@@ -186,6 +187,16 @@
     'icon num-9-outline': 'mdi mdi-numeric-9-circle-outline',
     'icon num-9-plus-outline': 'mdi mdi-numeric-9-plus-circle-outline',
 
+    'icon type-string': 'mdi mdi-code-string',
+    'icon type-object': 'mdi mdi-code-braces-box',
+    'icon type-array': 'mdi mdi-code-array',
+    'icon type-number': 'mdi mdi-pound-box',
+    'icon type-boolean': 'mdi mdi-code-equal',
+    'icon type-date': 'mdi mdi-alpha-d-box',
+    'icon type-objectid': 'mdi mdi-alpha-i-box',
+    'icon type-null': 'mdi mdi-code-equal',
+    'icon type-unknown': 'mdi mdi-help-box',
+
     'img ok': 'mdi mdi-check-circle color-icon-green',
     'img ok-inv': 'mdi mdi-check-circle color-icon-inv-green',
     'img alert': 'mdi mdi-alert-circle color-icon-blue',

packages/web/src/modals/EditCellDataModal.svelte

@@ -1,18 +1,3 @@
-<script lang="ts" context="module">
-  export function shouldOpenMultilineDialog(value) {
-    if (_.isString(value)) {
-      if (value.includes('\n')) {
-        return true;
-      }
-      const parsed = safeJsonParse(value);
-      if (parsed && (_.isPlainObject(parsed) || _.isArray(parsed))) {
-        return true;
-      }
-    }
-    return false;
-  }
-</script>
-
 <script lang="ts">
   import { onMount } from 'svelte';
 
@@ -25,17 +10,20 @@
   import ModalBase from './ModalBase.svelte';
   import { closeCurrentModal, showModal } from './modalTools';
   import SelectField from '../forms/SelectField.svelte';
-  import { safeJsonParse } from 'dbgate-tools';
+  import { parseCellValue, safeJsonParse, stringifyCellValue } from 'dbgate-tools';
   import { showSnackbarError } from '../utility/snackbar';
   import ErrorMessageModal from './ErrorMessageModal.svelte';
+  import da from 'date-fns/locale/da';
 
   export let onSave;
   export let value;
 
+  export let dataEditorTypesBehaviour;
+
   let editor;
   let syntaxMode = 'text';
 
-  let textValue = value?.toString() || '';
+  let textValue = stringifyCellValue(value, 'multilineEditorIntent', dataEditorTypesBehaviour).value;
 
   onMount(() => {
     editor.getEditor().focus();
@@ -52,7 +40,7 @@
 
   function handleKeyDown(ev) {
     if (ev.keyCode == keycodes.enter && ev.ctrlKey) {
-      onSave(textValue);
+      onSave(parseCellValue(textValue, dataEditorTypesBehaviour));
       closeCurrentModal();
     }
   }
@@ -81,7 +69,7 @@
           value="OK"
           title="Ctrl+Enter"
           on:click={() => {
-            onSave(textValue);
+            onSave(parseCellValue(textValue, dataEditorTypesBehaviour));
             closeCurrentModal();
           }}
         />

packages/web/src/perspectives/PerspectiveCell.svelte

@@ -15,7 +15,7 @@
     if (force && value?.type == 'Buffer' && _.isArray(value.data)) {
       return String.fromCharCode.apply(String, value.data);
     }
-    return stringifyCellValue(value);
+    return stringifyCellValue(value, 'gridCellIntent').value;
   }
 </script>
 

packages/web/src/tabs/ArchiveFileTab.svelte

@@ -30,6 +30,7 @@
   import { changeSetContainsChanges, createChangeSet } from 'dbgate-datalib';
   import localforage from 'localforage';
   import { onMount, tick } from 'svelte';
+  import _ from 'lodash';
 
   import ToolStripCommandButton from '../buttons/ToolStripCommandButton.svelte';
   import ToolStripCommandSplitButton from '../buttons/ToolStripCommandSplitButton.svelte';
@@ -129,7 +130,13 @@
       await apiCall('archive/modify-file', {
         folder: archiveFolder,
         file: archiveFile,
-        changeSet: $changeSetStore.value,
+        changeSet: {
+          ...$changeSetStore.value,
+          updates: $changeSetStore.value.updates.map(update => ({
+            ...update,
+            fields: _.mapValues(update.fields, (v, k) => (v === undefined ? { $$undefined$$: true } : v)),
+          })),
+        },
       });
       await afterSaveChangeSet();
     }
@@ -172,6 +179,10 @@
   <svelte:fragment slot="toolstrip">
     <ToolStripCommandButton command="dataGrid.refresh" />
     <ToolStripExportButton command="jslTableGrid.export" {quickExportHandlerRef} />
+    <ToolStripCommandButton command="dataGrid.revertAllChanges" hideDisabled />
+    <ToolStripCommandButton command="dataGrid.insertNewRow" hideDisabled />
+    <ToolStripCommandButton command="dataGrid.deleteSelectedRows" hideDisabled />
+    <ToolStripCommandButton command="dataGrid.addNewColumn" hideDisabled />
     <ToolStripCommandButton command="archiveFile.save" />
     <ToolStripCommandButton command="archiveFile.saveAs" />
   </svelte:fragment>

packages/web/src/tabs/CollectionDataTab.svelte

@@ -121,7 +121,13 @@
     const resp = await apiCall('database-connections/update-collection', {
       conid,
       database,
-      changeSet,
+      changeSet: {
+        ...changeSet,
+        updates: changeSet.updates.map(update => ({
+          ...update,
+          fields: _.mapValues(update.fields, (v, k) => (v === undefined ? { $$undefined$$: true } : v)),
+        })),
+      },
     });
     const { errorMessage } = resp || {};
     if (errorMessage) {
@@ -206,6 +212,7 @@
     <ToolStripCommandButton command="dataGrid.revertAllChanges" hideDisabled />
     <ToolStripCommandButton command="dataGrid.insertNewRow" hideDisabled />
     <ToolStripCommandButton command="dataGrid.deleteSelectedRows" hideDisabled />
+    <ToolStripCommandButton command="dataGrid.addNewColumn" hideDisabled />
     <ToolStripCommandButton command="dataGrid.switchToJson" hideDisabled />
     <ToolStripCommandButton command="dataGrid.switchToTable" hideDisabled />
     <ToolStripExportButton {quickExportHandlerRef} command="collectionDataGrid.export" />

packages/web/src/utility/clipboard.ts

@@ -1,6 +1,7 @@
 import _ from 'lodash';
 import { arrayToHexString, stringifyCellValue } from 'dbgate-tools';
 import yaml from 'js-yaml';
+import { DataEditorTypesBehaviour } from 'dbgate-types';
 
 export function copyTextToClipboard(text) {
   const oldFocus = document.activeElement;
@@ -71,13 +72,13 @@ export async function getClipboardText() {
   return await navigator.clipboard.readText();
 }
 
-export function extractRowCopiedValue(row, col) {
+export function extractRowCopiedValue(row, col, editorTypes?: DataEditorTypesBehaviour) {
   let value = row[col];
   if (value === undefined) value = _.get(row, col);
-  return stringifyCellValue(value);
+  return stringifyCellValue(value, 'exportIntent', editorTypes).value;
 }
 
-const clipboardHeadersFormatter = (delimiter) => (columns) => {
+const clipboardHeadersFormatter = delimiter => columns => {
   return columns.join(delimiter);
 };
 

plugins/dbgate-plugin-mongo/package.json

@@ -31,14 +31,15 @@
     "prepublishOnly": "yarn build"
   },
   "devDependencies": {
+    "bson": "^6.8.0",
     "dbgate-plugin-tools": "^1.0.7",
     "dbgate-query-splitter": "^4.10.1",
-    "lodash": "^4.17.21",
-    "webpack": "^5.91.0",
-    "webpack-cli": "^5.1.4",
     "dbgate-tools": "^5.0.0-alpha.1",
     "is-promise": "^4.0.0",
+    "lodash": "^4.17.21",
     "mongodb": "^6.3.0",
-    "mongodb-client-encryption": "^6.0.0"
+    "mongodb-client-encryption": "^6.0.0",
+    "webpack": "^5.91.0",
+    "webpack-cli": "^5.1.4"
   }
 }

plugins/dbgate-plugin-mongo/src/backend/createBulkInsertStream.js

@@ -1,5 +1,6 @@
 const ObjectId = require('mongodb').ObjectId;
 const { getLogger } = global.DBGATE_PACKAGES['dbgate-tools'];
+const { EJSON } = require('bson');
 
 const logger = getLogger('mongoBulkInsert');
 
@@ -26,7 +27,7 @@ function createBulkInsertStream(driver, stream, pool, name, options) {
         ...row,
       };
     }
-    writable.buffer.push(row);
+    writable.buffer.push(EJSON.deserialize(row));
   };
 
   writable.checkStructure = async () => {

plugins/dbgate-plugin-mongo/src/backend/driver.js

@@ -3,9 +3,8 @@ const stream = require('stream');
 const isPromise = require('is-promise');
 const driverBase = require('../frontend/driver');
 const Analyser = require('./Analyser');
-const MongoClient = require('mongodb').MongoClient;
+const { MongoClient, ObjectId, AbstractCursor } = require('mongodb');
-const ObjectId = require('mongodb').ObjectId;
+const { EJSON } = require('bson');
-const AbstractCursor = require('mongodb').AbstractCursor;
 const createBulkInsertStream = require('./createBulkInsertStream');
 const {
   convertToMongoCondition,
@@ -14,9 +13,7 @@ const {
 } = require('../frontend/convertToMongoCondition');
 
 function transformMongoData(row) {
-  return _.cloneDeepWith(row, (x) => {
+  return EJSON.serialize(row);
-    if (x && x.constructor == ObjectId) return { $oid: x.toString() };
-  });
 }
 
 async function readCursor(cursor, options) {
@@ -27,11 +24,7 @@ async function readCursor(cursor, options) {
 }
 
 function convertObjectId(condition) {
-  return _.cloneDeepWith(condition, (x) => {
+  return EJSON.deserialize(condition);
-    if (x && x.$oid) {
-      return ObjectId.createFromHexString(x.$oid);
-    }
-  });
 }
 
 function findArrayResult(resValue) {
@@ -252,8 +245,23 @@ const driver = {
     const db = await getScriptableDb(pool);
     exprValue = func(db, ObjectId.createFromHexString);
 
+    const pass = new stream.PassThrough({
+      objectMode: true,
+      highWaterMark: 100,
+    });
+
+    exprValue
+      .forEach((row) => pass.write(transformMongoData(row)))
+      .then(() => {
+        pass.end();
+        // pass.end(() => {
+        //   pass.emit('end');
+        // })
+      });
+
+    return pass;
     // return directly stream without header row
-    return exprValue.stream();
+    // return exprValue.stream();
 
     // pass.write(structure || { __isDynamicStructure: true });
     // exprValue.on('data', (row) => pass.write(row));
@@ -337,7 +345,14 @@ const driver = {
           }
         } else {
           const resdoc = await collection.updateOne(convertObjectId(update.condition), {
-            $set: convertObjectId(update.fields),
+            $set: convertObjectId(_.pickBy(update.fields, (v, k) => !v?.$$undefined$$)),
+            $unset: _.fromPairs(
+              Object.keys(update.fields)
+                .filter((k) => update.fields[k]?.$$undefined$$)
+                .map((k) => [k, ''])
+            ),
+
+            // $set: convertObjectId(update.fields),
           });
           res.updated.push(resdoc._id);
         }

plugins/dbgate-plugin-mongo/src/frontend/driver.js

@@ -2,6 +2,8 @@ const { driverBase } = global.DBGATE_PACKAGES['dbgate-tools'];
 const { convertToMongoCondition, convertToMongoSort } = require('./convertToMongoCondition');
 const Dumper = require('./Dumper');
 const { mongoSplitterOptions } = require('dbgate-query-splitter/lib/options');
+const _pickBy = require('lodash/pickBy');
+const _fromPairs = require('lodash/fromPairs');
 
 function jsonStringifyWithObjectId(obj) {
   return JSON.stringify(obj, undefined, 2).replace(
@@ -96,7 +98,12 @@ const driver = {
         res += `db.${update.pureName}.updateOne(${jsonStringifyWithObjectId(
           update.condition
         )}, ${jsonStringifyWithObjectId({
-          $set: update.fields,
+          $set: _pickBy(update.fields, (v, k) => v !== undefined),
+          $unset: _fromPairs(
+            Object.keys(update.fields)
+              .filter((k) => update.fields[k] === undefined)
+              .map((k) => [k, ''])
+          ),
         })});\n`;
       }
     }
@@ -122,6 +129,27 @@ const driver = {
       sort: convertToMongoSort(sort) || {},
     };
   },
+
+  dataEditorTypesBehaviour: {
+    parseJsonNull: true,
+    parseJsonBoolean: true,
+    parseNumber: true,
+    parseJsonArray: true,
+    parseJsonObject: true,
+    parseObjectIdAsDollar: true,
+    parseDateAsDollar: true,
+
+    explicitDataType: true,
+    supportNumberType: true,
+    supportStringType: true,
+    supportBooleanType: true,
+    supportDateType: true,
+    supportJsonType: true,
+    supportObjectIdType: true,
+    supportNullType: true,
+
+    supportFieldRemoval: true,
+  },
 };
 
 module.exports = driver;

yarn.lock

@@ -2458,6 +2458,11 @@ bson@^6.7.0:
   resolved "https://registry.yarnpkg.com/bson/-/bson-6.7.0.tgz#51973b132cdc424c8372fda3cb43e3e3e2ae2227"
   integrity sha512-w2IquM5mYzYZv6rs3uN2DZTOBe2a0zXLj53TGDqwF4l6Sz/XsISrisXOJihArF9+BZ6Cq/GjVht7Sjfmri7ytQ==
 
+bson@^6.8.0:
+  version "6.8.0"
+  resolved "https://registry.yarnpkg.com/bson/-/bson-6.8.0.tgz#5063c41ba2437c2b8ff851b50d9e36cb7aaa7525"
+  integrity sha512-iOJg8pr7wq2tg/zSlCCHMi3hMm5JTOxLTagf3zxhcenHsFp+c6uOs6K7W5UE7A4QIJGtqh/ZovFNMP4mOPJynQ==
+
 buffer-crc32@^0.2.5:
   version "0.2.13"
   resolved "https://registry.yarnpkg.com/buffer-crc32/-/buffer-crc32-0.2.13.tgz#0d333e3f00eac50aa1454abd30ef8c2a5d9a7242"