Merge branch 'grid-refactor'

packages/api/src/proc/sessionProcess.js

@@ -19,7 +19,7 @@ class TableWriter {
     this.jslid = uuidv1();
     this.currentFile = path.join(jsldir(), `${this.jslid}.jsonl`);
     this.currentRowCount = 0;
-    this.currentChangeIndex = 0;
+    this.currentChangeIndex = 1;
     fs.writeFileSync(this.currentFile, JSON.stringify({ columns }) + '\n');
     this.currentStream = fs.createWriteStream(this.currentFile, { flags: 'a' });
     this.writeCurrentStats(false, false);

packages/web/src/datagrid/ChangeSetGrider.ts

@@ -0,0 +1,157 @@
+import {
+  ChangeSet,
+  changeSetContainsChanges,
+  changeSetInsertNewRow,
+  createChangeSet,
+  deleteChangeSetRows,
+  findExistingChangeSetItem,
+  getChangeSetInsertedRows,
+  GridDisplay,
+  revertChangeSetRowChanges,
+  setChangeSetValue,
+} from '@dbgate/datalib';
+import Grider, { GriderRowStatus } from './Grider';
+
+export default class ChangeSetGrider extends Grider {
+  public insertedRows: any[];
+  public changeSet: ChangeSet;
+  public setChangeSet: Function;
+  private rowCacheIndexes: Set<number>;
+  private rowDataCache;
+  private rowStatusCache;
+  private rowDefinitionsCache;
+  private batchChangeSet: ChangeSet;
+
+  constructor(public sourceRows: any[], public changeSetState, public dispatchChangeSet, public display: GridDisplay) {
+    super();
+    this.changeSet = changeSetState && changeSetState.value;
+    this.insertedRows = getChangeSetInsertedRows(this.changeSet, display.baseTable);
+    this.setChangeSet = (value) => dispatchChangeSet({ type: 'set', value });
+    this.rowCacheIndexes = new Set();
+    this.rowDataCache = {};
+    this.rowStatusCache = {};
+    this.rowDefinitionsCache = {};
+    this.batchChangeSet = null;
+  }
+
+  getRowSource(index: number) {
+    if (index < this.sourceRows.length) return this.sourceRows[index];
+    return null;
+  }
+
+  getInsertedRowIndex(index) {
+    return index >= this.sourceRows.length ? index - this.sourceRows.length : null;
+  }
+
+  requireRowCache(index: number) {
+    if (this.rowCacheIndexes.has(index)) return;
+    const row = this.getRowSource(index);
+    const insertedRowIndex = this.getInsertedRowIndex(index);
+    const rowDefinition = this.display.getChangeSetRow(row, insertedRowIndex);
+    const [matchedField, matchedChangeSetItem] = findExistingChangeSetItem(this.changeSet, rowDefinition);
+    const rowUpdated = matchedChangeSetItem ? { ...row, ...matchedChangeSetItem.fields } : row;
+    let status = 'regular';
+    if (matchedChangeSetItem && matchedField == 'updates') status = 'updated';
+    if (matchedField == 'deletes') status = 'deleted';
+    if (insertedRowIndex != null) status = 'inserted';
+    const rowStatus = {
+      status,
+      modifiedFields: new Set(
+        matchedChangeSetItem && matchedChangeSetItem.fields ? Object.keys(matchedChangeSetItem.fields) : []
+      ),
+    };
+    this.rowDataCache[index] = rowUpdated;
+    this.rowStatusCache[index] = rowStatus;
+    this.rowDefinitionsCache[index] = rowDefinition;
+    this.rowCacheIndexes.add(index);
+  }
+
+  getRowData(index: number) {
+    this.requireRowCache(index);
+    return this.rowDataCache[index];
+  }
+
+  getRowStatus(index): GriderRowStatus {
+    this.requireRowCache(index);
+    return this.rowStatusCache[index];
+  }
+
+  get rowCount() {
+    return this.sourceRows.length + this.insertedRows.length;
+  }
+
+  applyModification(changeSetReducer) {
+    if (this.batchChangeSet) {
+      this.batchChangeSet = changeSetReducer(this.batchChangeSet);
+    } else {
+      this.setChangeSet(changeSetReducer(this.changeSet));
+    }
+  }
+
+  setCellValue(index: number, uniqueName: string, value: any) {
+    const row = this.getRowSource(index);
+    const definition = this.display.getChangeSetField(row, uniqueName, this.getInsertedRowIndex(index));
+    this.applyModification((chs) => setChangeSetValue(chs, definition, value));
+  }
+
+  deleteRow(index: number) {
+    this.requireRowCache(index);
+    this.applyModification((chs) => deleteChangeSetRows(chs, this.rowDefinitionsCache[index]));
+  }
+
+  get rowCountInUpdate() {
+    if (this.batchChangeSet) {
+      const newRows = getChangeSetInsertedRows(this.batchChangeSet, this.display.baseTable);
+      return this.sourceRows.length + newRows.length;
+    } else {
+      return this.rowCount;
+    }
+  }
+
+  insertRow(): number {
+    const res = this.rowCountInUpdate;
+    this.applyModification((chs) => changeSetInsertNewRow(chs, this.display.baseTable));
+    return res;
+  }
+
+  beginUpdate() {
+    this.batchChangeSet = this.changeSet;
+  }
+  endUpdate() {
+    this.setChangeSet(this.batchChangeSet);
+    this.batchChangeSet = null;
+  }
+
+  revertRowChanges(index: number) {
+    this.requireRowCache(index);
+    this.applyModification((chs) => revertChangeSetRowChanges(chs, this.rowDefinitionsCache[index]));
+  }
+  revertAllChanges() {
+    this.applyModification((chs) => createChangeSet());
+  }
+  undo() {
+    this.dispatchChangeSet({ type: 'undo' });
+  }
+  redo() {
+    this.dispatchChangeSet({ type: 'redo' });
+  }
+  get canUndo() {
+    return this.changeSetState.canUndo;
+  }
+  get canRedo() {
+    return this.changeSetState.canRedo;
+  }
+  get containsChanges() {
+    return changeSetContainsChanges(this.changeSet);
+  }
+  get disableLoadNextPage() {
+    return this.insertedRows.length > 0;
+  }
+
+  static factory({ sourceRows, changeSetState, dispatchChangeSet, display }): ChangeSetGrider {
+    return new ChangeSetGrider(sourceRows, changeSetState, dispatchChangeSet, display);
+  }
+  static factoryDeps({ sourceRows, changeSetState, dispatchChangeSet, display }) {
+    return [sourceRows, changeSetState ? changeSetState.value : null, dispatchChangeSet, display];
+  }
+}

packages/web/src/datagrid/DataGrid.js

@@ -41,8 +41,8 @@ const DataGridContainer = styled.div`
   flex-grow: 1;
 `;
 
-/** @param props {import('./types').DataGridProps} */
 export default function DataGrid(props) {
+  const { GridCore } = props;
   const Container1 = props.showReferences ? ManagerOuterContainer1 : ManagerOuterContainerFull;
   const [managerSize, setManagerSize] = React.useState(0);
   return (
@@ -50,18 +50,18 @@ export default function DataGrid(props) {
       <LeftContainer>
         <ManagerMainContainer>
           <Container1>
-            <ColumnManager {...props} managerSize={managerSize}/>
+            <ColumnManager {...props} managerSize={managerSize} />
           </Container1>
           {props.showReferences && (
             <ManagerOuterContainer2>
-              <ReferenceManager {...props} managerSize={managerSize}/>
+              <ReferenceManager {...props} managerSize={managerSize} />
             </ManagerOuterContainer2>
           )}
         </ManagerMainContainer>
       </LeftContainer>
 
       <DataGridContainer>
-        <DataGridCore {...props} />
+        <GridCore {...props} />
       </DataGridContainer>
     </HorizontalSplitter>
 

packages/web/src/datagrid/DataGridCore.js

@@ -4,7 +4,6 @@ import ReactDOM from 'react-dom';
 import styled from 'styled-components';
 import { HorizontalScrollBar, VerticalScrollBar } from './ScrollBars';
 import useDimensions from '../utility/useDimensions';
-import axios from '../utility/axios';
 import DataFilterControl from './DataFilterControl';
 import stableStringify from 'json-stable-stringify';
 import { getFilterType, getFilterValueExpression } from '@dbgate/filterparser';
@@ -18,19 +17,6 @@ import {
   filterCellsForRow,
   cellIsSelected,
 } from './gridutil';
-import useModalState from '../modals/useModalState';
-import ConfirmSqlModal from '../modals/ConfirmSqlModal';
-import {
-  changeSetToSql,
-  createChangeSet,
-  revertChangeSetRowChanges,
-  getChangeSetInsertedRows,
-  changeSetInsertNewRow,
-  deleteChangeSetRows,
-  batchUpdateChangeSet,
-  setChangeSetValue,
-} from '@dbgate/datalib';
-import { scriptToSql } from '@dbgate/sqltree';
 import { copyTextToClipboard } from '../utility/clipboard';
 import DataGridToolbar from './DataGridToolbar';
 // import usePropsCompare from '../utility/usePropsCompare';
@@ -38,14 +24,8 @@ import ColumnHeaderControl from './ColumnHeaderControl';
 import InlineButton from '../widgets/InlineButton';
 import { showMenu } from '../modals/DropDownMenu';
 import DataGridContextMenu from './DataGridContextMenu';
-import useSocket from '../utility/SocketProvider';
 import LoadingInfo from '../widgets/LoadingInfo';
 import ErrorInfo from '../widgets/ErrorInfo';
-import useShowModal from '../modals/showModal';
-import ErrorMessageModal from '../modals/ErrorMessageModal';
-import ImportExportModal from '../modals/ImportExportModal';
-import { openNewTab } from '../utility/common';
-import { useSetOpenedTabs } from '../utility/globalState';
 
 const GridContainer = styled.div`
   position: absolute;
@@ -126,98 +106,31 @@ const LoadingInfoBox = styled.div`
   border: 1px solid gray;
 `;
 
-/** @param props {import('./types').DataGridProps} */
-async function loadDataPage(props, offset, limit) {
-  const { display, conid, database, jslid } = props;
-
-  if (jslid) {
-    const response = await axios.request({
-      url: 'jsldata/get-rows',
-      method: 'get',
-      params: {
-        jslid,
-        offset,
-        limit,
-      },
-    });
-    return response.data;
-  }
-
-  const sql = display.getPageQuery(offset, limit);
-
-  const response = await axios.request({
-    url: 'database-connections/query-data',
-    method: 'post',
-    params: {
-      conid,
-      database,
-    },
-    data: { sql },
-  });
-
-  if (response.data.errorMessage) return response.data;
-  return response.data.rows;
-}
-
-function dataPageAvailable(props) {
-  const { display, jslid } = props;
-  if (jslid) return true;
-  const sql = display.getPageQuery(0, 1);
-  return !!sql;
-}
-
-/** @param props {import('./types').DataGridProps} */
-async function loadRowCount(props) {
-  const { display, conid, database, jslid } = props;
-
-  if (jslid) {
-    const response = await axios.request({
-      url: 'jsldata/get-stats',
-      method: 'get',
-      params: {
-        jslid,
-      },
-    });
-    return response.data.rowCount;
-  }
-
-  const sql = display.getCountQuery();
-
-  const response = await axios.request({
-    url: 'database-connections/query-data',
-    method: 'post',
-    params: {
-      conid,
-      database,
-    },
-    data: { sql },
-  });
-
-  return parseInt(response.data.rows[0].count);
-}
-
 /** @param props {import('./types').DataGridProps} */
 export default function DataGridCore(props) {
-  const { conid, database, display, changeSetState, dispatchChangeSet, tabVisible, jslid } = props;
+  const {
+    display,
+    conid,
+    database,
+    tabVisible,
+    loadNextData,
+    errorMessage,
+    isLoadedAll,
+    loadedTime,
+    exportGrid,
+    allRowCount,
+    openQuery,
+    onSave,
+    isLoading,
+    grider,
+  } = props;
   // console.log('RENDER GRID', display.baseTable.pureName);
   const columns = React.useMemo(() => display.allColumns, [display]);
 
   // usePropsCompare(props);
 
   // console.log(`GRID, conid=${conid}, database=${database}, sql=${sql}`);
-  const [loadProps, setLoadProps] = React.useState({
-    isLoading: false,
-    loadedRows: [],
-    isLoadedAll: false,
-    loadedTime: new Date().getTime(),
-    allRowCount: null,
-    errorMessage: null,
-    jslStatsCounter: 0,
-    jslChangeIndex: 0,
-  });
-  const { isLoading, loadedRows, isLoadedAll, loadedTime, allRowCount, errorMessage } = loadProps;
 
-  const loadedTimeRef = React.useRef(0);
   const focusFieldRef = React.useRef(null);
 
   const [vScrollValueToSet, setvScrollValueToSet] = React.useState();
@@ -234,19 +147,6 @@ export default function DataGridCore(props) {
   const [autofillSelectedCells, setAutofillSelectedCells] = React.useState(emptyCellArray);
   const [focusFilterInputs, setFocusFilterInputs] = React.useState({});
 
-  // const [inplaceEditorCell, setInplaceEditorCell] = React.useState(nullCell);
-  // const [inplaceEditorInitText, setInplaceEditorInitText] = React.useState('');
-  // const [inplaceEditorShouldSave, setInplaceEditorShouldSave] = React.useState(false);
-  // const [inplaceEditorChangedOnCreate, setInplaceEditorChangedOnCreate] = React.useState(false);
-
-  const changeSet = changeSetState && changeSetState.value;
-  const setChangeSet = React.useCallback((value) => dispatchChangeSet({ type: 'set', value }), [dispatchChangeSet]);
-  const setOpenedTabs = useSetOpenedTabs();
-
-  const changeSetRef = React.useRef(changeSet);
-
-  changeSetRef.current = changeSet;
-
   const autofillMarkerCell = React.useMemo(
     () =>
       selectedCells && selectedCells.length > 0 && _.uniq(selectedCells.map((x) => x[0])).length == 1
@@ -255,76 +155,12 @@ export default function DataGridCore(props) {
     [selectedCells]
   );
 
-  const showModal = useShowModal();
-
-  const handleLoadRowCount = async () => {
-    const rowCount = await loadRowCount(props);
-    setLoadProps((oldLoadProps) => ({
-      ...oldLoadProps,
-      allRowCount: rowCount,
-    }));
-  };
-
-  const loadNextData = async () => {
-    if (isLoading) return;
-    setLoadProps((oldLoadProps) => ({
-      ...oldLoadProps,
-      isLoading: true,
-    }));
-    const loadStart = new Date().getTime();
-    loadedTimeRef.current = loadStart;
-
-    const nextRows = await loadDataPage(props, loadedRows.length, 100);
-    if (loadedTimeRef.current !== loadStart) {
-      // new load was dispatched
-      return;
-    }
-    // if (!_.isArray(nextRows)) {
-    //   console.log('Error loading data from server', nextRows);
-    //   nextRows = [];
-    // }
-    // console.log('nextRows', nextRows);
-    if (nextRows.errorMessage) {
-      setLoadProps((oldLoadProps) => ({
-        ...oldLoadProps,
-        isLoading: false,
-        errorMessage: nextRows.errorMessage,
-      }));
-    } else {
-      if (allRowCount == null) handleLoadRowCount();
-      const loadedInfo = {
-        loadedRows: [...loadedRows, ...nextRows],
-        loadedTime,
-      };
-      setLoadProps((oldLoadProps) => ({
-        ...oldLoadProps,
-        isLoading: false,
-        isLoadedAll: oldLoadProps.jslStatsCounter == loadProps.jslStatsCounter && nextRows.length === 0,
-        ...loadedInfo,
-      }));
-    }
-  };
-
-  // const data = useFetch({
-  //   url: 'database-connections/query-data',
-  //   method: 'post',
-  //   params: {
-  //     conid,
-  //     database,
-  //   },
-  //   data: { sql },
-  // });
-  // const { rows, columns } = data || {};
   const [firstVisibleRowScrollIndex, setFirstVisibleRowScrollIndex] = React.useState(0);
   const [firstVisibleColumnScrollIndex, setFirstVisibleColumnScrollIndex] = React.useState(0);
-  const socket = useSocket();
 
   const [headerRowRef, { height: rowHeight }] = useDimensions();
   const [tableBodyRef] = useDimensions();
   const [containerRef, { height: containerHeight, width: containerWidth }] = useDimensions();
-  // const [tableRef, { height: tableHeight, width: tableWidth }] = useDimensions();
-  const confirmSqlModalState = useModalState();
-  const [confirmSql, setConfirmSql] = React.useState('');
 
   const [inplaceEditorState, dispatchInsplaceEditor] = React.useReducer((state, action) => {
     switch (action.type) {
@@ -355,8 +191,8 @@ export default function DataGridCore(props) {
 
   // usePropsCompare({ loadedRows, columns, containerWidth, display });
 
-  const columnSizes = React.useMemo(() => countColumnSizes(loadedRows, columns, containerWidth, display), [
+  const columnSizes = React.useMemo(() => countColumnSizes(grider, columns, containerWidth, display), [
-    loadedRows,
+    grider,
     columns,
     containerWidth,
     display,
@@ -376,43 +212,6 @@ export default function DataGridCore(props) {
   // console.log('visibleRowCountUpperBound', visibleRowCountUpperBound);
   // console.log('rowHeight', rowHeight);
 
-  const reload = () => {
-    setLoadProps({
-      allRowCount: null,
-      isLoading: false,
-      loadedRows: [],
-      isLoadedAll: false,
-      loadedTime: new Date().getTime(),
-      errorMessage: null,
-      jslStatsCounter: 0,
-      jslChangeIndex: 0,
-    });
-  };
-
-  const insertedRows = getChangeSetInsertedRows(changeSet, display.baseTable);
-
-  const rowCountNewIncluded = loadedRows.length + insertedRows.length;
-
-  React.useEffect(() => {
-    if (
-      !isLoadedAll &&
-      !errorMessage &&
-      firstVisibleRowScrollIndex + visibleRowCountUpperBound >= loadedRows.length &&
-      insertedRows.length == 0
-    ) {
-      if (dataPageAvailable(props)) {
-        // If not, callbacks to load missing metadata are dispatched
-        loadNextData();
-      }
-    }
-    if (props.masterLoadedTime && props.masterLoadedTime > loadedTime) {
-      display.reload();
-    }
-    if (display.cache.refreshTime > loadedTime) {
-      reload();
-    }
-  });
-
   React.useEffect(() => {
     if (tabVisible) {
       if (focusFieldRef.current) focusFieldRef.current.focus();
@@ -424,45 +223,11 @@ export default function DataGridCore(props) {
     return newColumn;
   }, [columnSizes, gridScrollAreaWidth]);
 
-  const handleJslDataStats = React.useCallback((stats) => {
-    if (stats.changeIndex < loadProps.jslChangeIndex) return;
-    setLoadProps((oldProps) => ({
-      ...oldProps,
-      allRowCount: stats.rowCount,
-      isLoadedAll: false,
-      jslStatsCounter: oldProps.jslStatsCounter + 1,
-      jslChangeIndex: stats.changeIndex,
-    }));
-  }, []);
-
   React.useEffect(() => {
-    if (jslid && socket) {
+    if (props.onReferenceSourceChanged && (grider.rowCount > 0 || isLoadedAll)) {
-      socket.on(`jsldata-stats-${jslid}`, handleJslDataStats);
-      return () => {
-        socket.off(`jsldata-stats-${jslid}`, handleJslDataStats);
-      };
-    }
-  }, [jslid]);
-
-  React.useEffect(() => {
-    if (props.onReferenceSourceChanged && ((loadedRows && loadedRows.length > 0) || isLoadedAll)) {
       props.onReferenceSourceChanged(getSelectedRowData(), loadedTime);
     }
-  }, [selectedCells, props.refReloadToken, loadedRows && loadedRows[0]]);
+  }, [selectedCells, props.refReloadToken, grider.getRowData(0)]);
-
-  // const handleCloseInplaceEditor = React.useCallback(
-  //   mode => {
-  //     const [row, col] = currentCell || [];
-  //     setInplaceEditorCell(null);
-  //     setInplaceEditorInitText(null);
-  //     setInplaceEditorShouldSave(false);
-  //     if (tableElement) tableElement.focus();
-  //     // @ts-ignore
-  //     if (mode == 'enter' && row) moveCurrentCell(row + 1, col);
-  //     if (mode == 'save') setTimeout(handleSave, 1);
-  //   },
-  //   [tableElement, currentCell]
-  // );
 
   // usePropsCompare({ columnSizes, firstVisibleColumnScrollIndex, gridScrollAreaWidth, columns });
 
@@ -487,6 +252,12 @@ export default function DataGridCore(props) {
     }
   }, [display && display.focusedColumn]);
 
+  React.useEffect(() => {
+    if (loadNextData && firstVisibleRowScrollIndex + visibleRowCountUpperBound >= grider.rowCount) {
+      loadNextData();
+    }
+  });
+
   React.useEffect(() => {
     if (display.groupColumns) {
       props.onReferenceClick({
@@ -504,7 +275,7 @@ export default function DataGridCore(props) {
   const rowCountInfo = React.useMemo(() => {
     if (selectedCells.length > 1 && selectedCells.every((x) => _.isNumber(x[0]) && _.isNumber(x[1]))) {
       let sum = _.sumBy(selectedCells, (cell) => {
-        const row = loadedRows[cell[0]];
+        const row = grider.getRowData(cell[0]);
         if (row) {
           const colName = realColumnUniqueNames[cell[1]];
           if (colName) {
@@ -523,12 +294,9 @@ export default function DataGridCore(props) {
     }
     if (allRowCount == null) return 'Loading row count...';
     return `Rows: ${allRowCount.toLocaleString()}`;
-    // if (this.isLoadingFirstPage) return "Loading first page...";
+  }, [selectedCells, allRowCount, grider, visibleRealColumns]);
-    // if (this.isFirstPageError) return "Error loading first page";
-    // return `Rows: ${this.rowCount.toLocaleString()}`;
-  }, [selectedCells, allRowCount, loadedRows, visibleRealColumns]);
 
-  if (!loadedRows || !columns || columns.length == 0)
+  if (!columns || columns.length == 0)
     return (
       <LoadingInfoWrapper>
         <LoadingInfoBox>
@@ -563,7 +331,7 @@ export default function DataGridCore(props) {
         setNull={setNull}
         exportGrid={exportGrid}
         filterSelectedValue={filterSelectedValue}
-        openQuery={display.baseTable ? openQuery : null}
+        openQuery={openQuery}
       />
     );
   };
@@ -617,39 +385,8 @@ export default function DataGridCore(props) {
     copyToClipboard();
   }
 
-  function exportGrid() {
+  function setCellValue(cell, value) {
-    const initialValues = {};
+    grider.setCellValue(cell[0], realColumnUniqueNames[cell[1]], value);
-    if (jslid) {
-      const archiveMatch = jslid.match(/^archive:\/\/([^/]+)\/(.*)$/);
-      if (archiveMatch) {
-        initialValues.sourceStorageType = 'archive';
-        initialValues.sourceArchiveFolder = archiveMatch[1];
-        initialValues.sourceList = [archiveMatch[2]];
-      } else {
-        initialValues.sourceStorageType = 'jsldata';
-        initialValues.sourceJslId = jslid;
-        initialValues.sourceList = ['query-data'];
-      }
-    } else {
-      initialValues.sourceStorageType = 'query';
-      initialValues.sourceConnectionId = conid;
-      initialValues.sourceDatabaseName = database;
-      initialValues.sourceSql = display.getExportQuery();
-      initialValues.sourceList = display.baseTable ? [display.baseTable.pureName] : [];
-    }
-    showModal((modalState) => <ImportExportModal modalState={modalState} initialValues={initialValues} />);
-  }
-
-  function setCellValue(chs, cell, value) {
-    return setChangeSetValue(
-      chs,
-      display.getChangeSetField(
-        loadedAndInsertedRows[cell[0]],
-        realColumnUniqueNames[cell[1]],
-        cell[0] >= loadedRows.length ? cell[0] - loadedRows.length : null
-      ),
-      value
-    );
   }
 
   function handlePaste(event) {
@@ -663,34 +400,22 @@ export default function DataGridCore(props) {
       pastedText = event.clipboardData.getData('text/plain');
     }
     event.preventDefault();
+    grider.beginUpdate();
     const pasteRows = pastedText
       .replace(/\r/g, '')
       .split('\n')
       .map((row) => row.split('\t'));
-    let chs = changeSet;
-    let allRows = loadedAndInsertedRows;
-
     if (selectedCells.length <= 1) {
-      const startRow = isRegularCell(currentCell) ? currentCell[0] : loadedAndInsertedRows.length;
+      const startRow = isRegularCell(currentCell) ? currentCell[0] : grider.rowCount;
       const startCol = isRegularCell(currentCell) ? currentCell[1] : 0;
       let rowIndex = startRow;
       for (const rowData of pasteRows) {
-        if (rowIndex >= allRows.length) {
+        if (rowIndex >= grider.rowCountInUpdate) {
-          chs = changeSetInsertNewRow(chs, display.baseTable);
+          grider.insertRow();
-          allRows = [...loadedRows, ...getChangeSetInsertedRows(chs, display.baseTable)];
         }
         let colIndex = startCol;
-        const row = allRows[rowIndex];
         for (const cell of rowData) {
-          chs = setChangeSetValue(
+          setCellValue([rowIndex, colIndex], cell == '(NULL)' ? null : cell);
-            chs,
-            display.getChangeSetField(
-              row,
-              realColumnUniqueNames[colIndex],
-              rowIndex >= loadedRows.length ? rowIndex - loadedRows.length : null
-            ),
-            cell == '(NULL)' ? null : cell
-          );
           colIndex += 1;
         }
         rowIndex += 1;
@@ -706,19 +431,18 @@ export default function DataGridCore(props) {
         const selectionCol = colIndex - startCol;
         const pasteRow = pasteRows[selectionRow % pasteRows.length];
         const pasteCell = pasteRow[selectionCol % pasteRow.length];
-        chs = setCellValue(chs, cell, pasteCell);
+        setCellValue(cell, pasteCell);
       }
     }
-
+    grider.endUpdate();
-    setChangeSet(chs);
   }
 
   function setNull() {
-    let chs = changeSet;
+    grider.beginUpdate();
     selectedCells.filter(isRegularCell).forEach((cell) => {
-      chs = setCellValue(chs, cell, null);
+      setCellValue(cell, null);
     });
-    setChangeSet(chs);
+    grider.endUpdate();
   }
 
   function cellsToRegularCells(cells) {
@@ -746,7 +470,7 @@ export default function DataGridCore(props) {
     const rowIndexes = _.sortBy(_.uniq(cells.map((x) => x[0])));
     const lines = rowIndexes.map((rowIndex) => {
       let colIndexes = _.sortBy(cells.filter((x) => x[0] == rowIndex).map((x) => x[1]));
-      const rowData = loadedAndInsertedRows[rowIndex];
+      const rowData = grider.getRowData(rowIndex);
       if (!rowData) return '';
       const line = colIndexes
         .map((col) => realColumnUniqueNames[col])
@@ -784,17 +508,11 @@ export default function DataGridCore(props) {
       const currentRowNumber = currentCell[0];
       if (_.isNumber(currentRowNumber)) {
         const rowIndexes = _.uniq((autofillSelectedCells || []).map((x) => x[0])).filter((x) => x != currentRowNumber);
-        // @ts-ignore
         const colNames = selectedCells.map((cell) => realColumnUniqueNames[cell[1]]);
-        const changeObject = _.pick(loadedAndInsertedRows[currentRowNumber], colNames);
+        const changeObject = _.pick(grider.getRowData(currentRowNumber), colNames);
-        setChangeSet(
+        grider.beginUpdate();
-          batchUpdateChangeSet(
+        for (const index of rowIndexes) grider.updateRow(index, changeObject);
-            changeSet,
+        grider.endUpdate();
-            getRowDefinitions(rowIndexes),
-            // @ts-ignore
-            rowIndexes.map(() => changeObject)
-          )
-        );
       }
 
       setAutofillDragStartCell(null);
@@ -803,36 +521,20 @@ export default function DataGridCore(props) {
     }
   }
 
-  function getRowDefinitions(rowIndexes) {
-    const res = [];
-    if (!loadedAndInsertedRows) return res;
-    for (const index of rowIndexes) {
-      if (loadedAndInsertedRows[index] && _.isNumber(index)) {
-        const insertedRowIndex = index >= loadedRows.length ? index - loadedRows.length : null;
-        res.push(display.getChangeSetRow(loadedAndInsertedRows[index], insertedRowIndex));
-      }
-    }
-    return res;
-  }
-
   function getSelectedRowIndexes() {
     return _.uniq((selectedCells || []).map((x) => x[0]));
   }
 
-  function getSelectedRowDefinitions() {
-    return getRowDefinitions(getSelectedRowIndexes());
-  }
-
   function getSelectedRowData() {
-    return _.compact(getSelectedRowIndexes().map((index) => loadedRows && loadedRows[index]));
+    return _.compact(getSelectedRowIndexes().map((index) => grider.getRowData(index)));
   }
 
   function revertRowChanges() {
-    const updatedChangeSet = getSelectedRowDefinitions().reduce(
+    grider.beginUpdate();
-      (chs, row) => revertChangeSetRowChanges(chs, row),
+    for (const index of getSelectedRowIndexes()) {
-      changeSet
+      if (_.isNumber(index)) grider.revertRowChanges(index);
-    );
+    }
-    setChangeSet(updatedChangeSet);
+    grider.endUpdate();
   }
 
   function filterSelectedValue() {
@@ -841,7 +543,7 @@ export default function DataGridCore(props) {
       if (!isRegularCell(cell)) continue;
       const modelIndex = columnSizes.realToModel(cell[1]);
       const columnName = columns[modelIndex].uniqueName;
-      let value = loadedRows[cell[0]][columnName];
+      let value = grider.getRowData(cell[0])[columnName];
       let svalue = getFilterValueExpression(value, columns[modelIndex].dataType);
       if (_.has(flts, columnName)) flts[columnName] += ',' + svalue;
       else flts[columnName] = svalue;
@@ -850,28 +552,12 @@ export default function DataGridCore(props) {
     display.setFilters(flts);
   }
 
-  function openQuery() {
-    openNewTab(setOpenedTabs, {
-      title: 'Query',
-      icon: 'sql.svg',
-      tabComponent: 'QueryTab',
-      props: {
-        initialScript: display.getExportQuery(),
-        schemaName: display.baseTable.schemaName,
-        pureName: display.baseTable.pureName,
-        conid,
-        database,
-      },
-    });
-  }
-
-  function revertAllChanges() {
-    setChangeSet(createChangeSet());
-  }
-
   function deleteSelectedRows() {
-    const updatedChangeSet = getSelectedRowDefinitions().reduce((chs, row) => deleteChangeSetRows(chs, row), changeSet);
+    grider.beginUpdate();
-    setChangeSet(updatedChangeSet);
+    for (const index of getSelectedRowIndexes()) {
+      if (_.isNumber(index)) grider.deleteRow(index);
+    }
+    grider.endUpdate();
   }
 
   function handleGridWheel(event) {
@@ -882,7 +568,7 @@ export default function DataGridCore(props) {
     if (event.deltaY < 0) {
       newFirstVisibleRowScrollIndex -= wheelRowCount;
     }
-    let rowCount = rowCountNewIncluded;
+    let rowCount = grider.rowCount;
     if (newFirstVisibleRowScrollIndex + visibleRowCountLowerBound > rowCount) {
       newFirstVisibleRowScrollIndex = rowCount - visibleRowCountLowerBound + 1;
     }
@@ -895,17 +581,11 @@ export default function DataGridCore(props) {
     setvScrollValueToSetDate(new Date());
   }
 
-  // async function blurEditorAndSave() {
-  //   setInplaceEditorCell(null);
-  //   setInplaceEditorInitText(null);
-  //   await sleep(1);
-  // }
-
   function undo() {
-    dispatchChangeSet({ type: 'undo' });
+    grider.undo();
   }
   function redo() {
-    dispatchChangeSet({ type: 'redo' });
+    grider.redo();
   }
 
   function handleSave() {
@@ -914,39 +594,13 @@ export default function DataGridCore(props) {
       dispatchInsplaceEditor({ type: 'shouldSave' });
       return;
     }
-    const script = changeSetToSql(changeSetRef.current, display.dbinfo);
+    if (onSave) onSave();
-    const sql = scriptToSql(display.driver, script);
-    setConfirmSql(sql);
-    confirmSqlModalState.open();
-  }
-
-  async function handleConfirmSql() {
-    const resp = await axios.request({
-      url: 'database-connections/query-data',
-      method: 'post',
-      params: {
-        conid,
-        database,
-      },
-      data: { sql: confirmSql },
-    });
-
-    const { errorMessage } = resp.data || {};
-    if (errorMessage) {
-      showModal((modalState) => (
-        <ErrorMessageModal modalState={modalState} message={errorMessage} title="Error when saving" />
-      ));
-    } else {
-      dispatchChangeSet({ type: 'reset', value: createChangeSet() });
-      setConfirmSql(null);
-      display.reload();
-    }
   }
 
   const insertNewRow = () => {
     if (display.baseTable) {
-      setChangeSet(changeSetInsertNewRow(changeSet, display.baseTable));
+      const rowIndex = grider.insertRow();
-      const cell = [rowCountNewIncluded, (currentCell && currentCell[1]) || 0];
+      const cell = [rowIndex, (currentCell && currentCell[1]) || 0];
       // @ts-ignore
       setCurrentCell(cell);
       // @ts-ignore
@@ -1060,7 +714,7 @@ export default function DataGridCore(props) {
 
   function handleCursorMove(event) {
     if (!isRegularCell(currentCell)) return null;
-    let rowCount = rowCountNewIncluded;
+    let rowCount = grider.rowCount;
     if (event.ctrlKey) {
       switch (event.keyCode) {
         case keycodes.upArrow:
@@ -1118,7 +772,7 @@ export default function DataGridCore(props) {
   }
 
   function moveCurrentCell(row, col, event = null) {
-    const rowCount = rowCountNewIncluded;
+    const rowCount = grider.rowCount;
 
     if (row < 0) row = 0;
     if (row >= rowCount) row = rowCount - 1;
@@ -1140,7 +794,7 @@ export default function DataGridCore(props) {
 
     if (row != null) {
       let newRow = null;
-      const rowCount = rowCountNewIncluded;
+      const rowCount = grider.rowCount;
       if (rowCount == 0) return;
 
       if (row < firstVisibleRowScrollIndex) newRow = row;
@@ -1202,7 +856,7 @@ export default function DataGridCore(props) {
   //   columnSizes.getVisibleScrollSizeSum()
   // );
 
-  const loadedAndInsertedRows = [...loadedRows, ...insertedRows];
+  // const loadedAndInsertedRows = [...loadedRows, ...insertedRows];
 
   // console.log('focusFieldRef.current', focusFieldRef.current);
 
@@ -1282,28 +936,20 @@ export default function DataGridCore(props) {
           )}
         </TableHead>
         <TableBody ref={tableBodyRef}>
-          {loadedAndInsertedRows
+          {_.range(firstVisibleRowScrollIndex, firstVisibleRowScrollIndex + visibleRowCountUpperBound)
-            .slice(firstVisibleRowScrollIndex, firstVisibleRowScrollIndex + visibleRowCountUpperBound)
+            .map((rowIndex) => (
-            .map((row, index) => (
               <DataGridRow
-                key={firstVisibleRowScrollIndex + index}
+                key={rowIndex}
-                rowIndex={firstVisibleRowScrollIndex + index}
+                grider={grider}
+                rowIndex={rowIndex}
                 rowHeight={rowHeight}
                 visibleRealColumns={visibleRealColumns}
                 inplaceEditorState={inplaceEditorState}
                 dispatchInsplaceEditor={dispatchInsplaceEditor}
                 autofillSelectedCells={autofillSelectedCells}
-                selectedCells={filterCellsForRow(selectedCells, firstVisibleRowScrollIndex + index)}
+                selectedCells={filterCellsForRow(selectedCells, rowIndex)}
-                insertedRowIndex={
+                autofillMarkerCell={filterCellForRow(autofillMarkerCell, rowIndex)}
-                  firstVisibleRowScrollIndex + index >= loadedRows.length
-                    ? firstVisibleRowScrollIndex + index - loadedRows.length
-                    : null
-                }
-                autofillMarkerCell={filterCellForRow(autofillMarkerCell, firstVisibleRowScrollIndex + index)}
-                changeSet={changeSet}
-                setChangeSet={setChangeSet}
                 display={display}
-                row={row}
                 focusedColumn={display.focusedColumn}
               />
             ))}
@@ -1321,15 +967,9 @@ export default function DataGridCore(props) {
         valueToSet={vScrollValueToSet}
         valueToSetDate={vScrollValueToSetDate}
         minimum={0}
-        maximum={rowCountNewIncluded - visibleRowCountUpperBound + 2}
+        maximum={grider.rowCount - visibleRowCountUpperBound + 2}
         onScroll={handleRowScroll}
-        viewportRatio={visibleRowCountUpperBound / rowCountNewIncluded}
+        viewportRatio={visibleRowCountUpperBound / grider.rowCount}
-      />
-      <ConfirmSqlModal
-        modalState={confirmSqlModalState}
-        sql={confirmSql}
-        engine={display.engine}
-        onConfirm={handleConfirmSql}
       />
       {allRowCount && <RowCountLabel>{rowCountInfo}</RowCountLabel>}
       {props.toolbarPortalRef &&
@@ -1338,9 +978,7 @@ export default function DataGridCore(props) {
           <DataGridToolbar
             reload={() => display.reload()}
             save={handleSave}
-            changeSetState={changeSetState}
+            grider={grider}
-            dispatchChangeSet={dispatchChangeSet}
-            revert={revertAllChanges}
           />,
           props.toolbarPortalRef.current
         )}

packages/web/src/datagrid/DataGridRow.js

@@ -81,8 +81,7 @@ const TableBodyCell = styled.td`
       // from http://www.patternify.com/
       background-repeat: repeat-x;
       background-position: 50% 50%;`}
-        
+`;
-  `;
 
 const HintSpan = styled.span`
   color: gray;
@@ -163,22 +162,20 @@ function CellFormattedValue({ value, dataType }) {
   return value.toString();
 }
 
-function DataGridRow({
+/** @param props {import('./types').DataGridProps} */
+function DataGridRow(props) {
+  const {
     rowHeight,
     rowIndex,
     visibleRealColumns,
     inplaceEditorState,
     dispatchInsplaceEditor,
-  row,
-  display,
-  changeSet,
-  setChangeSet,
-  insertedRowIndex,
     autofillMarkerCell,
     selectedCells,
     autofillSelectedCells,
     focusedColumn,
-}) {
+    grider,
+  } = props;
   // usePropsCompare({
   //   rowHeight,
   //   rowIndex,
@@ -197,18 +194,19 @@ function DataGridRow({
 
   // console.log('RENDER ROW', rowIndex);
 
-  const rowDefinition = display.getChangeSetRow(row, insertedRowIndex);
+  const rowData = grider.getRowData(rowIndex);
-  const [matchedField, matchedChangeSetItem] = findExistingChangeSetItem(changeSet, rowDefinition);
+  const rowStatus = grider.getRowStatus(rowIndex);
-  const rowUpdated = matchedChangeSetItem ? { ...row, ...matchedChangeSetItem.fields } : row;
+
   const hintFieldsAllowed = visibleRealColumns
     .filter((col) => {
       if (!col.hintColumnName) return false;
-      if (matchedChangeSetItem && matchedField == 'updates' && col.uniqueName in matchedChangeSetItem.fields)
+      if (rowStatus.status == 'updated' && rowStatus.modifiedFields.has(col.uniqueName)) return false;
-        return false;
       return true;
     })
     .map((col) => col.uniqueName);
 
+  if (!rowData) return null;
+
   return (
     <TableBodyRow style={{ height: `${rowHeight}px` }}>
       <TableHeaderCell data-row={rowIndex} data-col="header">
@@ -226,13 +224,11 @@ function DataGridRow({
           data-col={col.colIndex}
           isSelected={cellIsSelected(rowIndex, col.colIndex, selectedCells)}
           isAutofillSelected={cellIsSelected(rowIndex, col.colIndex, autofillSelectedCells)}
-          isModifiedRow={!!matchedChangeSetItem}
+          isModifiedRow={rowStatus.status == 'updated'}
           isFocusedColumn={col.uniqueName == focusedColumn}
-          isModifiedCell={
+          isModifiedCell={rowStatus.status == 'updated' && rowStatus.modifiedFields.has(col.uniqueName)}
-            matchedChangeSetItem && matchedField == 'updates' && col.uniqueName in matchedChangeSetItem.fields
+          isInsertedRow={rowStatus.status == 'inserted'}
-          }
+          isDeletedRow={rowStatus.status == 'deleted'}
-          isInsertedRow={insertedRowIndex != null}
-          isDeletedRow={matchedField == 'deletes'}
         >
           {inplaceEditorState.cell &&
           rowIndex == inplaceEditorState.cell[0] &&
@@ -241,16 +237,15 @@ function DataGridRow({
               widthPx={col.widthPx}
               inplaceEditorState={inplaceEditorState}
               dispatchInsplaceEditor={dispatchInsplaceEditor}
-              cellValue={rowUpdated[col.uniqueName]}
+              cellValue={rowData[col.uniqueName]}
-              changeSet={changeSet}
+              grider={grider}
-              setChangeSet={setChangeSet}
+              rowIndex={rowIndex}
-              insertedRowIndex={insertedRowIndex}
+              uniqueName={col.uniqueName}
-              definition={display.getChangeSetField(row, col.uniqueName, insertedRowIndex)}
             />
           ) : (
             <>
-              <CellFormattedValue value={rowUpdated[col.uniqueName]} dataType={col.dataType} />
+              <CellFormattedValue value={rowData[col.uniqueName]} dataType={col.dataType} />
-              {hintFieldsAllowed.includes(col.uniqueName) && <HintSpan>{row[col.hintColumnName]}</HintSpan>}
+              {hintFieldsAllowed.includes(col.uniqueName) && <HintSpan>{rowData[col.hintColumnName]}</HintSpan>}
             </>
           )}
           {autofillMarkerCell && autofillMarkerCell[1] == col.colIndex && autofillMarkerCell[0] == rowIndex && (

packages/web/src/datagrid/DataGridToolbar.js

@@ -1,23 +1,22 @@
 import React from 'react';
 import ToolbarButton from '../widgets/ToolbarButton';
-import { changeSetContainsChanges } from '@dbgate/datalib';
 
-export default function DataGridToolbar({ reload, changeSetState, dispatchChangeSet, save, revert }) {
+export default function DataGridToolbar({ reload, grider, save }) {
   return (
     <>
       <ToolbarButton onClick={reload} icon="fas fa-sync">
         Refresh
       </ToolbarButton>
-      <ToolbarButton disabled={!changeSetState.canUndo} onClick={() => dispatchChangeSet({ type: 'undo' })} icon="fas fa-undo">
+      <ToolbarButton disabled={!grider.canUndo} onClick={() => grider.undo()} icon="fas fa-undo">
         Undo
       </ToolbarButton>
-      <ToolbarButton disabled={!changeSetState.canRedo} onClick={() => dispatchChangeSet({ type: 'redo' })} icon="fas fa-redo">
+      <ToolbarButton disabled={!grider.canRedo} onClick={() => grider.redo()} icon="fas fa-redo">
         Redo
       </ToolbarButton>
-      <ToolbarButton disabled={!changeSetContainsChanges(changeSetState.value)} onClick={save} icon="fas fa-save">
+      <ToolbarButton disabled={!grider.containsChanges} onClick={save} icon="fas fa-save">
         Save
       </ToolbarButton>
-      <ToolbarButton disabled={!changeSetContainsChanges(changeSetState.value)} onClick={revert} icon="fas fa-times">
+      <ToolbarButton disabled={!grider.containsChanges} onClick={() => grider.revertAllChanges()} icon="fas fa-times">
         Revert
       </ToolbarButton>
     </>

packages/web/src/datagrid/Grider.ts

@@ -0,0 +1,52 @@
+export interface GriderRowStatus {
+  status: 'regular' | 'updated' | 'deleted' | 'inserted';
+  modifiedFields: Set<string>;
+}
+
+export default abstract class Grider {
+  abstract getRowData(index): any;
+  abstract get rowCount(): number;
+
+  getRowsSample() {
+    return [this.getRowData(0)];
+  }
+
+  getRowStatus(index): GriderRowStatus {
+    const res: GriderRowStatus = {
+      status: 'regular',
+      modifiedFields: new Set(),
+    };
+    return res;
+  }
+  beginUpdate() {}
+  endUpdate() {}
+  setCellValue(index: number, uniqueName: string, value: any) {}
+  deleteRow(index: number) {}
+  insertRow(): number {
+    return null;
+  }
+  revertRowChanges(index: number) {}
+  revertAllChanges() {}
+  undo() {}
+  redo() {}
+  get rowCountInUpdate() {
+    return this.rowCount;
+  }
+  get canUndo() {
+    return false;
+  }
+  get canRedo() {
+    return false;
+  }
+  get containsChanges() {
+    return false;
+  }
+  get disableLoadNextPage() {
+    return false;
+  }
+  updateRow(index, changeObject) {
+    for (const key of Object.keys(changeObject)) {
+      this.setCellValue(index, key, changeObject[key]);
+    }
+  }
+}

packages/web/src/datagrid/InplaceEditor.js

@@ -5,7 +5,6 @@ import React from 'react';
 import styled from 'styled-components';
 import theme from '../theme';
 import keycodes from '../utility/keycodes';
-import { setChangeSetValue } from '@dbgate/datalib';
 
 const StyledInput = styled.input`
   border: 0px solid;
@@ -16,13 +15,12 @@ const StyledInput = styled.input`
 
 export default function InplaceEditor({
   widthPx,
-  definition,
+  rowIndex,
-  changeSet,
+  uniqueName,
-  setChangeSet,
+  grider,
   cellValue,
   inplaceEditorState,
   dispatchInsplaceEditor,
-  isInsertedRow,
 }) {
   const editorRef = React.useRef();
   const isChangedRef = React.useRef(!!inplaceEditorState.text);
@@ -37,7 +35,7 @@ export default function InplaceEditor({
   function handleBlur() {
     if (isChangedRef.current) {
       const editor = editorRef.current;
-      setChangeSet(setChangeSetValue(changeSet, definition, editor.value));
+      grider.setCellValue(rowIndex, uniqueName, editor.value);
       isChangedRef.current = false;
     }
     dispatchInsplaceEditor({ type: 'close' });
@@ -45,7 +43,7 @@ export default function InplaceEditor({
   if (inplaceEditorState.shouldSave) {
     const editor = editorRef.current;
     if (isChangedRef.current) {
-      setChangeSet(setChangeSetValue(changeSet, definition, editor.value));
+      grider.setCellValue(rowIndex, uniqueName, editor.value);
       isChangedRef.current = false;
     }
     editor.blur();
@@ -60,7 +58,7 @@ export default function InplaceEditor({
         break;
       case keycodes.enter:
         if (isChangedRef.current) {
-          setChangeSet(setChangeSetValue(changeSet, definition, editor.value));
+          grider.setCellValue(rowIndex, uniqueName, editor.value);
           isChangedRef.current = false;
         }
         editor.blur();
@@ -69,7 +67,7 @@ export default function InplaceEditor({
       case keycodes.s:
         if (event.ctrlKey) {
           if (isChangedRef.current) {
-            setChangeSet(setChangeSetValue(changeSet, definition, editor.value));
+            grider.setCellValue(rowIndex, uniqueName, editor.value);
             isChangedRef.current = false;
           }
           event.preventDefault();

packages/web/src/datagrid/JslDataGridCore.js

@@ -0,0 +1,93 @@
+import React from 'react';
+import axios from '../utility/axios';
+import { useSetOpenedTabs } from '../utility/globalState';
+import useSocket from '../utility/SocketProvider';
+import useShowModal from '../modals/showModal';
+import ImportExportModal from '../modals/ImportExportModal';
+import LoadingDataGridCore from './LoadingDataGridCore';
+import RowsArrayGrider from './RowsArrayGrider';
+
+async function loadDataPage(props, offset, limit) {
+  const { jslid } = props;
+
+  const response = await axios.request({
+    url: 'jsldata/get-rows',
+    method: 'get',
+    params: {
+      jslid,
+      offset,
+      limit,
+    },
+  });
+  return response.data;
+}
+
+function dataPageAvailable(props) {
+  return true;
+}
+
+async function loadRowCount(props) {
+  const { jslid } = props;
+
+  const response = await axios.request({
+    url: 'jsldata/get-stats',
+    method: 'get',
+    params: {
+      jslid,
+    },
+  });
+  return response.data.rowCount;
+}
+
+export default function JslDataGridCore(props) {
+  const { jslid } = props;
+  const [changeIndex, setChangeIndex] = React.useState(0);
+
+  const showModal = useShowModal();
+
+  const setOpenedTabs = useSetOpenedTabs();
+  const socket = useSocket();
+
+  function exportGrid() {
+    const initialValues = {};
+    const archiveMatch = jslid.match(/^archive:\/\/([^/]+)\/(.*)$/);
+    if (archiveMatch) {
+      initialValues.sourceStorageType = 'archive';
+      initialValues.sourceArchiveFolder = archiveMatch[1];
+      initialValues.sourceList = [archiveMatch[2]];
+    } else {
+      initialValues.sourceStorageType = 'jsldata';
+      initialValues.sourceJslId = jslid;
+      initialValues.sourceList = ['query-data'];
+    }
+    showModal((modalState) => <ImportExportModal modalState={modalState} initialValues={initialValues} />);
+  }
+
+  const handleJslDataStats = React.useCallback((stats) => {
+    if (stats.changeIndex < changeIndex) return;
+    setChangeIndex(stats.changeIndex);
+  }, [changeIndex]);
+
+  React.useEffect(() => {
+    if (jslid && socket) {
+      socket.on(`jsldata-stats-${jslid}`, handleJslDataStats);
+      return () => {
+        socket.off(`jsldata-stats-${jslid}`, handleJslDataStats);
+      };
+    }
+  }, [jslid]);
+
+  return (
+    <LoadingDataGridCore
+      {...props}
+      exportGrid={exportGrid}
+      loadDataPage={loadDataPage}
+      dataPageAvailable={dataPageAvailable}
+      loadRowCount={loadRowCount}
+      loadNextDataToken={changeIndex}
+      onReload={() => setChangeIndex(0)}
+      griderFactory={RowsArrayGrider.factory}
+      griderFactoryDeps={RowsArrayGrider.factoryDeps}
+    />
+  );
+}

packages/web/src/datagrid/LoadingDataGridCore.js

@@ -0,0 +1,135 @@
+import React from 'react';
+import DataGridCore from './DataGridCore';
+
+export default function LoadingDataGridCore(props) {
+  const {
+    display,
+    loadDataPage,
+    dataPageAvailable,
+    loadRowCount,
+    loadNextDataToken,
+    onReload,
+    exportGrid,
+    openQuery,
+    griderFactory,
+    griderFactoryDeps,
+  } = props;
+
+  const [loadProps, setLoadProps] = React.useState({
+    isLoading: false,
+    loadedRows: [],
+    isLoadedAll: false,
+    loadedTime: new Date().getTime(),
+    allRowCount: null,
+    errorMessage: null,
+    loadNextDataToken: 0,
+  });
+  const { isLoading, loadedRows, isLoadedAll, loadedTime, allRowCount, errorMessage } = loadProps;
+
+  const loadedTimeRef = React.useRef(0);
+
+  const handleLoadRowCount = async () => {
+    const rowCount = await loadRowCount(props);
+    setLoadProps((oldLoadProps) => ({
+      ...oldLoadProps,
+      allRowCount: rowCount,
+    }));
+  };
+
+  const reload = () => {
+    setLoadProps({
+      allRowCount: null,
+      isLoading: false,
+      loadedRows: [],
+      isLoadedAll: false,
+      loadedTime: new Date().getTime(),
+      errorMessage: null,
+      loadNextDataToken: 0,
+    });
+    if (onReload) onReload();
+  };
+
+  React.useEffect(() => {
+    if (props.masterLoadedTime && props.masterLoadedTime > loadedTime) {
+      display.reload();
+    }
+    if (display.cache.refreshTime > loadedTime) {
+      reload();
+    }
+  });
+
+  const loadNextData = async () => {
+    if (isLoading) return;
+    setLoadProps((oldLoadProps) => ({
+      ...oldLoadProps,
+      isLoading: true,
+    }));
+    const loadStart = new Date().getTime();
+    loadedTimeRef.current = loadStart;
+
+    const nextRows = await loadDataPage(props, loadedRows.length, 100);
+    if (loadedTimeRef.current !== loadStart) {
+      // new load was dispatched
+      return;
+    }
+    // if (!_.isArray(nextRows)) {
+    //   console.log('Error loading data from server', nextRows);
+    //   nextRows = [];
+    // }
+    // console.log('nextRows', nextRows);
+    if (nextRows.errorMessage) {
+      setLoadProps((oldLoadProps) => ({
+        ...oldLoadProps,
+        isLoading: false,
+        errorMessage: nextRows.errorMessage,
+      }));
+    } else {
+      if (allRowCount == null) handleLoadRowCount();
+      const loadedInfo = {
+        loadedRows: [...loadedRows, ...nextRows],
+        loadedTime,
+      };
+      setLoadProps((oldLoadProps) => ({
+        ...oldLoadProps,
+        isLoading: false,
+        isLoadedAll: oldLoadProps.loadNextDataToken == loadNextDataToken && nextRows.length === 0,
+        loadNextDataToken,
+        ...loadedInfo,
+      }));
+    }
+  };
+
+  React.useEffect(() => {
+    setLoadProps((oldProps) => ({
+      ...oldProps,
+      isLoadedAll: false,
+    }));
+  }, [loadNextDataToken]);
+
+  const griderProps = { ...props, sourceRows: loadedRows };
+  const grider = React.useMemo(() => griderFactory(griderProps), griderFactoryDeps(griderProps));
+
+  const handleLoadNextData = () => {
+    if (!isLoadedAll && !errorMessage && !grider.disableLoadNextPage) {
+      if (dataPageAvailable(props)) {
+        // If not, callbacks to load missing metadata are dispatched
+        loadNextData();
+      }
+    }
+  };
+
+  return (
+    <DataGridCore
+      {...props}
+      loadNextData={handleLoadNextData}
+      errorMessage={errorMessage}
+      isLoadedAll={isLoadedAll}
+      loadedTime={loadedTime}
+      exportGrid={exportGrid}
+      allRowCount={allRowCount}
+      openQuery={openQuery}
+      isLoading={isLoading}
+      grider={grider}
+    />
+  );
+}

packages/web/src/datagrid/RowsArrayGrider.ts

@@ -0,0 +1,23 @@
+import Grider, { GriderRowStatus } from './Grider';
+
+export default class RowsArrayGrider extends Grider {
+  constructor(private rows: any[]) {
+    super();
+  }
+  getRowData(index: any) {
+    return this.rows[index];
+  }
+  get rowCount() {
+    return this.rows.length;
+  }
+
+  static factory({ sourceRows }): RowsArrayGrider {
+    return new RowsArrayGrider(sourceRows);
+  }
+  static factoryDeps({ sourceRows }) {
+    return [sourceRows];
+  }
+  getRowsSample() {
+    return this.rows;
+  }
+}

packages/web/src/datagrid/SqlDataGridCore.js

@@ -0,0 +1,151 @@
+import React from 'react';
+import axios from '../utility/axios';
+import { useSetOpenedTabs } from '../utility/globalState';
+import DataGridCore from './DataGridCore';
+import useSocket from '../utility/SocketProvider';
+import useShowModal from '../modals/showModal';
+import ImportExportModal from '../modals/ImportExportModal';
+import { changeSetToSql, createChangeSet, getChangeSetInsertedRows } from '@dbgate/datalib';
+import { openNewTab } from '../utility/common';
+import LoadingDataGridCore from './LoadingDataGridCore';
+import ChangeSetGrider from './ChangeSetGrider';
+import { scriptToSql } from '@dbgate/sqltree';
+import useModalState from '../modals/useModalState';
+import ConfirmSqlModal from '../modals/ConfirmSqlModal';
+import ErrorMessageModal from '../modals/ErrorMessageModal';
+
+/** @param props {import('./types').DataGridProps} */
+async function loadDataPage(props, offset, limit) {
+  const { display, conid, database } = props;
+
+  const sql = display.getPageQuery(offset, limit);
+
+  const response = await axios.request({
+    url: 'database-connections/query-data',
+    method: 'post',
+    params: {
+      conid,
+      database,
+    },
+    data: { sql },
+  });
+
+  if (response.data.errorMessage) return response.data;
+  return response.data.rows;
+}
+
+function dataPageAvailable(props) {
+  const { display } = props;
+  const sql = display.getPageQuery(0, 1);
+  return !!sql;
+}
+
+async function loadRowCount(props) {
+  const { display, conid, database } = props;
+
+  const sql = display.getCountQuery();
+
+  const response = await axios.request({
+    url: 'database-connections/query-data',
+    method: 'post',
+    params: {
+      conid,
+      database,
+    },
+    data: { sql },
+  });
+
+  return parseInt(response.data.rows[0].count);
+}
+
+/** @param props {import('./types').DataGridProps} */
+export default function SqlDataGridCore(props) {
+  const { conid, database, display, changeSetState, dispatchChangeSet, tabVisible } = props;
+  const showModal = useShowModal();
+  const setOpenedTabs = useSetOpenedTabs();
+
+  const confirmSqlModalState = useModalState();
+  const [confirmSql, setConfirmSql] = React.useState('');
+
+  const changeSet = changeSetState && changeSetState.value;
+  const changeSetRef = React.useRef(changeSet);
+  changeSetRef.current = changeSet;
+
+  function exportGrid() {
+    const initialValues = {};
+    initialValues.sourceStorageType = 'query';
+    initialValues.sourceConnectionId = conid;
+    initialValues.sourceDatabaseName = database;
+    initialValues.sourceSql = display.getExportQuery();
+    initialValues.sourceList = display.baseTable ? [display.baseTable.pureName] : [];
+    showModal((modalState) => <ImportExportModal modalState={modalState} initialValues={initialValues} />);
+  }
+  function openQuery() {
+    openNewTab(setOpenedTabs, {
+      title: 'Query',
+      icon: 'sql.svg',
+      tabComponent: 'QueryTab',
+      props: {
+        initialScript: display.getExportQuery(),
+        schemaName: display.baseTable.schemaName,
+        pureName: display.baseTable.pureName,
+        conid,
+        database,
+      },
+    });
+  }
+
+  function handleSave() {
+    const script = changeSetToSql(changeSetRef.current, display.dbinfo);
+    const sql = scriptToSql(display.driver, script);
+    setConfirmSql(sql);
+    confirmSqlModalState.open();
+  }
+
+  async function handleConfirmSql() {
+    const resp = await axios.request({
+      url: 'database-connections/query-data',
+      method: 'post',
+      params: {
+        conid,
+        database,
+      },
+      data: { sql: confirmSql },
+    });
+    const { errorMessage } = resp.data || {};
+    if (errorMessage) {
+      showModal((modalState) => (
+        <ErrorMessageModal modalState={modalState} message={errorMessage} title="Error when saving" />
+      ));
+    } else {
+      dispatchChangeSet({ type: 'reset', value: createChangeSet() });
+      setConfirmSql(null);
+      display.reload();
+    }
+  }
+
+  // const grider = React.useMemo(()=>new ChangeSetGrider())
+
+  return (
+    <>
+      <LoadingDataGridCore
+        {...props}
+        exportGrid={exportGrid}
+        openQuery={openQuery}
+        loadDataPage={loadDataPage}
+        dataPageAvailable={dataPageAvailable}
+        loadRowCount={loadRowCount}
+        griderFactory={ChangeSetGrider.factory}
+        griderFactoryDeps={ChangeSetGrider.factoryDeps}
+        // changeSet={changeSetState && changeSetState.value}
+        onSave={handleSave}
+      />
+      <ConfirmSqlModal
+        modalState={confirmSqlModalState}
+        sql={confirmSql}
+        engine={display.engine}
+        onConfirm={handleConfirmSql}
+      />
+    </>
+  );
+}

packages/web/src/datagrid/TableDataGrid.js

@@ -10,6 +10,7 @@ import useSocket from '../utility/SocketProvider';
 import { VerticalSplitter } from '../widgets/Splitter';
 import stableStringify from 'json-stable-stringify';
 import ReferenceHeader from './ReferenceHeader';
+import SqlDataGridCore from './SqlDataGridCore';
 
 const ReferenceContainer = styled.div`
   position: absolute;
@@ -162,6 +163,7 @@ export default function TableDataGrid({
         onReferenceSourceChanged={reference ? handleReferenceSourceChanged : null}
         refReloadToken={refReloadToken.toString()}
         masterLoadedTime={masterLoadedTime}
+        GridCore={SqlDataGridCore}
       />
       {reference && (
         <ReferenceContainer>

packages/web/src/datagrid/gridutil.ts

@@ -2,10 +2,11 @@ import _ from 'lodash';
 import { SeriesSizes } from './SeriesSizes';
 import { CellAddress } from './selection';
 import { GridDisplay } from '@dbgate/datalib';
+import Grider from './Grider';
 
-export function countColumnSizes(loadedRows, columns, containerWidth, display: GridDisplay) {
+export function countColumnSizes(grider: Grider, columns, containerWidth, display: GridDisplay) {
   const columnSizes = new SeriesSizes();
-  if (!loadedRows || !columns) return columnSizes;
+  if (!grider || !columns) return columnSizes;
 
   let canvas = document.createElement('canvas');
   let context = canvas.getContext('2d');
@@ -51,7 +52,8 @@ export function countColumnSizes(loadedRows, columns, containerWidth, display: G
   // if (headerWidth > this.rowHeaderWidth) this.rowHeaderWidth = headerWidth;
 
   context.font = '14px Helvetica';
-  for (let row of loadedRows.slice(0, 20)) {
+  for (let rowIndex = 0; rowIndex < Math.min(grider.rowCount, 20); rowIndex += 1) {
+    const row = grider.getRowData(rowIndex);
     for (let colIndex = 0; colIndex < columns.length; colIndex++) {
       const uqName = columns[colIndex].uniqueName;
 

packages/web/src/datagrid/types.ts

@@ -1,18 +1,46 @@
 import { GridDisplay, ChangeSet, GridReferenceDefinition } from '@dbgate/datalib';
+import Grider from './Grider';
 
 export interface DataGridProps {
-  conid?: string;
-  database?: string;
   display: GridDisplay;
   tabVisible?: boolean;
   changeSetState?: { value: ChangeSet };
   dispatchChangeSet?: Function;
   toolbarPortalRef?: any;
-  jslid?: string;
   showReferences?: boolean;
   onReferenceClick?: (def: GridReferenceDefinition) => void;
   onReferenceSourceChanged?: Function;
   refReloadToken?: string;
   masterLoadedTime?: number;
   managerSize?: number;
+  grider?: Grider;
+  conid?: string;
+  database?: string;
+  jslid?: string;
+
+  [field: string]: any;
 }
+
+// export interface DataGridCoreProps extends DataGridProps {
+//   rows: any[];
+//   loadNextData?: Function;
+//   exportGrid?: Function;
+//   openQuery?: Function;
+//   undo?: Function;
+//   redo?: Function;
+
+//   errorMessage?: string;
+//   isLoadedAll?: boolean;
+//   loadedTime?: any;
+//   allRowCount?: number;
+//   conid?: string;
+//   database?: string;
+//   insertedRowCount?: number;
+//   isLoading?: boolean;
+// }
+
+// export interface LoadingDataGridProps extends DataGridProps {
+//   conid?: string;
+//   database?: string;
+//   jslid?: string;
+// }

packages/web/src/sqleditor/JslDataGrid.js

@@ -2,6 +2,7 @@ import React from 'react';
 import DataGrid from '../datagrid/DataGrid';
 import { JslGridDisplay, createGridConfig, createGridCache } from '@dbgate/datalib';
 import useFetch from '../utility/useFetch';
+import JslDataGridCore from '../datagrid/JslDataGridCore';
 
 export default function JslDataGrid({ jslid }) {
   const info = useFetch({
@@ -19,5 +20,5 @@ export default function JslDataGrid({ jslid }) {
     cache,
   ]);
 
-  return <DataGrid display={display} jslid={jslid} />;
+  return <DataGrid display={display} jslid={jslid} GridCore={JslDataGridCore} />;
 }

packages/web/src/tabs/ViewDataTab.js

@@ -10,6 +10,7 @@ import useUndoReducer from '../utility/useUndoReducer';
 import usePropsCompare from '../utility/usePropsCompare';
 import { useUpdateDatabaseForTab } from '../utility/globalState';
 import useGridConfig from '../utility/useGridConfig';
+import SqlDataGridCore from '../datagrid/SqlDataGridCore';
 
 export default function ViewDataTab({ conid, database, schemaName, pureName, tabVisible, toolbarPortalRef, tabid }) {
   const viewInfo = useViewInfo({ conid, database, schemaName, pureName });
@@ -50,6 +51,7 @@ export default function ViewDataTab({ conid, database, schemaName, pureName, tab
       changeSetState={changeSetState}
       dispatchChangeSet={dispatchChangeSet}
       toolbarPortalRef={toolbarPortalRef}
+      GridCore={SqlDataGridCore}
       />
   );
 }