perspective data pattern

packages/datalib/src/PerspectiveCache.ts

@@ -5,6 +5,7 @@ import _zip from 'lodash/zip';
 import _difference from 'lodash/difference';
 import debug from 'debug';
 import stableStringify from 'json-stable-stringify';
+import { PerspectiveDataPattern } from './PerspectiveDataPattern';
 
 const dbg = debug('dbgate:PerspectiveCache');
 
@@ -86,6 +87,7 @@ export class PerspectiveCache {
   constructor() {}
 
   tables: { [tableKey: string]: PerspectiveCacheTable } = {};
+  dataPatterns: PerspectiveDataPattern[] = [];
 
   getTableCache(props: PerspectiveDataLoadProps) {
     const tableKey = stableStringify(
@@ -113,5 +115,6 @@ export class PerspectiveCache {
 
   clear() {
     this.tables = {};
+    this.dataPatterns = [];
   }
 }

packages/datalib/src/PerspectiveDataLoader.ts

@@ -93,8 +93,8 @@ export class PerspectiveDataLoader {
     }));
   }
 
-  async loadData(props: PerspectiveDataLoadProps) {
+  async loadDataSqlDb(props: PerspectiveDataLoadProps) {
-    const { schemaName, pureName, bindingColumns, bindingValues, dataColumns, orderBy, condition } = props;
+    const { schemaName, pureName, bindingColumns, bindingValues, dataColumns, orderBy, condition, engineType } = props;
 
     if (dataColumns?.length == 0) {
       return [];
@@ -143,7 +143,53 @@ export class PerspectiveDataLoader {
     return response.rows;
   }
 
-  async loadRowCount(props: PerspectiveDataLoadProps) {
+  getDocDbLoadOptions(props: PerspectiveDataLoadProps) {
+    const { pureName } = props;
+    return {
+      pureName,
+      skip: props.range?.offset,
+      limit: props.range?.limit,
+    };
+  }
+
+  async loadDataDocDb(props: PerspectiveDataLoadProps) {
+    const { schemaName, pureName, bindingColumns, bindingValues, dataColumns, orderBy, condition, engineType } = props;
+
+    if (dataColumns?.length == 0) {
+      return [];
+    }
+
+    if (dbg?.enabled) {
+      dbg(
+        `LOAD DATA, collection=${props.pureName}, columns=${props.dataColumns?.join(',')}, range=${
+          props.range?.offset
+        },${props.range?.limit}`
+      );
+    }
+
+    const options = this.getDocDbLoadOptions(props);
+
+    const response = await this.apiCall('database-connections/collection-data', {
+      conid: props.databaseConfig.conid,
+      database: props.databaseConfig.database,
+      options,
+    });
+
+    if (response.errorMessage) return response;
+    return response.rows;
+  }
+
+  async loadData(props: PerspectiveDataLoadProps) {
+    const { engineType } = props;
+    switch (engineType) {
+      case 'sqldb':
+        return this.loadDataSqlDb(props);
+      case 'docdb':
+        return this.loadDataDocDb(props);
+    }
+  }
+
+  async loadRowCountSqlDb(props: PerspectiveDataLoadProps) {
     const { schemaName, pureName, bindingColumns, bindingValues, dataColumns, orderBy, condition } = props;
 
     const select: Select = {
@@ -170,4 +216,31 @@ export class PerspectiveDataLoader {
     if (response.errorMessage) return response;
     return response.rows[0];
   }
+
+  async loadRowCountDocDb(props: PerspectiveDataLoadProps) {
+    const { schemaName, pureName, bindingColumns, bindingValues, dataColumns, orderBy, condition } = props;
+
+    const options = {
+      ...this.getDocDbLoadOptions(props),
+      countDocuments: true,
+    };
+
+    const response = await this.apiCall('database-connections/collection-data', {
+      conid: props.databaseConfig.conid,
+      database: props.databaseConfig.database,
+      options,
+    });
+
+    return response;
+  }
+
+  async loadRowCount(props: PerspectiveDataLoadProps) {
+    const { engineType } = props;
+    switch (engineType) {
+      case 'sqldb':
+        return this.loadRowCountSqlDb(props);
+      case 'docdb':
+        return this.loadRowCountDocDb(props);
+    }
+  }
 }

packages/datalib/src/PerspectiveDataPattern.ts

@@ -0,0 +1,65 @@
+import { PerspectiveDataLoader } from './PerspectiveDataLoader';
+import { PerspectiveDataLoadProps } from './PerspectiveDataProvider';
+import _isString from 'lodash/isString';
+import _isPlainObject from 'lodash/isPlainObject';
+import _isNumber from 'lodash/isNumber';
+import _isBoolean from 'lodash/isBoolean';
+
+export type PerspectiveDataPatternColumnType = 'null' | 'string' | 'number' | 'boolean' | 'object';
+
+export interface PerspectiveDataPatternColumn {
+  name: string;
+  types: PerspectiveDataPatternColumnType[];
+  columns: PerspectiveDataPatternColumn[];
+}
+
+export interface PerspectiveDataPattern {
+  conid: string;
+  database: string;
+  schemaName: string;
+  pureName: string;
+  columns: PerspectiveDataPatternColumn[];
+}
+
+export type PerspectiveDataPatternDict = { [designerId: string]: PerspectiveDataPattern };
+
+function detectValueType(value): PerspectiveDataPatternColumnType {
+  if (_isString(value)) return 'string';
+  if (_isNumber(value)) return 'number';
+  if (_isBoolean(value)) return 'boolean';
+  if (value == null) return 'null';
+}
+
+function addObjectToColumns(columns: PerspectiveDataPatternColumn[], row) {
+  if (_isPlainObject(row)) {
+    for (const key of Object.keys(row)) {
+      let column: PerspectiveDataPatternColumn = columns.find(x => x.name == key);
+      if (!column) {
+        column = {
+          name: key,
+          types: [],
+          columns: [],
+        };
+        columns.push(column);
+      }
+      const type = detectValueType(row[key]);
+      if (!column.types.includes(type)) {
+        column.types.push(type);
+      }
+    }
+  }
+}
+
+export function analyseDataPattern(
+  patternBase: Omit<PerspectiveDataPattern, 'columns'>,
+  rows: any[]
+): PerspectiveDataPattern {
+  const res: PerspectiveDataPattern = {
+    ...patternBase,
+    columns: [],
+  };
+  for (const row of rows) {
+    addObjectToColumns(res.columns, row);
+  }
+  return res;
+}

packages/datalib/src/PerspectiveDataProvider.ts

@@ -4,6 +4,7 @@ import { RangeDefinition } from 'dbgate-types';
 import { format } from 'path';
 import { PerspectiveBindingGroup, PerspectiveCache } from './PerspectiveCache';
 import { PerspectiveDataLoader } from './PerspectiveDataLoader';
+import { PerspectiveDataPatternDict } from './PerspectiveDataPattern';
 
 export const PERSPECTIVE_PAGE_SIZE = 100;
 
@@ -28,10 +29,15 @@ export interface PerspectiveDataLoadProps {
   range?: RangeDefinition;
   topCount?: number;
   condition?: Condition;
+  engineType: 'sqldb' | 'docdb';
 }
 
 export class PerspectiveDataProvider {
-  constructor(public cache: PerspectiveCache, public loader: PerspectiveDataLoader) {}
+  constructor(
+    public cache: PerspectiveCache,
+    public loader: PerspectiveDataLoader,
+    public dataPatterns: PerspectiveDataPatternDict
+  ) {}
   async loadData(props: PerspectiveDataLoadProps): Promise<{ rows: any[]; incomplete: boolean }> {
     dbg('load data', props);
     // console.log('LOAD DATA', props);

packages/datalib/src/PerspectiveTreeNode.ts

@@ -1,4 +1,5 @@
 import {
+  CollectionInfo,
   ColumnInfo,
   DatabaseInfo,
   ForeignKeyInfo,
@@ -313,7 +314,7 @@ export abstract class PerspectiveTreeNode {
     };
   }
 
-  getOrderBy(table: TableInfo | ViewInfo): PerspectiveDataLoadProps['orderBy'] {
+  getOrderBy(table: TableInfo | ViewInfo | CollectionInfo): PerspectiveDataLoadProps['orderBy'] {
     const res = _compact(
       this.childNodes.map(node => {
         const sort = this.nodeConfig?.sort?.find(x => x.columnName == node.columnName);
@@ -325,11 +326,15 @@ export abstract class PerspectiveTreeNode {
         }
       })
     );
-    return res.length > 0
+    if (res.length > 0) return res;
-      ? res
+    const pkColumns = (table as TableInfo)?.primaryKey?.columns.map(x => ({
-      : (table as TableInfo)?.primaryKey?.columns.map(x => ({ columnName: x.columnName, order: 'ASC' })) || [
+      columnName: x.columnName,
-          { columnName: table?.columns[0].columnName, order: 'ASC' },
+      order: 'ASC' as 'ASC',
-        ];
+    }));
+    if (pkColumns) return pkColumns;
+    const columns = (table as TableInfo | ViewInfo)?.columns;
+    if (columns) return [{ columnName: columns[0].columnName, order: 'ASC' }];
+    return [{ columnName: '_id', order: 'ASC' }];
   }
 
   getBaseTables() {
@@ -553,6 +558,7 @@ export class PerspectiveTableColumnNode extends PerspectiveTreeNode {
       databaseConfig: this.databaseConfig,
       orderBy: this.getOrderBy(this.refTable),
       condition: this.getChildrenCondition(),
+      engineType: 'sqldb',
     };
   }
 
@@ -715,6 +721,7 @@ export class PerspectiveTableNode extends PerspectiveTreeNode {
       databaseConfig: this.databaseConfig,
       orderBy: this.getOrderBy(this.table),
       condition: this.getChildrenCondition(),
+      engineType: 'sqldb',
     };
   }
 
@@ -771,6 +778,88 @@ export class PerspectiveTableNode extends PerspectiveTreeNode {
   }
 }
 
+export class PerspectiveCollectionNode extends PerspectiveTreeNode {
+  constructor(
+    public collection: CollectionInfo,
+    dbs: MultipleDatabaseInfo,
+    config: PerspectiveConfig,
+    setConfig: ChangePerspectiveConfigFunc,
+    public dataProvider: PerspectiveDataProvider,
+    databaseConfig: PerspectiveDatabaseConfig,
+    parentNode: PerspectiveTreeNode,
+    designerId: string
+  ) {
+    super(dbs, config, setConfig, parentNode, dataProvider, databaseConfig, designerId);
+  }
+
+  getNodeLoadProps(parentRows: any[]): PerspectiveDataLoadProps {
+    return {
+      schemaName: this.collection.schemaName,
+      pureName: this.collection.pureName,
+      dataColumns: this.getDataLoadColumns(),
+      databaseConfig: this.databaseConfig,
+      orderBy: this.getOrderBy(this.collection),
+      condition: this.getChildrenCondition(),
+      engineType: 'docdb',
+    };
+  }
+
+  get codeName() {
+    return this.collection.schemaName
+      ? `${this.collection.schemaName}:${this.collection.pureName}`
+      : this.collection.pureName;
+  }
+
+  get title() {
+    return this.nodeConfig?.alias || this.collection.pureName;
+  }
+
+  get isExpandable() {
+    return true;
+  }
+
+  generateChildNodes(): PerspectiveTreeNode[] {
+    return [];
+    // return getTableChildPerspectiveNodes(
+    //   this.table,
+    //   this.dbs,
+    //   this.config,
+    //   this.setConfig,
+    //   this.dataProvider,
+    //   this.databaseConfig,
+    //   this
+    // );
+  }
+
+  get icon() {
+    return 'img collection';
+  }
+
+  getBaseTableFromThis() {
+    return this.collection;
+  }
+
+  get headerTableAttributes() {
+    return {
+      schemaName: this.collection.schemaName,
+      pureName: this.collection.pureName,
+      conid: this.databaseConfig.conid,
+      database: this.databaseConfig.database,
+    };
+  }
+
+  get tableCode() {
+    return `${this.collection.schemaName}|${this.collection.pureName}`;
+  }
+
+  get namedObject(): NamedObjectInfo {
+    return {
+      schemaName: this.collection.schemaName,
+      pureName: this.collection.pureName,
+    };
+  }
+}
+
 // export class PerspectiveViewNode extends PerspectiveTreeNode {
 //   constructor(
 //     public view: ViewInfo,
@@ -873,6 +962,7 @@ export class PerspectiveTableReferenceNode extends PerspectiveTableNode {
       databaseConfig: this.databaseConfig,
       orderBy: this.getOrderBy(this.table),
       condition: this.getChildrenCondition(),
+      engineType: 'sqldb',
     };
   }
 
@@ -978,6 +1068,7 @@ export class PerspectiveCustomJoinTreeNode extends PerspectiveTableNode {
       databaseConfig: this.databaseConfig,
       orderBy: this.getOrderBy(this.table),
       condition: this.getChildrenCondition(),
+      engineType: 'sqldb',
     };
   }
 

packages/datalib/src/index.ts

@@ -19,3 +19,4 @@ export * from './PerspectiveDataProvider';
 export * from './PerspectiveCache';
 export * from './PerspectiveConfig';
 export * from './processPerspectiveDefaultColunns';
+export * from './PerspectiveDataPattern';

packages/web/src/appobj/DatabaseObjectAppObject.svelte

@@ -345,6 +345,12 @@
           },
         },
       },
+      {
+        label: 'Open perspective',
+        tab: 'PerspectiveTab',
+        forceNewTab: true,
+        icon: 'img perspective',
+      },
       {
         label: 'Export',
         isExport: true,

packages/web/src/designer/DesignerTable.svelte

@@ -238,6 +238,7 @@
     class:isGrayed
     class:isTable={objectTypeField == 'tables'}
     class:isView={objectTypeField == 'views'}
+    class:isCollection={objectTypeField == 'collections'}
     use:moveDrag={settings?.canSelectColumns ? [handleMoveStart, handleMove, handleMoveEnd] : null}
     use:contextMenu={settings?.canSelectColumns ? createMenu : '__no_menu'}
     style={getTableColorStyle($currentThemeDefinition, table)}
@@ -358,6 +359,10 @@
   .header.isView {
     background: var(--theme-bg-magenta);
   }
+  .header.isCollection {
+    background: var(--theme-bg-red);
+  }
+
   .header.isGrayed {
     background: var(--theme-bg-2);
   }

packages/web/src/perspectives/PerspectiveDesigner.svelte

@@ -3,10 +3,12 @@
     createPerspectiveNodeConfig,
     MultipleDatabaseInfo,
     PerspectiveConfig,
+    PerspectiveDataPatternDict,
     perspectiveNodesHaveStructure,
     PerspectiveTreeNode,
     switchPerspectiveReferenceDirection,
   } from 'dbgate-datalib';
+  import { CollectionInfo } from 'dbgate-types';
   import _ from 'lodash';
   import { tick } from 'svelte';
   import runCommand from '../commands/runCommand';
@@ -18,6 +20,7 @@
 
   export let config: PerspectiveConfig;
   export let dbInfos: MultipleDatabaseInfo;
+  export let dataPatterns: PerspectiveDataPatternDict;
   export let root: PerspectiveTreeNode;
 
   export let conid;
@@ -27,7 +30,11 @@
 
   export let onClickTableHeader = null;
 
-  function createDesignerModel(config: PerspectiveConfig, dbInfos: MultipleDatabaseInfo) {
+  function createDesignerModel(
+    config: PerspectiveConfig,
+    dbInfos: MultipleDatabaseInfo,
+    dataPatterns: PerspectiveDataPatternDict
+  ) {
     return {
       ...config,
       tables: _.compact(
@@ -38,11 +45,26 @@
           const view = dbInfos?.[node.conid || conid]?.[node.database || database]?.views?.find(
             x => x.pureName == node.pureName && x.schemaName == node.schemaName
           );
-          if (!table && !view) return null;
+          let collection: CollectionInfo & { columns?: any[] } = dbInfos?.[node.conid || conid]?.[
+            node.database || database
+          ]?.collections?.find(x => x.pureName == node.pureName && x.schemaName == node.schemaName);
+
+          if (collection) {
+            const pattern = dataPatterns?.[node.designerId];
+            if (!pattern) return null;
+            collection = {
+              ...collection,
+              columns: pattern.columns.map(x => ({
+                columnName: x.name,
+              })),
+            };
+          }
+
+          if (!table && !view && !collection) return null;
 
           const { designerId } = node;
           return {
-            ...(table || view),
+            ...(table || view || collection),
             left: node?.position?.x || 0,
             top: node?.position?.y || 0,
             alias: node.alias,
@@ -55,7 +77,7 @@
 
   function handleChange(value, skipUndoChain, settings) {
     setConfig(oldValue => {
-      const newValue = _.isFunction(value) ? value(createDesignerModel(oldValue, dbInfos)) : value;
+      const newValue = _.isFunction(value) ? value(createDesignerModel(oldValue, dbInfos, dataPatterns)) : value;
       let isArranged = oldValue.isArranged;
       if (settings?.isCalledFromArrange) {
         isArranged = true;
@@ -277,6 +299,6 @@
     onClickTableHeader,
   }}
   referenceComponent={QueryDesignerReference}
-  value={createDesignerModel(config, dbInfos)}
+  value={createDesignerModel(config, dbInfos, dataPatterns)}
   onChange={handleChange}
 />

packages/web/src/perspectives/PerspectiveView.svelte

@@ -28,6 +28,7 @@
   import {
     ChangePerspectiveConfigFunc,
     extractPerspectiveDatabases,
+    PerspectiveCollectionNode,
     PerspectiveConfig,
     PerspectiveDataProvider,
     PerspectiveTableNode,
@@ -65,6 +66,7 @@
   import { sleep } from '../utility/common';
   import FontIcon from '../icons/FontIcon.svelte';
   import InlineButton from '../buttons/InlineButton.svelte';
+  import { usePerspectiveDataPatterns } from '../utility/usePerspectiveDataPatterns';
 
   const dbg = debug('dbgate:PerspectiveView');
 
@@ -128,13 +130,17 @@
   }
 
   $: dbInfos = useMultipleDatabaseInfo(perspectiveDatabases);
+  $: loader = new PerspectiveDataLoader(apiCall);
+  $: dataPatterns = usePerspectiveDataPatterns({ conid, database }, config, cache, $dbInfos, loader);
   $: rootObject = config?.nodes?.find(x => x.designerId == config?.rootDesignerId);
   $: rootDb = rootObject ? $dbInfos?.[rootObject.conid || conid]?.[rootObject.database || database] : null;
   $: tableInfo = rootDb?.tables.find(x => x.pureName == rootObject?.pureName && x.schemaName == rootObject?.schemaName);
   $: viewInfo = rootDb?.views.find(x => x.pureName == rootObject?.pureName && x.schemaName == rootObject?.schemaName);
+  $: collectionInfo = rootDb?.collections.find(
+    x => x.pureName == rootObject?.pureName && x.schemaName == rootObject?.schemaName
+  );
 
-  $: loader = new PerspectiveDataLoader(apiCall);
+  $: dataProvider = new PerspectiveDataProvider(cache, loader, $dataPatterns);
-  $: dataProvider = new PerspectiveDataProvider(cache, loader);
   $: root =
     tableInfo || viewInfo
       ? new PerspectiveTableNode(
@@ -147,6 +153,17 @@
           null,
           config.rootDesignerId
         )
+      : collectionInfo
+      ? new PerspectiveCollectionNode(
+          collectionInfo,
+          $dbInfos,
+          config,
+          setConfig,
+          dataProvider,
+          { conid, database },
+          null,
+          config.rootDesignerId
+        )
       : null;
   $: tempRoot = root?.findNodeByDesignerId(tempRootDesignerId);
 
@@ -158,6 +175,7 @@
 
   // $: console.log('PERSPECTIVE', config);
   // $: console.log('VIEW ROOT', root);
+  // $: console.log('dataPatterns', $dataPatterns);
 </script>
 
 <HorizontalSplitter initialValue={getInitialManagerSize()} bind:size={managerSize} allowCollapseChild1>
@@ -205,6 +223,7 @@
           {database}
           {setConfig}
           dbInfos={$dbInfos}
+          dataPatterns={$dataPatterns}
           {root}
           onClickTableHeader={designerId => {
             sleep(100).then(() => {

packages/web/src/utility/usePerspectiveDataPatterns.ts

@@ -0,0 +1,77 @@
+import {
+  analyseDataPattern,
+  MultipleDatabaseInfo,
+  PerspectiveCache,
+  PerspectiveConfig,
+  PerspectiveDatabaseConfig,
+  PerspectiveDataLoadProps,
+  PerspectiveDataPattern,
+  PerspectiveDataPatternDict,
+} from 'dbgate-datalib';
+import { PerspectiveDataLoader } from 'dbgate-datalib/lib/PerspectiveDataLoader';
+import { writable, Readable } from 'svelte/store';
+
+export async function getPerspectiveDataPatterns(
+  databaseConfig: PerspectiveDatabaseConfig,
+  config: PerspectiveConfig,
+  cache: PerspectiveCache,
+  dbInfos: MultipleDatabaseInfo,
+  dataLoader: PerspectiveDataLoader
+): Promise<PerspectiveDataPatternDict> {
+  const res = {};
+
+  for (const node of config.nodes) {
+    const conid = node.conid || databaseConfig.conid;
+    const database = node.database || databaseConfig.database;
+    const { schemaName, pureName } = node;
+
+    const cached = cache.dataPatterns.find(
+      x => x.conid == conid && x.database == database && x.schemaName == schemaName && x.pureName == pureName
+    );
+    if (cached) {
+      res[node.designerId] = cached;
+      continue;
+    }
+
+    const db = dbInfos?.[conid]?.[database];
+
+    if (!db) continue;
+
+    const collection = db.collections?.find(x => x.pureName == pureName && x.schemaName == schemaName);
+    if (!collection) continue;
+
+    const props: PerspectiveDataLoadProps = {
+      databaseConfig: { conid, database },
+      engineType: 'docdb',
+      pureName,
+      orderBy: [],
+    };
+    const rows = await dataLoader.loadData(props);
+    const pattern = analyseDataPattern(
+      {
+        conid,
+        database,
+        pureName,
+        schemaName,
+      },
+      rows
+    );
+
+    cache.dataPatterns.push(pattern);
+    res[node.designerId] = pattern;
+  }
+
+  return res;
+}
+
+export function usePerspectiveDataPatterns(
+  databaseConfig: PerspectiveDatabaseConfig,
+  config: PerspectiveConfig,
+  cache: PerspectiveCache,
+  dbInfos: MultipleDatabaseInfo,
+  dataLoader: PerspectiveDataLoader
+): Readable<PerspectiveDataPatternDict> {
+  const res = writable({});
+  getPerspectiveDataPatterns(databaseConfig, config, cache, dbInfos, dataLoader).then(value => res.set(value));
+  return res;
+}