better type habndling, shell-tableReader for preserving table structure

packages/api/src/shell/index.js

@@ -3,6 +3,7 @@ const csvWriter = require('./csvWriter');
 const csvReader = require('./csvReader');
 const runScript = require('./runScript');
 const tableWriter = require('./tableWriter');
+const tableReader = require('./tableReader');
 const copyStream = require('./copyStream');
 const fakeObjectReader = require('./fakeObjectReader');
 const consoleObjectWriter = require('./consoleObjectWriter');
@@ -16,6 +17,7 @@ module.exports = {
   csvReader,
   runScript,
   tableWriter,
+  tableReader,
   copyStream,
   excelSheetReader,
   jsonLinesWriter,

packages/api/src/shell/tableReader.js

@@ -0,0 +1,28 @@
+const { quoteFullName } = require('@dbgate/tools');
+const driverConnect = require('../utility/driverConnect');
+
+const engines = require('@dbgate/engines');
+
+async function queryReader({ connection, pureName, schemaName }) {
+  const driver = engines(connection);
+  const pool = await driverConnect(driver, connection);
+  console.log(`Connected.`);
+
+  const fullName = { pureName, schemaName };
+
+  const table = await driver.analyseSingleObject(pool, fullName, 'tables');
+  const query = `select * from ${quoteFullName(driver.dialect, fullName)}`;
+  if (table) {
+    console.log(`Reading table ${table}`);
+    return await driver.readQuery(pool, query, table);
+  }
+  const view = await driver.analyseSingleObject(pool, fullName, 'views');
+  if (view) {
+    console.log(`Reading view ${table}`);
+    return await driver.readQuery(pool, query, view);
+  }
+
+  return await driver.readQuery(pool, query);
+}
+
+module.exports = queryReader;

packages/engines/mssql/MsSqlAnalyser.js

@@ -4,6 +4,7 @@ const sql = require('./sql');
 
 const DatabaseAnalyser = require('../default/DatabaseAnalyser');
 const { filter } = require('lodash');
+const { isTypeString } = require('@dbgate/tools');
 
 function objectTypeToField(type) {
   switch (type.trim()) {
@@ -24,10 +25,12 @@ function objectTypeToField(type) {
   }
 }
 
-function getColumnInfo({ isNullable, isIdentity, columnName, dataType }) {
+function getColumnInfo({ isNullable, isIdentity, columnName, dataType, charMaxLength }) {
+  let fullDataType = dataType;
+  if (charMaxLength && isTypeString(dataType)) fullDataType = `${dataType}(${charMaxLength})`;
   return {
     columnName,
-    dataType,
+    dataType: fullDataType,
     notNull: !isNullable,
     autoIncrement: !!isIdentity,
   };

packages/engines/mssql/index.js

@@ -22,7 +22,6 @@ function extractColumns(columns) {
     ...col,
     columnName: col.name,
     notNull: !col.nullable,
-    autoIncrement: !!col.identity,
   }));
 
   const generateName = () => {
@@ -164,7 +163,7 @@ const driver = {
 
     return request;
   },
-  async readQuery(pool, sql) {
+  async readQuery(pool, sql, structure) {
     const request = await pool.request();
     const { stream } = pool._nativeModules;
 
@@ -176,7 +175,7 @@ const driver = {
     request.stream = true;
     request.on('recordset', (driverColumns) => {
       const [columns, mapper] = extractColumns(driverColumns);
-      pass.write({ columns });
+      pass.write(structure || { columns });
     });
     request.on('row', (row) => pass.write(row));
     request.on('error', (err) => {

packages/engines/mssql/sql/columns.js

@@ -1,6 +1,7 @@
 module.exports = `
 select c.name as columnName, t.name as dataType, c.object_id as objectId, c.is_identity as isIdentity,
 	c.max_length as maxLength, c.precision, c.scale, c.is_nullable as isNullable,
+	col.CHARACTER_MAXIMUM_LENGTH as charMaxLength,
 	d.definition as defaultValue, d.name as defaultConstraint,
 	m.definition as computedExpression, m.is_persisted as isPersisted, c.column_id as columnId, 
 	-- TODO only if version >= 2008
@@ -8,6 +9,8 @@ select c.name as columnName, t.name as dataType, c.object_id as objectId, c.is_i
 from sys.columns c
 inner join sys.types t on c.system_type_id = t.system_type_id and c.user_type_id = t.user_type_id
 inner join sys.objects o on c.object_id = o.object_id
+INNER JOIN sys.schemas u ON u.schema_id=o.schema_id 
+INNER JOIN INFORMATION_SCHEMA.COLUMNS col ON col.TABLE_NAME = o.name AND col.TABLE_SCHEMA = u.name and col.COLUMN_NAME = c.name
 left join sys.default_constraints d on c.default_object_id = d.object_id
 left join sys.computed_columns m on m.object_id = c.object_id and m.column_id = c.column_id
 where o.type = 'U' and o.object_id =[OBJECT_ID_CONDITION]

packages/engines/mssql/sql/viewColumns.js

@@ -6,7 +6,7 @@ select
 	col.COLUMN_NAME as columnName,
 	col.IS_NULLABLE as isNullable,
 	col.DATA_TYPE as dataType,
-	col.CHARACTER_MAXIMUM_LENGTH,
+	col.CHARACTER_MAXIMUM_LENGTH as charMaxLength,
 	col.NUMERIC_PRECISION as precision,
 	col.NUMERIC_SCALE as scale,
 	col.COLUMN_DEFAULT

packages/engines/mysql/index.js

@@ -85,7 +85,7 @@ const driver = {
 
     return query;
   },
-  async readQuery(connection, sql) {
+  async readQuery(connection, sql, structure) {
     const query = connection.query(sql);
     const { stream } = connection._nativeModules;
 
@@ -99,7 +99,7 @@ const driver = {
         console.error(err);
         pass.end();
       })
-      .on('fields', (fields) => pass.write({ columns: extractColumns(fields) }))
+      .on('fields', (fields) => pass.write(structure || { columns: extractColumns(fields) }))
       .on('result', (row) => pass.write(row))
       .on('end', () => pass.end());
 

packages/engines/package.json

@@ -11,6 +11,7 @@
     "typescript": "^3.7.5"
   },
   "dependencies": {
-    "lodash": "^4.17.15"
+    "lodash": "^4.17.15",
+    "@dbgate/tools": "^0.1.0"
   }
 }

packages/tools/src/commonTypeParser.ts

@@ -15,7 +15,7 @@ export function isTypeNumber(dataType) {
 }
 
 export function isTypeString(dataType) {
-  return dataType && /char/i.test(dataType);
+  return dataType && /char|binary/i.test(dataType);
 }
 
 export function isTypeLogical(dataType) {

packages/tools/src/index.ts

@@ -1 +1,2 @@
 export * from './commonTypeParser';
+export * from './nameTools';

packages/types/engines.d.ts

@@ -23,7 +23,7 @@ export interface EngineDriver {
   connect(nativeModules, { server, port, user, password, database }): any;
   query(pool: any, sql: string): Promise<QueryResult>;
   stream(pool: any, sql: string, options: StreamOptions);
-  readQuery(pool: any, sql: string): Promise<stream.Readable>;
+  readQuery(pool: any, sql: string, structure?: TableInfo): Promise<stream.Readable>;
   writeTable(pool: any, name: NamedObjectInfo, options: WriteTableOptions): Promise<stream.Writeable>;
   analyseSingleObject(
     pool: any,

packages/web/src/impexp/createImpExpScript.js

@@ -4,7 +4,7 @@ import getAsArray from '../utility/getAsArray';
 import { getConnectionInfo } from '../utility/metadataLoaders';
 import engines from '@dbgate/engines';
 import { findObjectLike } from '@dbgate/datalib';
-import { quoteFullName, fullNameFromString } from '@dbgate/datalib';
+import { quoteFullName, fullNameFromString } from '@dbgate/tools';
 
 export function getTargetName(source, values) {
   const key = `targetName_${source}`;
@@ -37,11 +37,10 @@ function getSourceExpr(sourceName, values, sourceConnection, sourceDriver) {
   if (values.sourceStorageType == 'database') {
     const fullName = { schemaName: values.sourceSchemaName, pureName: sourceName };
     return [
-      'queryReader',
+      'tableReader',
       {
         connection: sourceConnection,
-        // @ts-ignore
+        ...fullName,
-        sql: `select * from ${quoteFullName(sourceDriver.dialect, fullName)}`,
       },
     ];
   }