This commit is contained in:
Nybkox
2025-04-01 17:32:24 +02:00
parent f2c109116c
commit 83f69d89ff
35 changed files with 1096 additions and 75 deletions

View File

@@ -0,0 +1,374 @@
/**
* @typedef {object} DuckDbStringList
* @property {string[]} items
*/
/**
* @typedef {object} DuckDbColumnRow
* @property {number | null} numeric_scale
* @property {number | null} numeric_precision_radix
* @property {number | null} numeric_precision
* @property {number | null} character_maximum_length
* @property {string | null} data_type_id
* @property {string} data_type
* @property {boolean} is_nullable
* @property {string | null} column_default
* @property {boolean} internal
* @property {string | null} comment
* @property {number} column_index
* @property {string} column_name
* @property {string} table_oid
* @property {string} table_name
* @property {string} schema_oid
* @property {string} schema_name
* @property {string} database_oid
* @property {string} database_name
*/
/**
* @typedef {object} DuckDbConstraintRow
* @property {DuckDbStringList} referenced_column_names
* @property {string | null} referenced_table
* @property {string | null} constraint_name
* @property {DuckDbStringList} constraint_column_names
* @property {DuckDbStringList} constraint_column_indexes
* @property {string | null} expression
* @property {string | null} constraint_text
* @property {string} constraint_type
* @property {string} constraint_index
* @property {string} table_oid
* @property {string} table_name
* @property {string} schema_oid
* @property {string} schema_name
* @property {string} database_oid
* @property {string} database_name
*/
/**
* @typedef {object} DuckDbTableRow
* @property {string | null} sql
* @property {string} check_constraint_count
* @property {string} index_count
* @property {string} column_count
* @property {string} estimated_size
* @property {boolean} has_primary_key
* @property {boolean} temporary
* @property {boolean} internal
* @property {{ entries: Array<any> }} tags
* @property {string | null} comment
* @property {string} table_oid
* @property {string} table_name
* @property {string} schema_oid
* @property {string} schema_name
* @property {string} database_oid
* @property {string} database_name
*/
/**
* Represents a single row returned from the duckdb_views() function.
* Note: Assumes OIDs and counts are represented as strings based on previous examples.
*
* @typedef {object} DuckDbViewRow
* @property {string} database_name
* @property {string} database_oid
* @property {string} schema_name
* @property {string} schema_oid
* @property {string} view_name
* @property {string} view_oid
* @property {string | null} comment
* @property {{ [key: string]: string } | null} tags
* @property {boolean} internal
* @property {boolean} temporary
* @property {string} column_count
* @property {string | null} sql
*/
/**
* @param {DuckDbViewRow} duckDbViewRow
* @returns {import("dbgate-types").ViewInfo}
*/
function mapViewRowToViewInfo(duckDbViewRow) {
const viewInfo = {
pureName: duckDbViewRow.view_name,
schemaName: duckDbViewRow.schema_name,
objectId: duckDbViewRow.view_oid,
objectTypeField: 'view',
columns: [],
};
if (duckDbViewRow.comment != null) {
viewInfo.objectComment = duckDbViewRow.comment;
}
if (duckDbViewRow.sql != null) {
viewInfo.createSql = duckDbViewRow.sql;
}
return /** @type {import("dbgate-types").ViewInfo} */ (viewInfo);
}
/**
* @param {DuckDbTableRow} rawTableData
*/
function mapRawTableToTableInfo(rawTableData) {
const pureName = rawTableData.table_name;
const schemaName = rawTableData.schema_name;
const objectId = rawTableData.table_oid;
const objectTypeField = 'table';
const objectComment = rawTableData.comment;
return {
pureName: pureName,
schemaName: schemaName,
objectId: objectId,
objectTypeField: objectTypeField,
objectComment: objectComment,
};
}
/**
* @typedef {object} DuckDbColumnDataTypeInfo
* @property {string} data_type
* @property {number | null} numeric_precision
* @property {number | null} numeric_scale
* @property {number | null} character_maximum_length
*/
/**
* @param {DuckDbColumnDataTypeInfo | null | undefined} columnInfo
* @returns {string}
*/
function extractDataType(columnInfo) {
const baseType = columnInfo.data_type.toUpperCase();
const precision = columnInfo.numeric_precision;
const scale = columnInfo.numeric_scale;
const maxLength = columnInfo.character_maximum_length;
switch (baseType) {
case 'DECIMAL':
case 'NUMERIC':
if (typeof precision === 'number' && precision > 0 && typeof scale === 'number' && scale >= 0) {
return `${baseType}(${precision}, ${scale})`;
}
return baseType;
case 'VARCHAR':
case 'CHAR':
console.log('this', maxLength);
if (typeof maxLength === 'number' && maxLength > 0) {
return `${baseType}(${maxLength})`;
}
return baseType;
default:
return baseType;
}
}
/**
* @param {DuckDbColumnRow} duckDbColumnData
*/
function mapRawColumnToColumnInfo(duckDbColumnData) {
const columnInfo = {
pureName: duckDbColumnData.table_name,
schemaName: duckDbColumnData.schema_name,
columnName: duckDbColumnData.column_name,
dataType: extractDataType(duckDbColumnData),
};
columnInfo.notNull = !duckDbColumnData.is_nullable;
if (duckDbColumnData.column_default != null) {
columnInfo.defaultValue = duckDbColumnData.column_default;
}
if (duckDbColumnData.comment != null) {
columnInfo.columnComment = duckDbColumnData.comment;
}
if (duckDbColumnData.numeric_precision != null) {
columnInfo.precision = duckDbColumnData.numeric_precision;
}
if (duckDbColumnData.numeric_scale != null) {
columnInfo.scale = duckDbColumnData.numeric_scale;
}
if (duckDbColumnData.character_maximum_length != null) {
columnInfo.length = duckDbColumnData.character_maximum_length;
}
return columnInfo;
}
/**
* @param {DuckDbConstraintRow} duckDbConstraintData
* @returns {import("dbgate-types").ForeignKeyInfo}
*/
function mapConstraintRowToForeignKeyInfo(duckDbConstraintData) {
if (
!duckDbConstraintData ||
duckDbConstraintData.constraint_type !== 'FOREIGN KEY' ||
duckDbConstraintData.referenced_table == null
) {
return null;
}
const columns = [];
const constraintColumns = duckDbConstraintData.constraint_column_names?.items;
const referencedColumns = duckDbConstraintData.referenced_column_names?.items;
for (let i = 0; i < constraintColumns.length; i++) {
columns.push({
columnName: constraintColumns[i],
refColumnName: referencedColumns[i],
});
}
const foreignKeyInfo = {
pureName: duckDbConstraintData.table_name,
schemaName: duckDbConstraintData.schema_name,
constraintType: 'foreignKey',
columns: columns,
refTableName: duckDbConstraintData.referenced_table,
};
if (duckDbConstraintData.constraint_name != null) {
foreignKeyInfo.constraintName = duckDbConstraintData.constraint_name;
}
return /** @type {import("dbgate-types").ForeignKeyInfo} */ (foreignKeyInfo);
}
/**
* @param {DuckDbConstraintRow} duckDbConstraintData
* @returns {import("dbgate-types").PrimaryKeyInfo}
*/
function mapConstraintRowToPrimaryKeyInfo(duckDbConstraintData) {
const columns = [];
const constraintColumns = duckDbConstraintData.constraint_column_names?.items;
for (let i = 0; i < constraintColumns.length; i++) {
columns.push({
columnName: constraintColumns[i],
});
}
const primaryKeyInfo = {
pureName: duckDbConstraintData.table_name,
schemaName: duckDbConstraintData.schema_name,
constraintType: 'primaryKey',
columns: columns,
};
if (duckDbConstraintData.constraint_name != null) {
primaryKeyInfo.constraintName = duckDbConstraintData.constraint_name;
}
return /** @type {import("dbgate-types").PrimaryKeyInfo} */ (primaryKeyInfo);
}
/**
* @typedef {object} DuckDbConstraintRow
* @property {DuckDbStringList} referenced_column_names
* @property {string | null} referenced_table
* @property {string | null} constraint_name
* @property {DuckDbStringList} constraint_column_names
* @property {DuckDbStringList} constraint_column_indexes
* @property {string | null} expression
* @property {string | null} constraint_text
* @property {string} constraint_type
* @property {string} constraint_index
* @property {string} table_oid
* @property {string} table_name
* @property {string} schema_oid
* @property {string} schema_name
* @property {string} database_oid
* @property {string} database_name
*/
/**
* Maps a single DuckDbConstraintRow object to a UniqueInfo object if it represents a UNIQUE constraint.
* Assumes UniqueInfo and DuckDbConstraintRow are defined types/interfaces.
* @param {DuckDbConstraintRow} duckDbConstraintData - A single object conforming to DuckDbConstraintRow.
* @returns {import("dbgate-types").UniqueInfo | null} An object structured like UniqueInfo, or null if the input is not a valid UNIQUE constraint.
*/
function mapConstraintRowToUniqueInfo(duckDbConstraintData) {
if (!duckDbConstraintData || duckDbConstraintData.constraint_type !== 'UNIQUE') {
return null;
}
const columns = [];
const constraintColumns = duckDbConstraintData.constraint_column_names?.items;
if (Array.isArray(constraintColumns) && constraintColumns.length > 0) {
for (let i = 0; i < constraintColumns.length; i++) {
columns.push({
columnName: constraintColumns[i],
});
}
} else {
return null;
}
const uniqueInfo = {
pureName: duckDbConstraintData.table_name,
schemaName: duckDbConstraintData.schema_name,
constraintType: 'unique',
columns: columns,
};
if (duckDbConstraintData.constraint_name != null) {
uniqueInfo.constraintName = duckDbConstraintData.constraint_name;
}
return /** @type {import("dbgate-types").UniqueInfo} */ (uniqueInfo);
}
/**
* @typedef {object} DuckDbIndexRow
* @property {string} database_name
* @property {string} database_oid
* @property {string} schema_name
* @property {string} schema_oid
* @property {string} index_name
* @property {string} index_oid
* @property {string} table_name
* @property {string} table_oid
* @property {string | null} comment
* @property {{ [key: string]: string } | null} tags
* @property {boolean} is_unique
* @property {boolean} is_primary
* @property {string | null} expressions
* @property {string | null} sql
*/
/**
* @param {DuckDbIndexRow} duckDbIndexRow
* @returns {import("dbgate-types").IndexInfo}
*/
function mapIndexRowToIndexInfo(duckDbIndexRow) {
const indexInfo = {
pureName: duckDbIndexRow.table_name,
schemaName: duckDbIndexRow.schema_name,
constraintType: 'index',
columns: [],
isUnique: duckDbIndexRow.is_unique,
};
if (duckDbIndexRow.index_name != null) {
indexInfo.constraintName = duckDbIndexRow.index_name;
}
return /** @type {import("dbgate-types").IndexInfo} */ (indexInfo);
}
module.exports = {
mapRawTableToTableInfo,
mapRawColumnToColumnInfo,
mapConstraintRowToForeignKeyInfo,
mapConstraintRowToPrimaryKeyInfo,
mapConstraintRowToUniqueInfo,
mapViewRowToViewInfo,
mapIndexRowToIndexInfo,
};

View File

@@ -0,0 +1,85 @@
const _ = require('lodash');
const { DatabaseAnalyser } = require('dbgate-tools');
const sql = require('./sql');
const {
mapRawTableToTableInfo,
mapRawColumnToColumnInfo,
mapConstraintRowToForeignKeyInfo: mapDuckDbFkConstraintToForeignKeyInfo,
mapConstraintRowToPrimaryKeyInfo,
mapIndexRowToIndexInfo,
mapConstraintRowToUniqueInfo,
mapViewRowToViewInfo,
} = require('./Analyser.helpers');
class Analyser extends DatabaseAnalyser {
constructor(dbhan, driver, version) {
super(dbhan, driver, version);
}
async _computeSingleObjectId() {
const { pureName } = this.singleObjectFilter;
this.singleObjectId = pureName;
}
async _getFastSnapshot() {
const tablesResult = await this.driver.query(this.dbhan, sql.tables);
const columnsResult = await this.driver.query(this.dbhan, sql.columns);
const foreignKeysResult = await this.driver.query(this.dbhan, sql.foreignKeys);
const primaryKeysResult = await this.driver.query(this.dbhan, sql.primaryKeys);
const uniquesResults = await this.driver.query(this.dbhan, sql.uniques);
const indexesResult = await this.driver.query(this.dbhan, sql.indexes);
const viewsResult = await this.driver.query(this.dbhan, sql.views);
/**
* @type {import('dbgate-types').ForeignKeyInfo[]}
*/
const foreignKeys = foreignKeysResult.rows?.map(mapDuckDbFkConstraintToForeignKeyInfo).filter(Boolean);
/**
* @type {import('dbgate-types').PrimaryKeyInfo[]}
*/
const primaryKeys = primaryKeysResult.rows?.map(mapConstraintRowToPrimaryKeyInfo).filter(Boolean);
/**
* @type {import('dbgate-types').UniqueInfo[]}
*/
const uniques = uniquesResults.rows?.map(mapConstraintRowToUniqueInfo).filter(Boolean);
/**
* @type {import('dbgate-types').IndexInfo[]}
*/
const indexes = indexesResult.rows?.map(mapIndexRowToIndexInfo).filter(Boolean);
const views = viewsResult.rows?.map(mapViewRowToViewInfo);
const columns = columnsResult.rows?.map(mapRawColumnToColumnInfo);
const tables = tablesResult.rows?.map(mapRawTableToTableInfo);
const tablesExtended = tables.map((table) => ({
...table,
columns: columns.filter((x) => x.pureName == table.pureName && x.schemaName == table.schemaName),
foreignKeys: foreignKeys.filter((x) => x.pureName == table.pureName && x.schemaName == table.schemaName),
primaryKey: primaryKeys.find((x) => x.pureName == table.pureName && x.schemaName == table.schemaName),
indexes: indexes.filter((x) => x.pureName == table.pureName && x.schemaName == table.schemaName),
uniques: uniques.filter((x) => x.pureName == table.pureName && x.schemaName == table.schemaName),
}));
const viewsExtended = views.map((view) => ({
...view,
columns: columns.filter((x) => x.pureName == view.pureName && x.schemaName == view.schemaName),
}));
return {
tables: tablesExtended,
views: viewsExtended,
};
}
async _runAnalysis() {
const structure = await this._getFastSnapshot();
return structure;
throw new Error('Not implemented');
return this._getFastSnapshot();
}
}
module.exports = Analyser;

View File

@@ -0,0 +1,165 @@
const Analyser = require('./Analyser');
const Dumper = require('../frontend/Dumper');
const driverBase = require('../frontend/driver');
const { getLogger, extractErrorLogData } = require('dbgate-tools');
const { getColumnsInfo, serializeRow, normalizeRow } = require('./helpers');
const logger = getLogger('sqliteDriver');
/**
* @type {import('@duckdb/node-api')}
*/
let duckDb;
function getDuckDb() {
if (!duckDb) {
duckDb = require('@duckdb/node-api');
}
return duckDb;
}
let fileToCon = {};
async function getConnection(file) {
if (fileToCon[file]) {
fileToCon[file].close();
}
const duckDb = getDuckDb();
const instance = await duckDb.DuckDBInstance.create(file);
console.log('DuckDB instance created', instance);
const connection = await instance.connect();
fileToCon[file] = connection;
return fileToCon[file];
}
/** @type {import('dbgate-types').EngineDriver<import('@duckdb/node-api').DuckDBConnection>} */
const driver = {
...driverBase,
analyserClass: Analyser,
async connect({ databaseFile, isReadOnly }) {
return {
client: await getConnection(databaseFile),
};
},
async close(dbhan) {
dbhan.client.disconnect();
dbhan.client.close();
},
async query(dbhan, sql, { readonly } = {}) {
const res = await dbhan.client.runAndReadAll(sql);
const rowsObjects = res.getRowObjects();
const columnNames = res.columnNames();
const columnTypes = res.columnTypes();
const columns = getColumnsInfo(columnNames, columnTypes).map(normalizeRow);
const rows = rowsObjects.map(normalizeRow);
return {
rows,
columns,
};
},
async stream(dbhan, sql, options) {
const duckdb = getDuckDb();
const statements = await dbhan.client.extractStatements(sql);
const count = statements.count;
try {
for (let i = 0; i < count; i++) {
let hasSentColumns = false;
const stmt = await statements.prepare(i);
const res = await stmt.runAndReadAll();
const returningStatemetes = [
duckdb.StatementType.SELECT,
duckdb.StatementType.EXPLAIN,
duckdb.StatementType.EXECUTE,
duckdb.StatementType.RELATION,
duckdb.StatementType.LOGICAL_PLAN,
];
if (!returningStatemetes.includes(stmt.statementType)) {
continue;
}
options.info({
message: JSON.stringify(res),
time: new Date(),
severity: 'info',
});
if (!hasSentColumns) {
const columnNames = res.columnNames();
const columnTypes = res.columnTypes();
const columns = getColumnsInfo(columnNames, columnTypes);
options.recordset(columns);
hasSentColumns = true;
}
const rows = res.getRowObjects();
for (const row of rows) {
options.row(normalizeRow(row));
}
}
options.done();
} catch (error) {
logger.error(extractErrorLogData(error), 'Stream error');
const { message, procName } = error;
options.info({
message,
line: 0,
procedure: procName,
time: new Date(),
severity: 'error',
});
options.done();
}
},
async script(dbhan, sql) {
const dmp1 = driver.createDumper();
dmp1.beginTransaction();
await dbhan.client.run(dmp1.s);
const statements = await dbhan.client.extractStatements(sql);
const count = statements.count;
for (let i = 0; i < count; i++) {
const stmt = await statements.prepare(i);
await stmt.run();
}
const dmp2 = driver.createDumper();
dmp2.commitTransaction();
await dbhan.client.run(dmp2.s);
},
async readQueryTask(stmt, pass) {
throw new Error('Not implemented');
},
async readQuery(dbhan, sql, structure) {
throw new Error('Not implemented');
},
async writeTable(dbhan, name, options) {
return createBulkInsertStreamBase(this, stream, dbhan, name, options);
},
async getVersion(dbhan) {
const { rows } = await this.query(dbhan, 'SELECT version() AS version;');
const { version } = rows[0];
return {
version,
versionText: `DuchDB ${version}`,
};
},
};
module.exports = driver;

View File

@@ -0,0 +1,65 @@
/**
* @param {string[} columnNames
* @param {import('@duckdb/node-api').DuckDBType[]} columnTypes
*/
function getColumnsInfo(columnNames, columnTypes) {
const columns = [];
for (let i = columnNames.length - 1; i >= 0; i--) {
columns.push({
columnName: columnNames[i],
dataType: columnTypes[i],
});
}
return columns;
}
function _normalizeValue(value) {
if (value === null) {
return null;
}
if (typeof value === 'bigint') {
return `${value}n`;
}
if (Array.isArray(value)) {
return value.map((item) => _normalizeValue(item));
}
if (typeof value === 'object') {
const normalizedObj = {};
for (const key in value) {
if (Object.hasOwnProperty.call(value, key)) {
normalizedObj[key] = _normalizeValue(value[key]);
}
}
return normalizedObj;
}
return value;
}
/**
* @param {Record<string, import('@duckdb/node-api').DuckDBValue>} obj
*
*/
function normalizeRow(obj) {
if (typeof obj !== 'object' || obj === null || Array.isArray(obj)) {
return _normalizeValue(obj);
}
const normalized = {};
for (const key in obj) {
if (Object.hasOwnProperty.call(obj, key)) {
normalized[key] = _normalizeValue(obj[key]);
}
}
return normalized;
}
module.exports = {
normalizeRow,
getColumnsInfo,
};

View File

@@ -0,0 +1,6 @@
const driver = require('./driver');
module.exports = {
packageName: 'dbgate-plugin-duckdb',
drivers: [driver],
};

View File

@@ -0,0 +1 @@
module.exports = `SELECT * from duckdb_columns() WHERE internal = false`;

View File

@@ -0,0 +1 @@
module.exports = `SELECT * FROM duckdb_constraints() WHERE constraint_type = 'FOREIGN KEY'`;

View File

@@ -0,0 +1,17 @@
const tables = require('./tables.js');
const columns = require('./columns.js');
const foreignKeys = require('./foreignKeys.js');
const primaryKeys = require('./primaryKeys.js');
const indexes = require('./indexes.js');
const uniques = require('./uniques.js');
const views = require('./views.js');
module.exports = {
tables,
columns,
foreignKeys,
primaryKeys,
indexes,
uniques,
views,
};

View File

@@ -0,0 +1 @@
module.exports = `SELECT * FROM duckdb_indexes()`;

View File

@@ -0,0 +1 @@
module.exports = `SELECT * FROM duckdb_constraints() WHERE constraint_type = 'PRIMARY KEY'`;

View File

@@ -0,0 +1 @@
module.exports = `SELECT * from duckdb_tables() WHERE internal = false`;

View File

@@ -0,0 +1 @@
module.exports = `SELECT * FROM duckdb_constraints() WHERE constraint_type = 'UNIQUE'`;

View File

@@ -0,0 +1 @@
module.exports = `SELECT * FROM duckdb_views() WHERE internal = false`;

View File

@@ -0,0 +1,7 @@
const { SqlDumper, arrayToHexString } = require('dbgate-tools');
class Dumper extends SqlDumper {
autoIncrement() {}
}
module.exports = Dumper;

View File

@@ -0,0 +1,72 @@
// @ts-check
const { driverBase } = global.DBGATE_PACKAGES['dbgate-tools'];
const Dumper = require('./Dumper');
const { sqliteSplitterOptions, noSplitSplitterOptions } = require('dbgate-query-splitter/lib/options');
/**
* @param {string} databaseFile
*/
function getDatabaseFileLabel(databaseFile) {
if (!databaseFile) return databaseFile;
const m = databaseFile.match(/[\/]([^\/]+)$/);
if (m) return m[1];
return databaseFile;
}
/** @type {import('dbgate-types').SqlDialect} */
const dialect = {
limitSelect: true,
rangeSelect: true,
offsetFetchRangeSyntax: false,
explicitDropConstraint: true,
stringEscapeChar: "'",
fallbackDataType: 'nvarchar',
allowMultipleValuesInsert: true,
dropColumnDependencies: ['indexes', 'primaryKey', 'uniques'],
quoteIdentifier(s) {
return `"${s}"`;
},
anonymousPrimaryKey: true,
requireStandaloneSelectForScopeIdentity: true,
createColumn: true,
dropColumn: true,
createIndex: true,
dropIndex: true,
createForeignKey: false,
enableForeignKeyChecks: false,
dropForeignKey: false,
createPrimaryKey: false,
dropPrimaryKey: false,
dropReferencesWhenDropTable: false,
filteredIndexes: true,
anonymousForeignKey: true,
};
/** @type {import('dbgate-types').EngineDriver} */
const driver = {
...driverBase,
dumperClass: Dumper,
dialect,
engine: 'duckdb@dbgate-plugin-duckdb',
title: 'DuckDB',
readOnlySessions: true,
supportsTransactions: true,
getQuerySplitterOptions: (usage) =>
usage == 'editor'
? { ...sqliteSplitterOptions, ignoreComments: true, preventSingleLineSplit: true }
: usage == 'stream'
? noSplitSplitterOptions
: sqliteSplitterOptions,
showConnectionTab: (field) => false,
showConnectionField: (field) => ['databaseFile'].includes(field),
beforeConnectionSave: (connection) => ({
...connection,
singleDatabase: true,
defaultDatabase: getDatabaseFileLabel(connection.databaseFile),
}),
};
module.exports = driver;

View File

@@ -0,0 +1,6 @@
import driver from './driver';
export default {
packageName: 'dbgate-plugin-duckdb',
drivers: [driver],
};