SYNC: Merge pull request #7 from dbgate/feature/applog

This commit is contained in:
Jan Prochazka
2025-08-05 17:13:33 +02:00
committed by Diflow
parent ac0aebd751
commit a96f1d0b49
83 changed files with 1014 additions and 242 deletions

View File

@@ -13,9 +13,9 @@ class Analyser extends DatabaseAnalyser {
}
async _runAnalysis() {
this.feedback({ analysingMessage: 'Loading tables' });
this.feedback({ analysingMessage: 'DBGM-00177 Loading tables' });
const tables = await this.analyserQuery('tables', ['tables']);
this.feedback({ analysingMessage: 'Loading columns' });
this.feedback({ analysingMessage: 'DBGM-00178 Loading columns' });
const columns = await this.analyserQuery('columns', ['tables']);
// this.feedback({ analysingMessage: 'Loading views' });
// const views = await this.analyserQuery('views', ['views']);

View File

@@ -29,11 +29,11 @@ class Analyser extends DatabaseAnalyser {
}
async _runAnalysis() {
this.feedback({ analysingMessage: 'Loading tables' });
this.feedback({ analysingMessage: 'DBGM-00181 Loading tables' });
const tables = await this.analyserQuery('tables', ['tables']);
this.feedback({ analysingMessage: 'Loading columns' });
this.feedback({ analysingMessage: 'DBGM-00182 Loading columns' });
const columns = await this.analyserQuery('columns', ['tables', 'views']);
this.feedback({ analysingMessage: 'Loading views' });
this.feedback({ analysingMessage: 'DBGM-00183 Loading views' });
let views = await this.analyserQuery('views', ['views']);
if (views?.isError) {
views = await this.analyserQuery('viewsNoDefinition', ['views']);

View File

@@ -27,7 +27,7 @@ class CsvPrepareStream extends stream.Transform {
}
async function writer({ fileName, encoding = 'utf-8', header = true, delimiter, quoted }) {
logger.info(`Writing file ${fileName}`);
logger.info(`DBGM-00133 Writing file ${fileName}`);
const csvPrepare = new CsvPrepareStream({ header });
const csvStream = csv.stringify({ delimiter, quoted });
const fileStream = fs.createWriteStream(fileName, encoding);

View File

@@ -75,7 +75,7 @@ async function reader({ fileName, encoding = 'ISO-8859-1', includeDeletedRecords
pass.end();
} catch (error) {
// If any error occurs, destroy the stream with the error
logger.error(extractErrorLogData(error), 'Error reading DBF file');
logger.error(extractErrorLogData(error), 'DBGM-00187 Error reading DBF file');
pass.end();
}
})();

View File

@@ -114,7 +114,7 @@ const driver = {
options.done();
} catch (error) {
logger.error(extractErrorLogData(error), 'Stream error');
logger.error(extractErrorLogData(error), 'DBGM-00188 Stream error');
const { message, procName } = error;
options.info({
message,
@@ -206,7 +206,7 @@ const driver = {
pass.end();
return pass;
} catch (error) {
logger.error(extractErrorLogData(error), 'ReadQuery error');
logger.error(extractErrorLogData(error), 'DBGM-00189 ReadQuery error');
const { message, procName } = error;
pass.write({
__isStreamInfo: true,

View File

@@ -97,7 +97,7 @@ const driver = {
options.done();
} catch (err) {
logger.error(extractErrorLogData(err), 'Stream error');
logger.error(extractErrorLogData(err), 'DBGM-00190 Stream error');
options.info({
message: err.message,
line: err.line,
@@ -199,8 +199,8 @@ const driver = {
await new Promise((resolve, reject) => {
transactionPromise.query(currentSql, function (err, result) {
if (err) {
logger.error(extractErrorLogData(err), 'Error executing SQL in transaction');
logger.error({ sql: currentSql }, 'SQL that caused the error');
logger.error(extractErrorLogData(err), 'DBGM-00191 Error executing SQL in transaction');
logger.error({ sql: currentSql }, 'DBGM-00192 SQL that caused the error');
return reject(err);
}
resolve(result);
@@ -211,19 +211,19 @@ const driver = {
await new Promise((resolve, reject) => {
transactionPromise.commit(function (err) {
if (err) {
logger.error(extractErrorLogData(err), 'Error committing transaction');
logger.error(extractErrorLogData(err), 'DBGM-00193 Error committing transaction');
return reject(err);
}
resolve();
});
});
} catch (error) {
logger.error(extractErrorLogData(error), 'Transaction error');
logger.error(extractErrorLogData(error), 'DBGM-00194 Transaction error');
if (transactionPromise) {
await new Promise((resolve, reject) => {
transactionPromise.rollback(function (rollbackErr) {
if (rollbackErr) {
logger.error(extractErrorLogData(rollbackErr), 'Error rolling back transaction');
logger.error(extractErrorLogData(rollbackErr), 'DBGM-00195 Error rolling back transaction');
return reject(rollbackErr); // Re-reject the rollback error
}
resolve();

View File

@@ -32,15 +32,15 @@ function createBulkInsertStream(driver, stream, dbhan, name, options) {
writable.checkStructure = async () => {
try {
if (options.dropIfExists) {
logger.info(`Dropping collection ${collectionName}`);
logger.info(`DBGM-00137 Dropping collection ${collectionName}`);
await db.collection(collectionName).drop();
}
if (options.truncate) {
logger.info(`Truncating collection ${collectionName}`);
logger.info(`DBGM-00138 Truncating collection ${collectionName}`);
await db.collection(collectionName).deleteMany({});
}
} catch (err) {
logger.error(extractErrorLogData(err), 'Error during preparing mongo bulk insert collection, stopped');
logger.error(extractErrorLogData(err), 'DBGM-00139 Error during preparing mongo bulk insert collection, stopped');
writable.destroy(err);
}
};
@@ -52,7 +52,7 @@ function createBulkInsertStream(driver, stream, dbhan, name, options) {
await db.collection(collectionName).insertMany(rows);
} catch (err) {
logger.error(extractErrorLogData(err), 'Error bulk insert collection, stopped');
logger.error(extractErrorLogData(err), 'DBGM-00197 Error bulk insert collection, stopped');
writable.destroy(err);
}
};

View File

@@ -75,7 +75,7 @@ function ensureDatabaseInMongoURI(uri, dbName) {
url.pathname = `/${dbName}`;
return url.toString();
} catch (error) {
logger.error('Invalid URI format:', error.message);
logger.error('DBGM-00198 Invalid URI format:', error.message);
return uri;
}
}

View File

@@ -100,24 +100,24 @@ class MsSqlAnalyser extends DatabaseAnalyser {
}
async _runAnalysis() {
this.feedback({ analysingMessage: 'Loading tables' });
this.feedback({ analysingMessage: 'DBGM-00205 Loading tables' });
const tablesRows = await this.analyserQuery('tables', ['tables']);
this.feedback({ analysingMessage: 'Loading columns' });
this.feedback({ analysingMessage: 'DBGM-00206 Loading columns' });
const columnsRows = await this.analyserQuery('columns', ['tables']);
this.feedback({ analysingMessage: 'Loading primary keys' });
this.feedback({ analysingMessage: 'DBGM-00207 Loading primary keys' });
const pkColumnsRows = await this.analyserQuery('primaryKeys', ['tables']);
this.feedback({ analysingMessage: 'Loading foreign keys' });
this.feedback({ analysingMessage: 'DBGM-00208 Loading foreign keys' });
const fkColumnsRows = await this.analyserQuery('foreignKeys', ['tables']);
this.feedback({ analysingMessage: 'Loading indexes' });
this.feedback({ analysingMessage: 'DBGM-00209 Loading indexes' });
const indexesRows = await this.analyserQuery('indexes', ['tables']);
this.feedback({ analysingMessage: 'Loading index columns' });
this.feedback({ analysingMessage: 'DBGM-00210 Loading index columns' });
const indexcolsRows = await this.analyserQuery('indexcols', ['tables']);
this.feedback({ analysingMessage: 'Loading table sizes' });
this.feedback({ analysingMessage: 'DBGM-00211 Loading table sizes' });
const tableSizes = await this.analyserQuery('tableSizes');
const tableSizesDict = _.mapValues(_.keyBy(tableSizes.rows, 'objectId'), 'tableRowCount');
this.feedback({ analysingMessage: 'Loading SQL code' });
this.feedback({ analysingMessage: 'DBGM-00212 Loading SQL code' });
const sqlCodeRows = await this.analyserQuery('loadSqlCode', ['views', 'procedures', 'functions', 'triggers']);
const getCreateSql = row =>
sqlCodeRows.rows
@@ -125,21 +125,21 @@ class MsSqlAnalyser extends DatabaseAnalyser {
.map(x => x.codeText)
.join('');
this.feedback({ analysingMessage: 'Loading views' });
this.feedback({ analysingMessage: 'DBGM-00213 Loading views' });
const viewsRows = await this.analyserQuery('views', ['views']);
this.feedback({ analysingMessage: 'Loading procedures & functions' });
this.feedback({ analysingMessage: 'DBGM-00214 Loading procedures & functions' });
const programmableRows = await this.analyserQuery('programmables', ['procedures', 'functions']);
const procedureParameterRows = await this.analyserQuery('proceduresParameters');
const functionParameterRows = await this.analyserQuery('functionParameters');
this.feedback({ analysingMessage: 'Loading triggers' });
this.feedback({ analysingMessage: 'DBGM-00215 Loading triggers' });
const triggerRows = await this.analyserQuery('triggers');
this.feedback({ analysingMessage: 'Loading view columns' });
this.feedback({ analysingMessage: 'DBGM-00216 Loading view columns' });
const viewColumnRows = await this.analyserQuery('viewColumns', ['views']);
this.feedback({ analysingMessage: 'Finalizing DB structure' });
this.feedback({ analysingMessage: 'DBGM-00217 Finalizing DB structure' });
const tables = tablesRows.rows.map(row => ({
...row,
contentHash: row.modifyDate && row.modifyDate.toISOString(),

View File

@@ -72,7 +72,7 @@ function createTediousBulkInsertStream(driver, stream, dbhan, name, options) {
try {
await runBulkInsertBatch(dbhan, fullName, writable, rows);
} catch (err) {
logger.error(extractErrorLogData(err), 'Error during bulk insert, insert stopped');
logger.error(extractErrorLogData(err), 'DBGM-00199 Error during bulk insert, insert stopped');
// writable.emit('error', err);
writable.destroy(err);
}

View File

@@ -169,7 +169,7 @@ const driver = {
const defaultSchemaRows = await this.query(dbhan, 'SELECT SCHEMA_NAME() as name');
const defaultSchema = defaultSchemaRows.rows[0]?.name;
logger.debug(`Loaded ${rows.length} mssql schemas`);
logger.debug(`DBGM-00140 Loaded ${rows.length} mssql schemas`);
return rows.map(x => ({
...x,

View File

@@ -118,17 +118,17 @@ class Analyser extends DatabaseAnalyser {
}
async _runAnalysis() {
this.feedback({ analysingMessage: 'Loading tables' });
this.feedback({ analysingMessage: 'DBGM-00218 Loading tables' });
const tables = await this.analyserQuery('tables', ['tables']);
this.feedback({ analysingMessage: 'Loading columns' });
this.feedback({ analysingMessage: 'DBGM-00219 Loading columns' });
const columns = await this.analyserQuery('columns', ['tables', 'views']);
this.feedback({ analysingMessage: 'Loading primary keys' });
this.feedback({ analysingMessage: 'DBGM-00220 Loading primary keys' });
const pkColumns = await this.analyserQuery('primaryKeys', ['tables']);
this.feedback({ analysingMessage: 'Loading foreign keys' });
this.feedback({ analysingMessage: 'DBGM-00221 Loading foreign keys' });
const fkColumns = await this.analyserQuery('foreignKeys', ['tables']);
this.feedback({ analysingMessage: 'Loading views' });
this.feedback({ analysingMessage: 'DBGM-00222 Loading views' });
const views = await this.analyserQuery('views', ['views']);
this.feedback({ analysingMessage: 'Loading programmables' });
this.feedback({ analysingMessage: 'DBGM-00223 Loading programmables' });
const programmables = await this.analyserQuery('programmables', ['procedures', 'functions']);
const parameters = await this.analyserQuery('parameters', ['procedures', 'functions']);
@@ -155,20 +155,20 @@ class Analyser extends DatabaseAnalyser {
return acc;
}, {});
this.feedback({ analysingMessage: 'Loading view texts' });
this.feedback({ analysingMessage: 'DBGM-00224 Loading view texts' });
const viewTexts = await this.getViewTexts(views.rows.map(x => x.pureName));
this.feedback({ analysingMessage: 'Loading indexes' });
this.feedback({ analysingMessage: 'DBGM-00225 Loading indexes' });
const indexes = await this.analyserQuery('indexes', ['tables']);
this.feedback({ analysingMessage: 'Loading uniques' });
this.feedback({ analysingMessage: 'DBGM-00226 Loading uniques' });
this.feedback({ analysingMessage: 'Loading triggers' });
this.feedback({ analysingMessage: 'DBGM-00227 Loading triggers' });
const triggers = await this.analyserQuery('triggers');
this.feedback({ analysingMessage: 'Loading scheduler events' });
this.feedback({ analysingMessage: 'DBGM-00228 Loading scheduler events' });
const schedulerEvents = await this.analyserQuery('schedulerEvents');
const uniqueNames = await this.analyserQuery('uniqueNames', ['tables']);
this.feedback({ analysingMessage: 'Finalizing DB structure' });
this.feedback({ analysingMessage: 'DBGM-00229 Finalizing DB structure' });
const res = {
tables: tables.rows.map(table => ({

View File

@@ -138,7 +138,7 @@ const drivers = driverBases.map(driverBase => ({
};
const handleError = error => {
logger.error(extractErrorLogData(error), 'Stream error');
logger.error(extractErrorLogData(error), 'DBGM-00200 Stream error');
const { message } = error;
options.info({
message,

View File

@@ -45,26 +45,26 @@ class Analyser extends DatabaseAnalyser {
}
async _runAnalysis() {
this.feedback({ analysingMessage: 'Loading tables' });
this.feedback({ analysingMessage: 'DBGM-00230 Loading tables' });
const tables = await this.analyserQuery('tableList', ['tables'], { $owner: this.dbhan.database });
this.feedback({ analysingMessage: 'Loading columns' });
this.feedback({ analysingMessage: 'DBGM-00231 Loading columns' });
const columns = await this.analyserQuery('columns', ['tables', 'views'], { $owner: this.dbhan.database });
this.feedback({ analysingMessage: 'Loading primary keys' });
this.feedback({ analysingMessage: 'DBGM-00232 Loading primary keys' });
const pkColumns = await this.analyserQuery('primaryKeys', ['tables'], { $owner: this.dbhan.database });
//let fkColumns = null;
this.feedback({ analysingMessage: 'Loading foreign keys' });
this.feedback({ analysingMessage: 'DBGM-00233 Loading foreign keys' });
const fkColumns = await this.analyserQuery('foreignKeys', ['tables'], { $owner: this.dbhan.database });
this.feedback({ analysingMessage: 'Loading views' });
this.feedback({ analysingMessage: 'DBGM-00234 Loading views' });
const views = await this.analyserQuery('views', ['views'], { $owner: this.dbhan.database });
this.feedback({ analysingMessage: 'Loading materialized views' });
this.feedback({ analysingMessage: 'DBGM-00235 Loading materialized views' });
const matviews = this.driver.dialect.materializedViews
? await this.analyserQuery('matviews', ['matviews'], { $owner: this.dbhan.database })
: null;
this.feedback({ analysingMessage: 'Loading routines' });
this.feedback({ analysingMessage: 'DBGM-00236 Loading routines' });
const routines = await this.analyserQuery('routines', ['procedures', 'functions'], {
$owner: this.dbhan.database,
});
@@ -91,15 +91,15 @@ class Analyser extends DatabaseAnalyser {
return acc;
}, {});
this.feedback({ analysingMessage: 'Loading indexes' });
this.feedback({ analysingMessage: 'DBGM-00237 Loading indexes' });
const indexes = await this.analyserQuery('indexes', ['tables'], { $owner: this.dbhan.database });
this.feedback({ analysingMessage: 'Loading unique names' });
this.feedback({ analysingMessage: 'DBGM-00238 Loading unique names' });
const triggers = await this.analyserQuery('triggers', undefined, { $owner: this.dbhan.database });
this.feedback({ analysingMessage: 'Loading triggers' });
this.feedback({ analysingMessage: 'DBGM-00239 Loading triggers' });
const uniqueNames = await this.analyserQuery('uniqueNames', ['tables'], { $owner: this.dbhan.database });
this.feedback({ analysingMessage: 'Finalizing DB structure' });
this.feedback({ analysingMessage: 'DBGM-00240 Finalizing DB structure' });
const fkColumnsMapped = fkColumns.rows.map(x => ({
pureName: x.pure_name,

View File

@@ -83,26 +83,26 @@ class Analyser extends DatabaseAnalyser {
}
async _runAnalysis() {
this.feedback({ analysingMessage: 'Loading tables' });
this.feedback({ analysingMessage: 'DBGM-00241 Loading tables' });
const tables = await this.analyserQuery(this.driver.dialect.stringAgg ? 'tableModifications' : 'tableList', [
'tables',
]);
this.feedback({ analysingMessage: 'Loading columns' });
this.feedback({ analysingMessage: 'DBGM-00242 Loading columns' });
const columns = await this.analyserQuery('columns', ['tables', 'views']);
this.feedback({ analysingMessage: 'Loading primary keys' });
this.feedback({ analysingMessage: 'DBGM-00243 Loading primary keys' });
const pkColumns = await this.analyserQuery('primaryKeys', ['tables']);
let fkColumns = null;
this.feedback({ analysingMessage: 'Loading foreign key constraints' });
this.feedback({ analysingMessage: 'DBGM-00244 Loading foreign key constraints' });
// const fk_tableConstraints = await this.analyserQuery('fk_tableConstraints', ['tables']);
this.feedback({ analysingMessage: 'Loading foreign key refs' });
this.feedback({ analysingMessage: 'DBGM-00245 Loading foreign key refs' });
const foreignKeys = await this.analyserQuery('foreignKeys', ['tables']);
this.feedback({ analysingMessage: 'Loading foreign key columns' });
this.feedback({ analysingMessage: 'DBGM-00246 Loading foreign key columns' });
const fk_keyColumnUsage = await this.analyserQuery('fk_keyColumnUsage', ['tables']);
// const cntKey = x => `${x.constraint_name}|${x.constraint_schema}`;
@@ -149,52 +149,52 @@ class Analyser extends DatabaseAnalyser {
}
fkColumns = { rows: fkRows };
this.feedback({ analysingMessage: 'Loading views' });
this.feedback({ analysingMessage: 'DBGM-00247 Loading views' });
const views = await this.analyserQuery('views', ['views']);
this.feedback({ analysingMessage: 'Loading materialized views' });
this.feedback({ analysingMessage: 'DBGM-00248 Loading materialized views' });
const matviews = this.driver.dialect.materializedViews ? await this.analyserQuery('matviews', ['matviews']) : null;
this.feedback({ analysingMessage: 'Loading materialized view columns' });
this.feedback({ analysingMessage: 'DBGM-00249 Loading materialized view columns' });
const matviewColumns = this.driver.dialect.materializedViews
? await this.analyserQuery('matviewColumns', ['matviews'])
: null;
this.feedback({ analysingMessage: 'Loading routines' });
this.feedback({ analysingMessage: 'DBGM-00250 Loading routines' });
const routines = await this.analyserQuery('routines', ['procedures', 'functions']);
this.feedback({ analysingMessage: 'Loading routine parameters' });
this.feedback({ analysingMessage: 'DBGM-00251 Loading routine parameters' });
const routineParametersRows = await this.analyserQuery('proceduresParameters');
this.feedback({ analysingMessage: 'Loading indexes' });
this.feedback({ analysingMessage: 'DBGM-00252 Loading indexes' });
const indexes = this.driver.__analyserInternals.skipIndexes
? { rows: [] }
: await this.analyserQuery('indexes', ['tables']);
this.feedback({ analysingMessage: 'Loading index columns' });
this.feedback({ analysingMessage: 'DBGM-00253 Loading index columns' });
const indexcols = this.driver.__analyserInternals.skipIndexes
? { rows: [] }
: await this.analyserQuery('indexcols', ['tables']);
this.feedback({ analysingMessage: 'Loading unique names' });
this.feedback({ analysingMessage: 'DBGM-00254 Loading unique names' });
const uniqueNames = await this.analyserQuery('uniqueNames', ['tables']);
let geometryColumns = { rows: [] };
if (views.rows.find(x => x.pure_name == 'geometry_columns' && x.schema_name == 'public')) {
this.feedback({ analysingMessage: 'Loading geometry columns' });
this.feedback({ analysingMessage: 'DBGM-00255 Loading geometry columns' });
geometryColumns = await this.analyserQuery('geometryColumns', ['tables']);
}
let geographyColumns = { rows: [] };
if (views.rows.find(x => x.pure_name == 'geography_columns' && x.schema_name == 'public')) {
this.feedback({ analysingMessage: 'Loading geography columns' });
this.feedback({ analysingMessage: 'DBGM-00256 Loading geography columns' });
geographyColumns = await this.analyserQuery('geographyColumns', ['tables']);
}
this.feedback({ analysingMessage: 'Loading triggers' });
this.feedback({ analysingMessage: 'DBGM-00257 Loading triggers' });
const triggers = await this.analyserQuery('triggers');
this.feedback({ analysingMessage: 'Finalizing DB structure' });
this.feedback({ analysingMessage: 'DBGM-00258 Finalizing DB structure' });
const columnColumnsMapped = fkColumns.rows.map(x => ({
pureName: x.pure_name,
@@ -387,7 +387,7 @@ class Analyser extends DatabaseAnalyser {
procedures: res.procedures?.length,
functions: res.functions?.length,
},
'Database structured finalized'
'DBGM-00141 Database structured finalized'
);
return res;

View File

@@ -228,7 +228,7 @@ const drivers = driverBases.map(driverBase => ({
});
query.on('error', error => {
logger.error(extractErrorLogData(error), 'Stream error');
logger.error(extractErrorLogData(error), 'DBGM-00201 Stream error');
const { message, position, procName } = error;
let line = null;
if (position) {
@@ -382,7 +382,7 @@ const drivers = driverBases.map(driverBase => ({
const defaultSchemaRows = await this.query(dbhan, 'SELECT current_schema');
const defaultSchema = defaultSchemaRows.rows[0]?.current_schema?.trim();
logger.debug(`Loaded ${schemaRows.rows.length} postgres schemas`);
logger.debug(`DBGM-00142 Loaded ${schemaRows.rows.length} postgres schemas`);
const schemas = schemaRows.rows.map(x => ({
schemaName: x.schema_name,

View File

@@ -81,7 +81,7 @@ const libsqlDriver = {
try {
inTransaction();
} catch (error) {
logger.error(extractErrorLogData(error), 'Stream error');
logger.error(extractErrorLogData(error), 'DBGM-00202 Stream error');
const { message, procName } = error;
options.info({
message,

View File

@@ -76,7 +76,7 @@ const driver = {
try {
inTransaction();
} catch (error) {
logger.error(extractErrorLogData(error), 'Stream error');
logger.error(extractErrorLogData(error), 'DBGM-00203 Stream error');
const { message, procName } = error;
options.info({
message,

View File

@@ -59,7 +59,7 @@ class ParseStream extends stream.Transform {
}
async function reader({ fileName, encoding = 'utf-8', itemElementName }) {
logger.info(`Reading file ${fileName}`);
logger.info(`DBGM-00143 Reading file ${fileName}`);
const fileStream = fs.createReadStream(fileName, encoding);
const parser = new ParseStream({ itemElementName });

View File

@@ -70,7 +70,7 @@ class StringifyStream extends stream.Transform {
}
async function writer({ fileName, encoding = 'utf-8', itemElementName, rootElementName }) {
logger.info(`Writing file ${fileName}`);
logger.info(`DBGM-00144 Writing file ${fileName}`);
const stringify = new StringifyStream({ itemElementName, rootElementName });
const fileStream = fs.createWriteStream(fileName, encoding);
return [stringify, fileStream];