Merge pull request #1069 from dbgate/feature/libsql

feat: libsql basic support
This commit is contained in:
Jan Prochazka
2025-03-14 14:45:14 +01:00
committed by GitHub
17 changed files with 432 additions and 79 deletions

View File

@@ -27,11 +27,11 @@ services:
# - MYSQL_ROOT_PASSWORD=Pwd2020Db
#
cassandradb:
image: cassandra:5.0.2
ports:
- 15942:9042
# cassandradb:
# image: cassandra:5.0.2
# ports:
# - 15942:9042
#
# clickhouse:
# image: bitnami/clickhouse:24.8.4
# restart: always
@@ -55,28 +55,37 @@ services:
# ports:
# - 15003:26257
# command: start-single-node --insecure
# mongodb:
# image: mongo:4.0.12
# restart: always
# volumes:
# - mongo-data:/data/db
# - mongo-config:/data/configdb
# ports:
# - 27017:27017
# mongodb:
# image: mongo:4.0.12
# restart: always
# volumes:
# - mongo-data:/data/db
# - mongo-config:/data/configdb
# ports:
# - 27017:27017
# cockroachdb-init:
# image: cockroachdb/cockroach
# # build: cockroach
# # entrypoint: /cockroach/init.sh
# entrypoint: ./cockroach sql --insecure --host="cockroachdb" --execute="CREATE DATABASE IF NOT EXISTS test;"
# cockroachdb-init:
# image: cockroachdb/cockroach
# # build: cockroach
# # entrypoint: /cockroach/init.sh
# entrypoint: ./cockroach sql --insecure --host="cockroachdb" --execute="CREATE DATABASE IF NOT EXISTS test;"
# depends_on:
# - cockroachdb
# restart: on-failure
# depends_on:
# - cockroachdb
# restart: on-failure
oracle:
image: gvenzl/oracle-xe:21-slim
environment:
ORACLE_PASSWORD: Pwd2020Db
# oracle:
# image: gvenzl/oracle-xe:21-slim
# environment:
# ORACLE_PASSWORD: Pwd2020Db
# ports:
# - 15006:1521
libsql:
image: ghcr.io/tursodatabase/libsql-server:latest
platform: linux/amd64
ports:
- 15006:1521
- '8080:8080'
- '5002:5001'
volumes:
- ./data/libsql:/var/lib/sqld

View File

@@ -506,6 +506,23 @@ const sqliteEngine = {
],
};
const libsqlFileEngine = {
...sqliteEngine,
label: 'LibSQL FILE',
connection: {
engine: 'libsql@dbgate-plugin-sqlite',
},
};
const libsqlWsEngine = {
...sqliteEngine,
label: 'LibSQL WS',
connection: {
engine: 'libsql@dbgate-plugin-sqlite',
databaseUrl: 'ws://localhost:8080',
},
};
/** @type {import('dbgate-types').TestEngineInfo} */
const cockroachDbEngine = {
label: 'CockroachDB',
@@ -644,6 +661,8 @@ const enginesOnCi = [
postgreSqlEngine,
sqlServerEngine,
sqliteEngine,
libsqlFileEngine,
libsqlWsEngine,
// cockroachDbEngine,
clickhouseEngine,
oracleEngine,
@@ -659,7 +678,9 @@ const enginesOnLocal = [
// sqlServerEngine,
// sqliteEngine,
// cockroachDbEngine,
clickhouseEngine,
// clickhouseEngine,
// libsqlFileEngine,
libsqlWsEngine,
// oracleEngine,
];

View File

@@ -128,7 +128,7 @@ export class ScriptDrivedDeployer {
logger.debug(`Running ${category} script ${file.name}`);
try {
await this.driver.script(this.dbhan, file.text);
await this.driver.script(this.dbhan, file.text, { useTransaction: false });
await this.saveToJournal(file, category, hash);
} catch (err) {
logger.error(extractErrorLogData(err), `Error running ${category} script ${file.name}`);

View File

@@ -17,6 +17,7 @@ export interface SqlDialect {
defaultSchemaName?: string;
enableConstraintsPerTable?: boolean;
enableAllForeignKeys?: boolean;
enableForeignKeyChecks?: boolean;
requireStandaloneSelectForScopeIdentity?: boolean;
allowMultipleValuesInsert?: boolean;

View File

@@ -210,6 +210,7 @@ export interface EngineDriver<TClient = any> extends FilterBehaviourProvider {
beforeConnectionSave?: (values: any) => any;
databaseUrlPlaceholder?: string;
defaultAuthTypeName?: string;
authTypeFirst?: boolean;
defaultLocalDataCenter?: string;
defaultSocketPath?: string;
authTypeLabel?: string;

View File

@@ -17,6 +17,7 @@
import FontIcon from '../icons/FontIcon.svelte';
import FormDropDownTextField from '../forms/FormDropDownTextField.svelte';
import { getConnectionLabel } from 'dbgate-tools';
import { _t } from '../translations';
export let getDatabaseList;
export let currentConnection;
@@ -153,6 +154,15 @@
/>
{/if}
{#if driver?.showConnectionField('authToken', $values, showConnectionFieldArgs)}
<FormTextField
label={_t('authToken', { defaultMessage: 'Auth token' })}
name="authToken"
data-testid="ConnectionDriverFields_authToken"
disabled={isConnected || disabledFields.includes('authToken')}
/>
{/if}
{#if $authTypes && driver?.showConnectionField('authType', $values, showConnectionFieldArgs)}
{#key $authTypes}
<FormSelectField

View File

@@ -40,6 +40,7 @@
"dbgate-query-splitter": "^4.11.3"
},
"optionalDependencies": {
"libsql": "0.5.0-pre.6",
"better-sqlite3": "11.8.1"
}
}

View File

@@ -0,0 +1,181 @@
// @ts-check
const _ = require('lodash');
const stream = require('stream');
const sqliteDriver = require('./driver.sqlite');
const driverBases = require('../frontend/drivers');
const Analyser = require('./Analyser');
const { splitQuery, sqliteSplitterOptions } = require('dbgate-query-splitter');
const { runStreamItem, waitForDrain } = require('./helpers');
const { getLogger, createBulkInsertStreamBase, extractErrorLogData } = global.DBGATE_PACKAGES['dbgate-tools'];
const logger = getLogger('sqliteDriver');
let libsqlValue;
function getLibsql() {
if (!libsqlValue) {
libsqlValue = require('libsql');
}
return libsqlValue;
}
function extractColumns(row) {
if (!row) return [];
const columns = Object.keys(row).map((columnName) => ({ columnName }));
return columns;
}
/** @type {import('dbgate-types').EngineDriver<import('libsql').Database>} */
const libsqlDriver = {
...driverBases[1],
analyserClass: Analyser,
async connect({ databaseFile, isReadOnly, authToken, databaseUrl, ...rest }) {
console.log('connect', databaseFile, isReadOnly, authToken, databaseUrl, rest);
const Database = getLibsql();
const client = databaseFile
? new Database(databaseFile, { readonly: !!isReadOnly })
: new Database(databaseUrl, { authToken, readonly: !!isReadOnly });
return {
client,
};
},
async close(dbhan) {
// sqlite close is sync, returns this
dbhan.client.close();
},
// @ts-ignore
async query(dbhan, sql) {
const stmt = dbhan.client.prepare(sql);
const rows = stmt.all();
const stmtColumns = stmt.columns();
const columns = stmtColumns.length > 0 ? stmtColumns : extractColumns(rows[0]);
return {
rows,
columns: columns.map((col) => ({
columnName: col.name,
dataType: col.type,
})),
};
},
async stream(dbhan, sql, options) {
const sqlSplitted = splitQuery(sql, sqliteSplitterOptions);
const rowCounter = { count: 0, date: null };
console.log('#stream', sql);
const inTransaction = dbhan.client.transaction(() => {
for (const sqlItem of sqlSplitted) {
runStreamItem(dbhan, sqlItem, options, rowCounter);
}
if (rowCounter.date) {
options.info({
message: `${rowCounter.count} rows affected`,
time: new Date(),
severity: 'info',
});
}
});
try {
inTransaction();
} catch (error) {
logger.error(extractErrorLogData(error), 'Stream error');
const { message, procName } = error;
options.info({
message,
line: 0,
procedure: procName,
time: new Date(),
severity: 'error',
});
}
options.done();
// return stream;
},
async script(dbhan, sql, { useTransaction } = {}) {
const runScript = () => {
for (const sqlItem of splitQuery(sql, this.getQuerySplitterOptions('script'))) {
const stmt = dbhan.client.prepare(sqlItem);
stmt.run();
}
};
if (useTransaction) {
dbhan.client.transaction(() => {
runScript();
})();
} else {
runScript();
}
},
async readQueryTask(stmt, pass) {
// let sent = 0;
for (const row of stmt.iterate()) {
// sent++;
if (!pass.write(row)) {
// console.log('WAIT DRAIN', sent);
await waitForDrain(pass);
}
}
pass.end();
},
async readQuery(dbhan, sql, structure) {
const pass = new stream.PassThrough({
objectMode: true,
highWaterMark: 100,
});
const stmt = dbhan.client.prepare(sql);
const columns = stmt.columns();
pass.write({
__isStreamHeader: true,
...(structure || {
columns: columns.map((col) => ({
columnName: col.name,
dataType: col.type,
})),
}),
});
this.readQueryTask(stmt, pass);
return pass;
},
async writeTable(dbhan, name, options) {
return createBulkInsertStreamBase(this, stream, dbhan, name, options);
},
async getVersion(dbhan) {
const { rows } = await this.query(dbhan, 'select sqlite_version() as version');
const { version } = rows[0];
return {
version,
versionText: `SQLite ${version}`,
};
},
getAuthTypes() {
const res = [
{
title: 'File',
name: 'file',
disabledFields: ['databaseUrl', 'authToken'],
},
{
title: 'URL',
name: 'url',
disabledFields: ['databaseFile'],
},
];
return res;
},
};
module.exports = libsqlDriver;

View File

@@ -1,9 +1,11 @@
// @ts-check
const _ = require('lodash');
const stream = require('stream');
const driverBase = require('../frontend/driver');
const Analyser = require('./Analyser');
const driverBases = require('../frontend/drivers');
const { splitQuery, sqliteSplitterOptions } = require('dbgate-query-splitter');
const { getLogger, createBulkInsertStreamBase, extractErrorLogData } = global.DBGATE_PACKAGES['dbgate-tools'];
const { runStreamItem, waitForDrain } = require('./helpers');
const logger = getLogger('sqliteDriver');
@@ -15,50 +17,9 @@ function getBetterSqlite() {
return betterSqliteValue;
}
async function waitForDrain(stream) {
return new Promise((resolve) => {
stream.once('drain', () => {
// console.log('CONTINUE DRAIN');
resolve();
});
});
}
function runStreamItem(dbhan, sql, options, rowCounter) {
const stmt = dbhan.client.prepare(sql);
if (stmt.reader) {
const columns = stmt.columns();
// const rows = stmt.all();
options.recordset(
columns.map((col) => ({
columnName: col.name,
dataType: col.type,
}))
);
for (const row of stmt.iterate()) {
options.row(row);
}
} else {
const info = stmt.run();
rowCounter.count += info.changes;
if (!rowCounter.date) rowCounter.date = new Date().getTime();
if (new Date().getTime() > rowCounter.date > 1000) {
options.info({
message: `${rowCounter.count} rows affected`,
time: new Date(),
severity: 'info',
});
rowCounter.count = 0;
rowCounter.date = null;
}
}
}
/** @type {import('dbgate-types').EngineDriver} */
const driver = {
...driverBase,
...driverBases[0],
analyserClass: Analyser,
async connect({ databaseFile, isReadOnly }) {
const Database = getBetterSqlite();
@@ -186,6 +147,4 @@ const driver = {
},
};
driver.initialize = (dbgateEnv) => {};
module.exports = driver;

View File

@@ -0,0 +1,9 @@
//R@ts-check
const sqliteDriver = require('./driver.sqlite');
const libsqlDriver = require('./driver.libsql');
const drivers = [sqliteDriver, libsqlDriver];
drivers.initialize = (dbgateEnv) => {};
module.exports = drivers;

View File

@@ -0,0 +1,49 @@
// @ts-check
function runStreamItem(dbhan, sql, options, rowCounter) {
const stmt = dbhan.client.prepare(sql);
console.log(stmt);
console.log(stmt.reader);
if (stmt.reader) {
const columns = stmt.columns();
// const rows = stmt.all();
options.recordset(
columns.map((col) => ({
columnName: col.name,
dataType: col.type,
}))
);
for (const row of stmt.iterate()) {
options.row(row);
}
} else {
const info = stmt.run();
rowCounter.count += info.changes;
if (!rowCounter.date) rowCounter.date = new Date().getTime();
if (new Date().getTime() - rowCounter.date > 1000) {
options.info({
message: `${rowCounter.count} rows affected`,
time: new Date(),
severity: 'info',
});
rowCounter.count = 0;
rowCounter.date = null;
}
}
}
async function waitForDrain(stream) {
return new Promise((resolve) => {
stream.once('drain', () => {
// console.log('CONTINUE DRAIN');
resolve();
});
});
}
module.exports = {
runStreamItem,
waitForDrain,
};

View File

@@ -1,9 +1,9 @@
const driver = require('./driver');
const drivers = require('./drivers');
module.exports = {
packageName: 'dbgate-plugin-sqlite',
drivers: [driver],
drivers: drivers,
initialize(dbgateEnv) {
driver.initialize(dbgateEnv);
drivers.initialize(dbgateEnv);
},
};

View File

@@ -1,3 +1,5 @@
// @ts-check
const { driverBase } = global.DBGATE_PACKAGES['dbgate-tools'];
const Dumper = require('./Dumper');
const { sqliteSplitterOptions, noSplitSplitterOptions } = require('dbgate-query-splitter/lib/options');
@@ -69,4 +71,44 @@ const driver = {
predefinedDataTypes: ['integer', 'real', 'text', 'blob'],
};
module.exports = driver;
/** @type {import('dbgate-types').EngineDriver} */
const libsqlDriver = {
...driverBase,
dumperClass: Dumper,
dialect,
engine: 'libsql@dbgate-plugin-sqlite',
title: 'LibSQL',
readOnlySessions: true,
supportsTransactions: true,
showConnectionField: (field, values) => {
if ((values?.authType ?? 'url') === 'url') {
return ['databaseUrl', 'authToken', 'isReadOnly', 'authType'].includes(field);
}
return ['databaseFile', 'isReadOnly', 'authType'].includes(field);
},
showConnectionTab: (field) => false,
defaultAuthTypeName: 'url',
authTypeFirst: true,
beforeConnectionSave: (connection) => ({
...connection,
singleDatabase: true,
defaultDatabase: getDatabaseFileLabel(connection.databaseFile || connection.databaseUrl),
}),
getQuerySplitterOptions: (usage) =>
usage == 'editor'
? { ...sqliteSplitterOptions, ignoreComments: true, preventSingleLineSplit: true }
: usage == 'stream'
? noSplitSplitterOptions
: sqliteSplitterOptions,
// isFileDatabase: true,
// isElectronOnly: true,
predefinedDataTypes: ['integer', 'real', 'text', 'blob'],
};
module.exports = [driver, libsqlDriver];

View File

@@ -1,6 +1,6 @@
import driver from './driver';
import drivers from './drivers';
export default {
packageName: 'dbgate-plugin-sqlite',
drivers: [driver],
drivers: drivers,
};

View File

@@ -1,6 +1,7 @@
{
"app.preparingPlugins": "Preparing plugins ...",
"app.starting": "Starting DbGate",
"authToken": "Auth token",
"command.datagrid.addNewColumn": "Add new column",
"command.datagrid.addNewColumn.toolbar": "New column",
"command.datagrid.cloneRows": "Clone rows",

View File

@@ -111,3 +111,11 @@ jobs:
image: cassandra:5.0.2
ports:
- 15942:9042
libsql:
image: ghcr.io/tursodatabase/libsql-server:latest
platform: linux/amd64
ports:
- '8080:8080'
volumes:
- ./data/libsql:/var/lib/sqld

View File

@@ -1703,6 +1703,41 @@
resolved "https://registry.yarnpkg.com/@kurkle/color/-/color-0.3.2.tgz#5acd38242e8bde4f9986e7913c8fdf49d3aa199f"
integrity sha512-fuscdXJ9G1qb7W8VdHi+IwRqij3lBkosAm4ydQtEmbY58OzHXqQhvlxqEkoz0yssNVn38bcpRWgA9PP+OGoisw==
"@libsql/darwin-arm64@0.5.0-pre.6":
version "0.5.0-pre.6"
resolved "https://registry.yarnpkg.com/@libsql/darwin-arm64/-/darwin-arm64-0.5.0-pre.6.tgz#1afb2e55aa25bd5fed6b37cbc2cbe9cfcca2e61e"
integrity sha512-T99Ap/ui7xqFe9ZjWUWRbSCqh9Bo/uZ/wOFtVi9U/2YlBdG4Vv2A7Uz1USYnivJm0nvyYjcy2N9enaRl2cyKkQ==
"@libsql/darwin-x64@0.5.0-pre.6":
version "0.5.0-pre.6"
resolved "https://registry.yarnpkg.com/@libsql/darwin-x64/-/darwin-x64-0.5.0-pre.6.tgz#0c410db574310ab32d52ee73ce218c7049d818ea"
integrity sha512-09fTHmTrxltuQ4oyM7RCz4qRF1oiZS9uf0IIIOI7do6dQu8830a7rrqhpg33LKBs9eBfKWuzpC6n8SuuatSFOw==
"@libsql/linux-arm64-gnu@0.5.0-pre.6":
version "0.5.0-pre.6"
resolved "https://registry.yarnpkg.com/@libsql/linux-arm64-gnu/-/linux-arm64-gnu-0.5.0-pre.6.tgz#e220cc86924c8ef6b197033e0c161ef0fe327949"
integrity sha512-HDQH42ZxzhPMcFdcARV2I7oH7LK/jk2eqhODtIbnVn0kiklHY98F4wk1rbqFIzJljMuzq9HSycJtntUyubpnWg==
"@libsql/linux-arm64-musl@0.5.0-pre.6":
version "0.5.0-pre.6"
resolved "https://registry.yarnpkg.com/@libsql/linux-arm64-musl/-/linux-arm64-musl-0.5.0-pre.6.tgz#41ba6db61e1ff64f29602c520d8e61903d2c143a"
integrity sha512-v6NmFwkQutzud5ZWbo0BWhfIe4OyfQ1qXq/uihpcLOjPUFyWl5vHelOQn1hflJeQ2PcaYxFQ6XPQimSs1HsqMw==
"@libsql/linux-x64-gnu@0.5.0-pre.6":
version "0.5.0-pre.6"
resolved "https://registry.yarnpkg.com/@libsql/linux-x64-gnu/-/linux-x64-gnu-0.5.0-pre.6.tgz#2ce956c960eb54fa5a10d89856cf3d36bb9772b7"
integrity sha512-IOSlRJWNUoEdtL9Y2RrGJyNR4X9t2aWTcbVkYMRtKfDIqylYO8UXDtMLFdLLnjR4p5yZKnO9OqOkbMko8DKdOw==
"@libsql/linux-x64-musl@0.5.0-pre.6":
version "0.5.0-pre.6"
resolved "https://registry.yarnpkg.com/@libsql/linux-x64-musl/-/linux-x64-musl-0.5.0-pre.6.tgz#c5e58c067a52effd7ea119c58ca189596b28ae6e"
integrity sha512-BGICFHvEKIZtrD4UYjIg7SfGmak6zGRUAp/MVS+40FMe4eh7d6uvmQFs6JAaHErazLEKHLKbIKIYAXe1srHKVQ==
"@libsql/win32-x64-msvc@0.5.0-pre.6":
version "0.5.0-pre.6"
resolved "https://registry.yarnpkg.com/@libsql/win32-x64-msvc/-/win32-x64-msvc-0.5.0-pre.6.tgz#0fe64844aaef3c5edb1c8e59f949c17ce6cad2ce"
integrity sha512-1RUqZ9wURWlHOXvafbnhRe2HGh+B7yfqvUQV3RWzS9c7oh1rJbeO7p0XDFFWyRjN8yYFbjlZDmRUcYr1Lo83qQ==
"@mdi/font@^7.1.96":
version "7.4.47"
resolved "https://registry.yarnpkg.com/@mdi/font/-/font-7.4.47.tgz#2ae522867da3a5c88b738d54b403eb91471903af"
@@ -1759,6 +1794,11 @@
call-me-maybe "^1.0.1"
glob-to-regexp "^0.3.0"
"@neon-rs/load@^0.0.4":
version "0.0.4"
resolved "https://registry.yarnpkg.com/@neon-rs/load/-/load-0.0.4.tgz#2a2a3292c6f1fef043f49886712d3c96a547532e"
integrity sha512-kTPhdZyTQxB+2wpiRcFWrDcejc4JI6tkPuS7UZCG4l6Zvc5kU/gGQ/ozvHTh1XR5tS+UlfAfGuPajjzQjCiHCw==
"@nodelib/fs.scandir@2.1.5":
version "2.1.5"
resolved "https://registry.yarnpkg.com/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz#7619c2eb21b25483f6d167548b4cfd5a7488c3d5"
@@ -4543,6 +4583,11 @@ detect-indent@^6.0.0:
resolved "https://registry.yarnpkg.com/detect-indent/-/detect-indent-6.1.0.tgz#592485ebbbf6b3b1ab2be175c8393d04ca0d57e6"
integrity sha512-reYkTUJAZb9gUuZ2RvVCNhVHdg62RHnJ7WJl8ftMi4diZ6NWlciOzQN88pUhSELEwflJht4oQDv0F0BMlwaYtA==
detect-libc@2.0.2:
version "2.0.2"
resolved "https://registry.yarnpkg.com/detect-libc/-/detect-libc-2.0.2.tgz#8ccf2ba9315350e1241b88d0ac3b0e1fbd99605d"
integrity sha512-UX6sGumvvqSaXgdKGUsgZWqcUyIXZ/vZTrlRT/iobiKhGL0zL4d3osHj3uqllWJK+i+sixDS/3COVEOFbupFyw==
detect-libc@^2.0.0:
version "2.0.3"
resolved "https://registry.yarnpkg.com/detect-libc/-/detect-libc-2.0.3.tgz#f0cd503b40f9939b894697d19ad50895e30cf700"
@@ -8263,6 +8308,22 @@ levn@^0.3.0, levn@~0.3.0:
prelude-ls "~1.1.2"
type-check "~0.3.2"
libsql@0.5.0-pre.6:
version "0.5.0-pre.6"
resolved "https://registry.yarnpkg.com/libsql/-/libsql-0.5.0-pre.6.tgz#23e2b89977738baad10900104d3551d33fccc5b0"
integrity sha512-TvugJnL32QiZCvpu6Eh/uh2RjpzsxprqVd2hWygHFUK5abcotaRWYrHmuygXFe43QmsJrmYjN0DAKLqoiy8d2w==
dependencies:
"@neon-rs/load" "^0.0.4"
detect-libc "2.0.2"
optionalDependencies:
"@libsql/darwin-arm64" "0.5.0-pre.6"
"@libsql/darwin-x64" "0.5.0-pre.6"
"@libsql/linux-arm64-gnu" "0.5.0-pre.6"
"@libsql/linux-arm64-musl" "0.5.0-pre.6"
"@libsql/linux-x64-gnu" "0.5.0-pre.6"
"@libsql/linux-x64-musl" "0.5.0-pre.6"
"@libsql/win32-x64-msvc" "0.5.0-pre.6"
lie@3.1.1:
version "3.1.1"
resolved "https://registry.yarnpkg.com/lie/-/lie-3.1.1.tgz#9a436b2cc7746ca59de7a41fa469b3efb76bd87e"