feat: libsql basic support

integration-tests/engines.js

@@ -506,6 +506,14 @@ const sqliteEngine = {
   ],
 };
 
+const libsqlEngine = {
+  ...sqliteEngine,
+  label: 'LibSQL',
+  connection: {
+    engine: 'libsql@dbgate-plugin-sqlite',
+  },
+};
+
 /** @type {import('dbgate-types').TestEngineInfo} */
 const cockroachDbEngine = {
   label: 'CockroachDB',
@@ -644,6 +652,7 @@ const enginesOnCi = [
   postgreSqlEngine,
   sqlServerEngine,
   sqliteEngine,
+  libsqlEngine,
   // cockroachDbEngine,
   clickhouseEngine,
   oracleEngine,

packages/types/dialect.d.ts

@@ -17,6 +17,7 @@ export interface SqlDialect {
   defaultSchemaName?: string;
   enableConstraintsPerTable?: boolean;
   enableAllForeignKeys?: boolean;
+  enableForeignKeyChecks?: boolean;
   requireStandaloneSelectForScopeIdentity?: boolean;
   allowMultipleValuesInsert?: boolean;
 

packages/types/engines.d.ts

@@ -186,6 +186,7 @@ export interface EngineDriver<TClient = any> extends FilterBehaviourProvider {
   beforeConnectionSave?: (values: any) => any;
   databaseUrlPlaceholder?: string;
   defaultAuthTypeName?: string;
+  authTypeFirst?: boolean;
   defaultLocalDataCenter?: string;
   defaultSocketPath?: string;
   authTypeLabel?: string;

packages/web/src/settings/ConnectionDriverFields.svelte

@@ -17,6 +17,7 @@
   import FontIcon from '../icons/FontIcon.svelte';
   import FormDropDownTextField from '../forms/FormDropDownTextField.svelte';
   import { getConnectionLabel } from 'dbgate-tools';
+  import { _t } from '../translations';
 
   export let getDatabaseList;
   export let currentConnection;
@@ -153,6 +154,15 @@
   />
 {/if}
 
+{#if driver?.showConnectionField('authToken', $values, showConnectionFieldArgs)}
+  <FormTextField
+    label={_t('authToken', { defaultMessage: 'Auth token' })}
+    name="authToken"
+    data-testid="ConnectionDriverFields_authToken"
+    disabled={isConnected || disabledFields.includes('authToken')}
+  />
+{/if}
+
 {#if $authTypes && driver?.showConnectionField('authType', $values, showConnectionFieldArgs)}
   {#key $authTypes}
     <FormSelectField

plugins/dbgate-plugin-sqlite/package.json

@@ -40,6 +40,7 @@
     "dbgate-query-splitter": "^4.11.3"
   },
   "optionalDependencies": {
+    "libsql": "0.5.0-pre.6",
     "better-sqlite3": "11.8.1"
   }
 }

plugins/dbgate-plugin-sqlite/src/backend/driver.libsql.js

@@ -0,0 +1,174 @@
+// @ts-check
+const _ = require('lodash');
+const stream = require('stream');
+const sqliteDriver = require('./driver.sqlite');
+const driverBases = require('../frontend/drivers');
+const Analyser = require('./Analyser');
+const { splitQuery, sqliteSplitterOptions } = require('dbgate-query-splitter');
+const { runStreamItem, waitForDrain } = require('./helpers');
+const { getLogger, createBulkInsertStreamBase, extractErrorLogData } = global.DBGATE_PACKAGES['dbgate-tools'];
+
+const logger = getLogger('sqliteDriver');
+
+let libsqlValue;
+function getLibsql() {
+  if (!libsqlValue) {
+    libsqlValue = require('libsql');
+  }
+  return libsqlValue;
+}
+
+function extractColumns(row) {
+  if (!row) return [];
+
+  const columns = Object.keys(row).map((columnName) => ({ columnName }));
+  return columns;
+}
+
+/** @type {import('dbgate-types').EngineDriver<import('libsql').Database>} */
+const libsqlDriver = {
+  ...driverBases[1],
+  analyserClass: Analyser,
+  async connect({ databaseFile, isReadOnly, authToken, databaseUrl, ...rest }) {
+    console.log('connect', databaseFile, isReadOnly, authToken, databaseUrl, rest);
+    const Database = getLibsql();
+    const client = databaseFile
+      ? new Database(databaseFile, { readonly: !!isReadOnly })
+      : new Database(databaseUrl, { authToken, readonly: !!isReadOnly });
+
+    return {
+      client,
+    };
+  },
+  async close(dbhan) {
+    // sqlite close is sync, returns this
+    dbhan.client.close();
+  },
+  // @ts-ignore
+  async query(dbhan, sql) {
+    const stmt = dbhan.client.prepare(sql);
+
+    const rows = stmt.all();
+    const stmtColumns = stmt.columns();
+    const columns = stmtColumns.length > 0 ? stmtColumns : extractColumns(rows[0]);
+
+    return {
+      rows,
+      columns: columns.map((col) => ({
+        columnName: col.name,
+        dataType: col.type,
+      })),
+    };
+  },
+  async stream(dbhan, sql, options) {
+    const sqlSplitted = splitQuery(sql, sqliteSplitterOptions);
+
+    const rowCounter = { count: 0, date: null };
+
+    console.log('#stream', sql);
+    const inTransaction = dbhan.client.transaction(() => {
+      for (const sqlItem of sqlSplitted) {
+        runStreamItem(dbhan, sqlItem, options, rowCounter);
+      }
+
+      if (rowCounter.date) {
+        options.info({
+          message: `${rowCounter.count} rows affected`,
+          time: new Date(),
+          severity: 'info',
+        });
+      }
+    });
+
+    try {
+      inTransaction();
+    } catch (error) {
+      logger.error(extractErrorLogData(error), 'Stream error');
+      const { message, procName } = error;
+      options.info({
+        message,
+        line: 0,
+        procedure: procName,
+        time: new Date(),
+        severity: 'error',
+      });
+    }
+
+    options.done();
+    // return stream;
+  },
+  async script(dbhan, sql) {
+    const inTransaction = dbhan.client.transaction(() => {
+      for (const sqlItem of splitQuery(sql, this.getQuerySplitterOptions('script'))) {
+        const stmt = dbhan.client.prepare(sqlItem);
+        stmt.run();
+      }
+    });
+    inTransaction();
+  },
+
+  async readQueryTask(stmt, pass) {
+    // let sent = 0;
+    for (const row of stmt.iterate()) {
+      // sent++;
+      if (!pass.write(row)) {
+        // console.log('WAIT DRAIN', sent);
+        await waitForDrain(pass);
+      }
+    }
+    pass.end();
+  },
+  async readQuery(dbhan, sql, structure) {
+    const pass = new stream.PassThrough({
+      objectMode: true,
+      highWaterMark: 100,
+    });
+
+    const stmt = dbhan.client.prepare(sql);
+    const columns = stmt.columns();
+
+    pass.write({
+      __isStreamHeader: true,
+      ...(structure || {
+        columns: columns.map((col) => ({
+          columnName: col.name,
+          dataType: col.type,
+        })),
+      }),
+    });
+    this.readQueryTask(stmt, pass);
+
+    return pass;
+  },
+  async writeTable(dbhan, name, options) {
+    return createBulkInsertStreamBase(this, stream, dbhan, name, options);
+  },
+  async getVersion(dbhan) {
+    const { rows } = await this.query(dbhan, 'select sqlite_version() as version');
+    const { version } = rows[0];
+
+    return {
+      version,
+      versionText: `SQLite ${version}`,
+    };
+  },
+
+  getAuthTypes() {
+    const res = [
+      {
+        title: 'File',
+        name: 'file',
+        disabledFields: ['databaseUrl', 'authToken'],
+      },
+      {
+        title: 'URL',
+        name: 'url',
+        disabledFields: ['databaseFile'],
+      },
+    ];
+
+    return res;
+  },
+};
+
+module.exports = libsqlDriver;

plugins/dbgate-plugin-sqlite/src/backend/driver.sqlite.js

@@ -1,9 +1,11 @@
+// @ts-check
 const _ = require('lodash');
 const stream = require('stream');
-const driverBase = require('../frontend/driver');
 const Analyser = require('./Analyser');
+const driverBases = require('../frontend/drivers');
 const { splitQuery, sqliteSplitterOptions } = require('dbgate-query-splitter');
 const { getLogger, createBulkInsertStreamBase, extractErrorLogData } = global.DBGATE_PACKAGES['dbgate-tools'];
+const { runStreamItem, waitForDrain } = require('./helpers');
 
 const logger = getLogger('sqliteDriver');
 
@@ -15,50 +17,9 @@ function getBetterSqlite() {
   return betterSqliteValue;
 }
 
-async function waitForDrain(stream) {
-  return new Promise((resolve) => {
-    stream.once('drain', () => {
-      // console.log('CONTINUE DRAIN');
-      resolve();
-    });
-  });
-}
-
-function runStreamItem(dbhan, sql, options, rowCounter) {
-  const stmt = dbhan.client.prepare(sql);
-  if (stmt.reader) {
-    const columns = stmt.columns();
-    // const rows = stmt.all();
-
-    options.recordset(
-      columns.map((col) => ({
-        columnName: col.name,
-        dataType: col.type,
-      }))
-    );
-
-    for (const row of stmt.iterate()) {
-      options.row(row);
-    }
-  } else {
-    const info = stmt.run();
-    rowCounter.count += info.changes;
-    if (!rowCounter.date) rowCounter.date = new Date().getTime();
-    if (new Date().getTime() > rowCounter.date > 1000) {
-      options.info({
-        message: `${rowCounter.count} rows affected`,
-        time: new Date(),
-        severity: 'info',
-      });
-      rowCounter.count = 0;
-      rowCounter.date = null;
-    }
-  }
-}
-
 /** @type {import('dbgate-types').EngineDriver} */
 const driver = {
-  ...driverBase,
+  ...driverBases[0],
   analyserClass: Analyser,
   async connect({ databaseFile, isReadOnly }) {
     const Database = getBetterSqlite();
@@ -186,6 +147,4 @@ const driver = {
   },
 };
 
-driver.initialize = (dbgateEnv) => {};
-
 module.exports = driver;

plugins/dbgate-plugin-sqlite/src/backend/drivers.js

@@ -0,0 +1,9 @@
+//R@ts-check
+const sqliteDriver = require('./driver.sqlite');
+const libsqlDriver = require('./driver.libsql');
+
+const drivers = [sqliteDriver, libsqlDriver];
+
+drivers.initialize = (dbgateEnv) => {};
+
+module.exports = drivers;

plugins/dbgate-plugin-sqlite/src/backend/helpers.js

@@ -0,0 +1,49 @@
+// @ts-check
+
+function runStreamItem(dbhan, sql, options, rowCounter) {
+  const stmt = dbhan.client.prepare(sql);
+  console.log(stmt);
+  console.log(stmt.reader);
+  if (stmt.reader) {
+    const columns = stmt.columns();
+    // const rows = stmt.all();
+
+    options.recordset(
+      columns.map((col) => ({
+        columnName: col.name,
+        dataType: col.type,
+      }))
+    );
+
+    for (const row of stmt.iterate()) {
+      options.row(row);
+    }
+  } else {
+    const info = stmt.run();
+    rowCounter.count += info.changes;
+    if (!rowCounter.date) rowCounter.date = new Date().getTime();
+    if (new Date().getTime() - rowCounter.date > 1000) {
+      options.info({
+        message: `${rowCounter.count} rows affected`,
+        time: new Date(),
+        severity: 'info',
+      });
+      rowCounter.count = 0;
+      rowCounter.date = null;
+    }
+  }
+}
+
+async function waitForDrain(stream) {
+  return new Promise((resolve) => {
+    stream.once('drain', () => {
+      // console.log('CONTINUE DRAIN');
+      resolve();
+    });
+  });
+}
+
+module.exports = {
+  runStreamItem,
+  waitForDrain,
+};

plugins/dbgate-plugin-sqlite/src/backend/index.js

@@ -1,9 +1,9 @@
-const driver = require('./driver');
+const drivers = require('./drivers');
 
 module.exports = {
   packageName: 'dbgate-plugin-sqlite',
-  drivers: [driver],
+  drivers: drivers,
   initialize(dbgateEnv) {
-    driver.initialize(dbgateEnv);
+    drivers.initialize(dbgateEnv);
   },
 };

plugins/dbgate-plugin-sqlite/src/frontend/drivers.js

@@ -1,3 +1,5 @@
+// @ts-check
+
 const { driverBase } = global.DBGATE_PACKAGES['dbgate-tools'];
 const Dumper = require('./Dumper');
 const { sqliteSplitterOptions, noSplitSplitterOptions } = require('dbgate-query-splitter/lib/options');
@@ -69,4 +71,44 @@ const driver = {
   predefinedDataTypes: ['integer', 'real', 'text', 'blob'],
 };
 
-module.exports = driver;
+/** @type {import('dbgate-types').EngineDriver} */
+const libsqlDriver = {
+  ...driverBase,
+  dumperClass: Dumper,
+  dialect,
+  engine: 'libsql@dbgate-plugin-sqlite',
+  title: 'LibSQL',
+  readOnlySessions: true,
+  supportsTransactions: true,
+
+  showConnectionField: (field, values) => {
+    if ((values?.authType ?? 'url') === 'url') {
+      return ['databaseUrl', 'authToken', 'isReadOnly', 'authType'].includes(field);
+    }
+    return ['databaseFile', 'isReadOnly', 'authType'].includes(field);
+  },
+
+  showConnectionTab: (field) => false,
+  defaultAuthTypeName: 'url',
+  authTypeFirst: true,
+
+  beforeConnectionSave: (connection) => ({
+    ...connection,
+    singleDatabase: true,
+    defaultDatabase: getDatabaseFileLabel(connection.databaseFile || connection.databaseUrl),
+  }),
+
+  getQuerySplitterOptions: (usage) =>
+    usage == 'editor'
+      ? { ...sqliteSplitterOptions, ignoreComments: true, preventSingleLineSplit: true }
+      : usage == 'stream'
+      ? noSplitSplitterOptions
+      : sqliteSplitterOptions,
+
+  // isFileDatabase: true,
+  // isElectronOnly: true,
+
+  predefinedDataTypes: ['integer', 'real', 'text', 'blob'],
+};
+
+module.exports = [driver, libsqlDriver];

plugins/dbgate-plugin-sqlite/src/frontend/index.js

@@ -1,6 +1,6 @@
-import driver from './driver';
+import drivers from './drivers';
 
 export default {
   packageName: 'dbgate-plugin-sqlite',
-  drivers: [driver],
+  drivers: drivers,
 };

translations/en.json

@@ -1,6 +1,7 @@
 {
   "app.preparingPlugins": "Preparing plugins ...",
   "app.starting": "Starting DbGate",
+  "authToken": "Auth token",
   "command.datagrid.addNewColumn": "Add new column",
   "command.datagrid.addNewColumn.toolbar": "New column",
   "command.datagrid.cloneRows": "Clone rows",

yarn.lock

@@ -1703,6 +1703,41 @@
   resolved "https://registry.yarnpkg.com/@kurkle/color/-/color-0.3.2.tgz#5acd38242e8bde4f9986e7913c8fdf49d3aa199f"
   integrity sha512-fuscdXJ9G1qb7W8VdHi+IwRqij3lBkosAm4ydQtEmbY58OzHXqQhvlxqEkoz0yssNVn38bcpRWgA9PP+OGoisw==
 
+"@libsql/darwin-arm64@0.5.0-pre.6":
+  version "0.5.0-pre.6"
+  resolved "https://registry.yarnpkg.com/@libsql/darwin-arm64/-/darwin-arm64-0.5.0-pre.6.tgz#1afb2e55aa25bd5fed6b37cbc2cbe9cfcca2e61e"
+  integrity sha512-T99Ap/ui7xqFe9ZjWUWRbSCqh9Bo/uZ/wOFtVi9U/2YlBdG4Vv2A7Uz1USYnivJm0nvyYjcy2N9enaRl2cyKkQ==
+
+"@libsql/darwin-x64@0.5.0-pre.6":
+  version "0.5.0-pre.6"
+  resolved "https://registry.yarnpkg.com/@libsql/darwin-x64/-/darwin-x64-0.5.0-pre.6.tgz#0c410db574310ab32d52ee73ce218c7049d818ea"
+  integrity sha512-09fTHmTrxltuQ4oyM7RCz4qRF1oiZS9uf0IIIOI7do6dQu8830a7rrqhpg33LKBs9eBfKWuzpC6n8SuuatSFOw==
+
+"@libsql/linux-arm64-gnu@0.5.0-pre.6":
+  version "0.5.0-pre.6"
+  resolved "https://registry.yarnpkg.com/@libsql/linux-arm64-gnu/-/linux-arm64-gnu-0.5.0-pre.6.tgz#e220cc86924c8ef6b197033e0c161ef0fe327949"
+  integrity sha512-HDQH42ZxzhPMcFdcARV2I7oH7LK/jk2eqhODtIbnVn0kiklHY98F4wk1rbqFIzJljMuzq9HSycJtntUyubpnWg==
+
+"@libsql/linux-arm64-musl@0.5.0-pre.6":
+  version "0.5.0-pre.6"
+  resolved "https://registry.yarnpkg.com/@libsql/linux-arm64-musl/-/linux-arm64-musl-0.5.0-pre.6.tgz#41ba6db61e1ff64f29602c520d8e61903d2c143a"
+  integrity sha512-v6NmFwkQutzud5ZWbo0BWhfIe4OyfQ1qXq/uihpcLOjPUFyWl5vHelOQn1hflJeQ2PcaYxFQ6XPQimSs1HsqMw==
+
+"@libsql/linux-x64-gnu@0.5.0-pre.6":
+  version "0.5.0-pre.6"
+  resolved "https://registry.yarnpkg.com/@libsql/linux-x64-gnu/-/linux-x64-gnu-0.5.0-pre.6.tgz#2ce956c960eb54fa5a10d89856cf3d36bb9772b7"
+  integrity sha512-IOSlRJWNUoEdtL9Y2RrGJyNR4X9t2aWTcbVkYMRtKfDIqylYO8UXDtMLFdLLnjR4p5yZKnO9OqOkbMko8DKdOw==
+
+"@libsql/linux-x64-musl@0.5.0-pre.6":
+  version "0.5.0-pre.6"
+  resolved "https://registry.yarnpkg.com/@libsql/linux-x64-musl/-/linux-x64-musl-0.5.0-pre.6.tgz#c5e58c067a52effd7ea119c58ca189596b28ae6e"
+  integrity sha512-BGICFHvEKIZtrD4UYjIg7SfGmak6zGRUAp/MVS+40FMe4eh7d6uvmQFs6JAaHErazLEKHLKbIKIYAXe1srHKVQ==
+
+"@libsql/win32-x64-msvc@0.5.0-pre.6":
+  version "0.5.0-pre.6"
+  resolved "https://registry.yarnpkg.com/@libsql/win32-x64-msvc/-/win32-x64-msvc-0.5.0-pre.6.tgz#0fe64844aaef3c5edb1c8e59f949c17ce6cad2ce"
+  integrity sha512-1RUqZ9wURWlHOXvafbnhRe2HGh+B7yfqvUQV3RWzS9c7oh1rJbeO7p0XDFFWyRjN8yYFbjlZDmRUcYr1Lo83qQ==
+
 "@mdi/font@^7.1.96":
   version "7.4.47"
   resolved "https://registry.yarnpkg.com/@mdi/font/-/font-7.4.47.tgz#2ae522867da3a5c88b738d54b403eb91471903af"
@@ -1759,6 +1794,11 @@
     call-me-maybe "^1.0.1"
     glob-to-regexp "^0.3.0"
 
+"@neon-rs/load@^0.0.4":
+  version "0.0.4"
+  resolved "https://registry.yarnpkg.com/@neon-rs/load/-/load-0.0.4.tgz#2a2a3292c6f1fef043f49886712d3c96a547532e"
+  integrity sha512-kTPhdZyTQxB+2wpiRcFWrDcejc4JI6tkPuS7UZCG4l6Zvc5kU/gGQ/ozvHTh1XR5tS+UlfAfGuPajjzQjCiHCw==
+
 "@nodelib/fs.scandir@2.1.5":
   version "2.1.5"
   resolved "https://registry.yarnpkg.com/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz#7619c2eb21b25483f6d167548b4cfd5a7488c3d5"
@@ -4556,6 +4596,11 @@ detect-indent@^6.0.0:
   resolved "https://registry.yarnpkg.com/detect-indent/-/detect-indent-6.1.0.tgz#592485ebbbf6b3b1ab2be175c8393d04ca0d57e6"
   integrity sha512-reYkTUJAZb9gUuZ2RvVCNhVHdg62RHnJ7WJl8ftMi4diZ6NWlciOzQN88pUhSELEwflJht4oQDv0F0BMlwaYtA==
 
+detect-libc@2.0.2:
+  version "2.0.2"
+  resolved "https://registry.yarnpkg.com/detect-libc/-/detect-libc-2.0.2.tgz#8ccf2ba9315350e1241b88d0ac3b0e1fbd99605d"
+  integrity sha512-UX6sGumvvqSaXgdKGUsgZWqcUyIXZ/vZTrlRT/iobiKhGL0zL4d3osHj3uqllWJK+i+sixDS/3COVEOFbupFyw==
+
 detect-libc@^2.0.0:
   version "2.0.3"
   resolved "https://registry.yarnpkg.com/detect-libc/-/detect-libc-2.0.3.tgz#f0cd503b40f9939b894697d19ad50895e30cf700"
@@ -8276,6 +8321,22 @@ levn@^0.3.0, levn@~0.3.0:
     prelude-ls "~1.1.2"
     type-check "~0.3.2"
 
+libsql@0.5.0-pre.6:
+  version "0.5.0-pre.6"
+  resolved "https://registry.yarnpkg.com/libsql/-/libsql-0.5.0-pre.6.tgz#23e2b89977738baad10900104d3551d33fccc5b0"
+  integrity sha512-TvugJnL32QiZCvpu6Eh/uh2RjpzsxprqVd2hWygHFUK5abcotaRWYrHmuygXFe43QmsJrmYjN0DAKLqoiy8d2w==
+  dependencies:
+    "@neon-rs/load" "^0.0.4"
+    detect-libc "2.0.2"
+  optionalDependencies:
+    "@libsql/darwin-arm64" "0.5.0-pre.6"
+    "@libsql/darwin-x64" "0.5.0-pre.6"
+    "@libsql/linux-arm64-gnu" "0.5.0-pre.6"
+    "@libsql/linux-arm64-musl" "0.5.0-pre.6"
+    "@libsql/linux-x64-gnu" "0.5.0-pre.6"
+    "@libsql/linux-x64-musl" "0.5.0-pre.6"
+    "@libsql/win32-x64-msvc" "0.5.0-pre.6"
+
 lie@3.1.1:
   version "3.1.1"
   resolved "https://registry.yarnpkg.com/lie/-/lie-3.1.1.tgz#9a436b2cc7746ca59de7a41fa469b3efb76bd87e"