diff --git a/packages/api/src/controllers/archive.js b/packages/api/src/controllers/archive.js new file mode 100644 index 000000000..4924a4103 --- /dev/null +++ b/packages/api/src/controllers/archive.js @@ -0,0 +1,41 @@ +const fs = require('fs-extra'); +const path = require('path'); +const { archivedir } = require('../utility/directories'); +const socket = require('../utility/socket'); + +module.exports = { + folders_meta: 'get', + async folders() { + const folders = await fs.readdir(archivedir()); + return [ + { + name: 'default', + type: 'jsonl', + }, + ...folders + .filter((x) => x != 'default') + .map((name) => ({ + name, + type: 'jsonl', + })), + ]; + }, + + createFolder_meta: 'post', + async createFolder({ folder }) { + await fs.mkdir(path.join(archivedir(), folder)); + socket.emitChanged('archive-folders-changed'); + return true; + }, + + files_meta: 'get', + async files({ folder }) { + const files = await fs.readdir(path.join(archivedir(), folder)); + return files + .filter((name) => name.endsWith('.jsonl')) + .map((name) => ({ + name: name.slice(0, -'.jsonl'.length), + type: 'jsonl', + })); + }, +}; diff --git a/packages/api/src/controllers/jsldata.js b/packages/api/src/controllers/jsldata.js index fcc6f3171..72defc8b6 100644 --- a/packages/api/src/controllers/jsldata.js +++ b/packages/api/src/controllers/jsldata.js @@ -1,7 +1,7 @@ const path = require('path'); const fs = require('fs'); const lineReader = require('line-reader'); -const { jsldir } = require('../utility/directories'); +const { jsldir, archivedir } = require('../utility/directories'); const socket = require('../utility/socket'); function readFirstLine(file) { @@ -20,6 +20,14 @@ function readFirstLine(file) { }); } +function getJslFileName(jslid) { + const archiveMatch = jslid.match(/^archive:\/\/([^/]+)\/(.*)$/); + if (archiveMatch) { + return path.join(archivedir(), archiveMatch[1], `${archiveMatch[2]}.jsonl`); + } + return path.join(jsldir(), `${jslid}.jsonl`); +} + module.exports = { openedReaders: {}, @@ -57,7 +65,7 @@ module.exports = { // 'OPENING READER, LINES=', // fs.readFileSync(path.join(jsldir(), `${jslid}.jsonl`), 'utf-8').split('\n').length // ); - const file = path.join(jsldir(), `${jslid}.jsonl`); + const file = getJslFileName(jslid); return new Promise((resolve, reject) => lineReader.open(file, (err, reader) => { if (err) reject(err); @@ -93,7 +101,7 @@ module.exports = { getInfo_meta: 'get', async getInfo({ jslid }) { - const file = path.join(jsldir(), `${jslid}.jsonl`); + const file = getJslFileName(jslid); const firstLine = await readFirstLine(file); if (firstLine) return JSON.parse(firstLine); return null; @@ -119,8 +127,9 @@ module.exports = { getStats_meta: 'get', getStats({ jslid }) { - const file = path.join(jsldir(), `${jslid}.jsonl.stats`); - return JSON.parse(fs.readFileSync(file, 'utf-8')); + const file = `${getJslFileName(jslid)}.stats`; + if (fs.existsSync(file)) return JSON.parse(fs.readFileSync(file, 'utf-8')); + return {}; }, async notifyChangedStats(stats) { diff --git a/packages/api/src/main.js b/packages/api/src/main.js index d2bd44e4e..de0ece6bf 100644 --- a/packages/api/src/main.js +++ b/packages/api/src/main.js @@ -20,6 +20,7 @@ const runners = require('./controllers/runners'); const jsldata = require('./controllers/jsldata'); const config = require('./controllers/config'); const files = require('./controllers/files'); +const archive = require('./controllers/archive'); const { rundir } = require('./utility/directories'); @@ -55,6 +56,7 @@ function start(argument = null) { useController(app, '/jsldata', jsldata); useController(app, '/config', config); useController(app, '/files', files); + useController(app, '/archive', archive); if (process.env.PAGES_DIRECTORY) { app.use('/pages', express.static(process.env.PAGES_DIRECTORY)); diff --git a/packages/api/src/shell/archiveWriter.js b/packages/api/src/shell/archiveWriter.js new file mode 100644 index 000000000..f195f2fd9 --- /dev/null +++ b/packages/api/src/shell/archiveWriter.js @@ -0,0 +1,14 @@ +const path = require('path'); +const { archivedir, ensureDirectory } = require('../utility/directories'); +// const socket = require('../utility/socket'); +const jsonLinesWriter = require('./jsonLinesWriter'); + +function archiveWriter({ folderName, fileName }) { + if (folderName == 'default') ensureDirectory(path.join(archivedir(), folderName)); + const jsonlFile = path.join(archivedir(), folderName, `${fileName}.jsonl`); + const res = jsonLinesWriter({ fileName: jsonlFile }); + // socket.emitChanged(`archive-files-changed-${folderName}`); + return res; +} + +module.exports = archiveWriter; diff --git a/packages/api/src/shell/index.js b/packages/api/src/shell/index.js index c30e0a30c..b60af005a 100644 --- a/packages/api/src/shell/index.js +++ b/packages/api/src/shell/index.js @@ -11,6 +11,7 @@ const excelSheetReader = require('./excelSheetReader'); const jsonLinesWriter = require('./jsonLinesWriter'); const jsonLinesReader = require('./jsonLinesReader'); const jslDataReader = require('./jslDataReader'); +const archiveWriter = require('./archiveWriter'); module.exports = { queryReader, @@ -26,4 +27,5 @@ module.exports = { fakeObjectReader, consoleObjectWriter, jslDataReader, + archiveWriter, }; diff --git a/packages/api/src/utility/directories.js b/packages/api/src/utility/directories.js index 067a4562b..a564cfcda 100644 --- a/packages/api/src/utility/directories.js +++ b/packages/api/src/utility/directories.js @@ -2,31 +2,28 @@ const os = require('os'); const path = require('path'); const fs = require('fs'); -let createdDatadir = false; const createDirectories = {}; +const ensureDirectory = (dir) => { + if (!createDirectories[dir]) { + if (!fs.existsSync(dir)) { + console.log(`Creating directory ${dir}`); + fs.mkdirSync(dir); + } + createDirectories[dir] = true; + } +}; + function datadir() { const dir = path.join(os.homedir(), 'dbgate-data'); - if (!createdDatadir) { - if (!fs.existsSync(dir)) { - console.log(`Creating data directory ${dir}`); - fs.mkdirSync(dir); - } - createdDatadir = true; - } + ensureDirectory(dir); return dir; } const dirFunc = (dirname) => () => { const dir = path.join(datadir(), dirname); - if (!createDirectories[dirname]) { - if (!fs.existsSync(dir)) { - console.log(`Creating jsl directory ${dir}`); - fs.mkdirSync(dir); - } - createDirectories[dirname] = true; - } + ensureDirectory(dir); return dir; }; @@ -34,10 +31,13 @@ const dirFunc = (dirname) => () => { const jsldir = dirFunc('jsl'); const rundir = dirFunc('run'); const uploadsdir = dirFunc('uploads'); +const archivedir = dirFunc('archive'); module.exports = { datadir, jsldir, rundir, uploadsdir, + archivedir, + ensureDirectory, }; diff --git a/packages/web/public/icons/archtable.svg b/packages/web/public/icons/archtable.svg new file mode 100644 index 000000000..2607b823d --- /dev/null +++ b/packages/web/public/icons/archtable.svg @@ -0,0 +1,10 @@ + + + + + + + + + + \ No newline at end of file diff --git a/packages/web/src/App.js b/packages/web/src/App.js index 352bfbf80..4dcbe3e0c 100644 --- a/packages/web/src/App.js +++ b/packages/web/src/App.js @@ -8,6 +8,7 @@ import { SavedSqlFilesProvider, OpenedConnectionsProvider, LeftPanelWidthProvider, + CurrentArchiveProvider, } from './utility/globalState'; import { SocketProvider } from './utility/SocketProvider'; import ConnectionsPinger from './utility/ConnectionsPinger'; @@ -24,7 +25,9 @@ function App() { - + + + diff --git a/packages/web/src/appobj/archiveFileAppObject.js b/packages/web/src/appobj/archiveFileAppObject.js new file mode 100644 index 000000000..e01a35c3e --- /dev/null +++ b/packages/web/src/appobj/archiveFileAppObject.js @@ -0,0 +1,39 @@ +import React from 'react'; +import _ from 'lodash'; +import moment from 'moment'; +import { DatabaseIcon, getIconImage, ArchiveTableIcon } from '../icons'; +import { DropDownMenuItem } from '../modals/DropDownMenu'; +import { openNewTab } from '../utility/common'; + +function Menu({ data, setOpenedTabs }) { + const handleDelete = () => { + // setOpenedTabs((tabs) => tabs.filter((x) => x.tabid != data.tabid)); + }; + return ( + <> + Delete + + ); +} + +const archiveFileAppObject = () => ({ fileName, folderName }, { setOpenedTabs }) => { + const key = fileName; + // const Icon = (props) => ; + const Icon = ArchiveTableIcon; + const onClick = () => { + openNewTab(setOpenedTabs, { + title: fileName, + icon: 'archtable.svg', + tooltip: `${folderName}\n${fileName}`, + tabComponent: 'ArchiveFileTab', + props: { + fileName, + folderName, + }, + }); + }; + + return { title: fileName, key, Icon, Menu, onClick }; +}; + +export default archiveFileAppObject; diff --git a/packages/web/src/appobj/archiveFolderAppObject.js b/packages/web/src/appobj/archiveFolderAppObject.js new file mode 100644 index 000000000..06eec909b --- /dev/null +++ b/packages/web/src/appobj/archiveFolderAppObject.js @@ -0,0 +1,27 @@ +import React from 'react'; +import _ from 'lodash'; +import moment from 'moment'; +import { LocalDbIcon, getIconImage } from '../icons'; +import { DropDownMenuItem } from '../modals/DropDownMenu'; + +function Menu({ data, setOpenedTabs }) { + const handleDelete = () => { + // setOpenedTabs((tabs) => tabs.filter((x) => x.tabid != data.tabid)); + }; + return ( + <> + Delete + + ); +} + +const archiveFolderAppObject = () => ({ name }, { setOpenedTabs, currentArchive }) => { + const key = name; + // const Icon = (props) => ; + const Icon = LocalDbIcon; + const isBold = name == currentArchive; + + return { title: name, key, Icon, isBold, Menu }; +}; + +export default archiveFolderAppObject; diff --git a/packages/web/src/icons.js b/packages/web/src/icons.js index 074e8909b..b4215c53e 100644 --- a/packages/web/src/icons.js +++ b/packages/web/src/icons.js @@ -58,6 +58,7 @@ export function ExpandIcon({ export const TableIcon = (props) => getIconImage('table2.svg', props); export const ViewIcon = (props) => getIconImage('view2.svg', props); +export const ArchiveTableIcon = (props) => getIconImage('archtable.svg', props); export const DatabaseIcon = (props) => getIconImage('database.svg', props); export const ServerIcon = (props) => getIconImage('server.svg', props); diff --git a/packages/web/src/impexp/ImportExportConfigurator.js b/packages/web/src/impexp/ImportExportConfigurator.js index 427e0f00f..9d12acdb0 100644 --- a/packages/web/src/impexp/ImportExportConfigurator.js +++ b/packages/web/src/impexp/ImportExportConfigurator.js @@ -9,6 +9,7 @@ import { FormDatabaseSelect, FormTablesSelect, FormSchemaSelect, + FormArchiveFolderSelect, } from '../utility/forms'; import { useConnectionInfo, useDatabaseInfo } from '../utility/metadataLoaders'; import TableControl, { TableColumn } from '../utility/TableControl'; @@ -147,6 +148,7 @@ function SourceTargetConfig({ storageTypeField, connectionIdField, databaseNameField, + archiveFolderField, schemaNameField, tablesField = undefined, engine = undefined, @@ -161,6 +163,7 @@ function SourceTargetConfig({ { value: 'jsonl', label: 'JSON lines file(s)', directions: ['source', 'target'] }, { value: 'excel', label: 'MS Excel file(s)', directions: ['source'] }, { value: 'query', label: 'SQL Query', directions: ['source'] }, + { value: 'archive', label: 'Archive', directions: ['source', 'target'] }, ]; const storageType = values[storageTypeField]; const dbinfo = useDatabaseInfo({ conid: values[connectionIdField], database: values[databaseNameField] }); @@ -231,6 +234,13 @@ function SourceTargetConfig({ )} + {storageType == 'archive' && ( + <> + + + + )} + {isFileStorage(storageType) && direction == 'source' && } ); @@ -270,6 +280,7 @@ export default function ImportExportConfigurator() { storageTypeField="sourceStorageType" connectionIdField="sourceConnectionId" databaseNameField="sourceDatabaseName" + archiveFolderField="sourceArchiveFolder" schemaNameField="sourceSchemaName" tablesField="sourceList" engine={sourceEngine} @@ -279,6 +290,7 @@ export default function ImportExportConfigurator() { storageTypeField="targetStorageType" connectionIdField="targetConnectionId" databaseNameField="targetDatabaseName" + archiveFolderField="targetArchiveFolder" schemaNameField="targetSchemaName" /> diff --git a/packages/web/src/impexp/createImpExpScript.js b/packages/web/src/impexp/createImpExpScript.js index 9bc1c29b7..16c38ac64 100644 --- a/packages/web/src/impexp/createImpExpScript.js +++ b/packages/web/src/impexp/createImpExpScript.js @@ -68,7 +68,7 @@ function getSourceExpr(sourceName, values, sourceConnection, sourceDriver) { if (sourceStorageType == 'jsldata') { return ['jslDataReader', { jslid: values.sourceJslId }]; } - throw new Error(`Unknown storage type: ${sourceStorageType}`); + throw new Error(`Unknown source storage type: ${sourceStorageType}`); } function getFlagsFroAction(action) { @@ -91,7 +91,8 @@ function getFlagsFroAction(action) { } function getTargetExpr(sourceName, values, targetConnection, targetDriver) { - if (values.targetStorageType == 'csv') { + const { targetStorageType } = values; + if (targetStorageType == 'csv') { return [ 'csvWriter', { @@ -99,7 +100,7 @@ function getTargetExpr(sourceName, values, targetConnection, targetDriver) { }, ]; } - if (values.targetStorageType == 'jsonl') { + if (targetStorageType == 'jsonl') { return [ 'jsonLinesWriter', { @@ -107,7 +108,7 @@ function getTargetExpr(sourceName, values, targetConnection, targetDriver) { }, ]; } - if (values.targetStorageType == 'database') { + if (targetStorageType == 'database') { return [ 'tableWriter', { @@ -118,6 +119,17 @@ function getTargetExpr(sourceName, values, targetConnection, targetDriver) { }, ]; } + if (targetStorageType == 'archive') { + return [ + 'archiveWriter', + { + folderName: values.targetArchiveFolder, + fileName: getTargetName(sourceName, values), + }, + ]; + } + + throw new Error(`Unknown target storage type: ${targetStorageType}`); } export default async function createImpExpScript(values, addEditorInfo = true) { diff --git a/packages/web/src/modals/ImportExportModal.js b/packages/web/src/modals/ImportExportModal.js index 8f5716abb..5d7ca4cec 100644 --- a/packages/web/src/modals/ImportExportModal.js +++ b/packages/web/src/modals/ImportExportModal.js @@ -9,7 +9,7 @@ import ModalContent from './ModalContent'; import ImportExportConfigurator from '../impexp/ImportExportConfigurator'; import createImpExpScript from '../impexp/createImpExpScript'; import { openNewTab } from '../utility/common'; -import { useSetOpenedTabs } from '../utility/globalState'; +import { useCurrentArchive, useSetOpenedTabs } from '../utility/globalState'; import RunnerOutputPane from '../query/RunnerOutputPane'; import axios from '../utility/axios'; @@ -41,6 +41,7 @@ function GenerateSctriptButton({ modalState }) { export default function ImportExportModal({ modalState, initialValues }) { const [executeNumber, setExecuteNumber] = React.useState(0); const [runnerId, setRunnerId] = React.useState(null); + const archive = useCurrentArchive(); const handleExecute = async (values) => { const script = await createImpExpScript(values); @@ -57,7 +58,13 @@ export default function ImportExportModal({ modalState, initialValues }) {
Import/Export diff --git a/packages/web/src/tabs/ArchiveFileTab.js b/packages/web/src/tabs/ArchiveFileTab.js new file mode 100644 index 000000000..fbdda6f71 --- /dev/null +++ b/packages/web/src/tabs/ArchiveFileTab.js @@ -0,0 +1,6 @@ +import React from 'react'; +import JslDataGrid from '../sqleditor/JslDataGrid'; + +export default function ArchiveFileTab({ folderName, fileName, tabVisible, toolbarPortalRef, tabid }) { + return ; +} diff --git a/packages/web/src/tabs/index.js b/packages/web/src/tabs/index.js index 5b3d72ab5..262a7ee2d 100644 --- a/packages/web/src/tabs/index.js +++ b/packages/web/src/tabs/index.js @@ -4,6 +4,7 @@ import TableStructureTab from './TableStructureTab'; import QueryTab from './QueryTab'; import ShellTab from './ShellTab'; import InfoPageTab from './InfoPageTab'; +import ArchiveFileTab from './ArchiveFileTab'; export default { TableDataTab, @@ -12,4 +13,5 @@ export default { QueryTab, InfoPageTab, ShellTab, + ArchiveFileTab, }; diff --git a/packages/web/src/utility/forms.js b/packages/web/src/utility/forms.js index 047d375f8..7a3d07d2e 100644 --- a/packages/web/src/utility/forms.js +++ b/packages/web/src/utility/forms.js @@ -1,12 +1,14 @@ import React from 'react'; import styled from 'styled-components'; import Select from 'react-select'; +import Creatable from 'react-select/creatable'; import { TextField, SelectField } from './inputs'; import { Field, useFormikContext } from 'formik'; import FormStyledButton from '../widgets/FormStyledButton'; -import { useConnectionList, useDatabaseList, useDatabaseInfo } from './metadataLoaders'; +import { useConnectionList, useDatabaseList, useDatabaseInfo, useArchiveFolders } from './metadataLoaders'; import useSocket from './SocketProvider'; import getAsArray from './getAsArray'; +import axios from './axios'; export const FormRow = styled.div` display: flex; @@ -85,11 +87,11 @@ export function FormRadioGroupItem({ name, text, value }) { ); } -export function FormReactSelect({ options, name, isMulti = false }) { +export function FormReactSelect({ options, name, isMulti = false, Component = Select, ...other }) { const { setFieldValue, values } = useFormikContext(); return ( -