archive, export into archive

packages/api/src/controllers/archive.js

@@ -0,0 +1,41 @@
+const fs = require('fs-extra');
+const path = require('path');
+const { archivedir } = require('../utility/directories');
+const socket = require('../utility/socket');
+
+module.exports = {
+  folders_meta: 'get',
+  async folders() {
+    const folders = await fs.readdir(archivedir());
+    return [
+      {
+        name: 'default',
+        type: 'jsonl',
+      },
+      ...folders
+        .filter((x) => x != 'default')
+        .map((name) => ({
+          name,
+          type: 'jsonl',
+        })),
+    ];
+  },
+
+  createFolder_meta: 'post',
+  async createFolder({ folder }) {
+    await fs.mkdir(path.join(archivedir(), folder));
+    socket.emitChanged('archive-folders-changed');
+    return true;
+  },
+
+  files_meta: 'get',
+  async files({ folder }) {
+    const files = await fs.readdir(path.join(archivedir(), folder));
+    return files
+      .filter((name) => name.endsWith('.jsonl'))
+      .map((name) => ({
+        name: name.slice(0, -'.jsonl'.length),
+        type: 'jsonl',
+      }));
+  },
+};

packages/api/src/controllers/jsldata.js

@@ -1,7 +1,7 @@
 const path = require('path');
 const fs = require('fs');
 const lineReader = require('line-reader');
-const { jsldir } = require('../utility/directories');
+const { jsldir, archivedir } = require('../utility/directories');
 const socket = require('../utility/socket');
 
 function readFirstLine(file) {
@@ -20,6 +20,14 @@ function readFirstLine(file) {
   });
 }
 
+function getJslFileName(jslid) {
+  const archiveMatch = jslid.match(/^archive:\/\/([^/]+)\/(.*)$/);
+  if (archiveMatch) {
+    return path.join(archivedir(), archiveMatch[1], `${archiveMatch[2]}.jsonl`);
+  }
+  return path.join(jsldir(), `${jslid}.jsonl`);
+}
+
 module.exports = {
   openedReaders: {},
 
@@ -57,7 +65,7 @@ module.exports = {
     //   'OPENING READER, LINES=',
     //   fs.readFileSync(path.join(jsldir(), `${jslid}.jsonl`), 'utf-8').split('\n').length
     // );
-    const file = path.join(jsldir(), `${jslid}.jsonl`);
+    const file = getJslFileName(jslid);
     return new Promise((resolve, reject) =>
       lineReader.open(file, (err, reader) => {
         if (err) reject(err);
@@ -93,7 +101,7 @@ module.exports = {
 
   getInfo_meta: 'get',
   async getInfo({ jslid }) {
-    const file = path.join(jsldir(), `${jslid}.jsonl`);
+    const file = getJslFileName(jslid);
     const firstLine = await readFirstLine(file);
     if (firstLine) return JSON.parse(firstLine);
     return null;
@@ -119,8 +127,9 @@ module.exports = {
 
   getStats_meta: 'get',
   getStats({ jslid }) {
-    const file = path.join(jsldir(), `${jslid}.jsonl.stats`);
-    return JSON.parse(fs.readFileSync(file, 'utf-8'));
+    const file = `${getJslFileName(jslid)}.stats`;
+    if (fs.existsSync(file)) return JSON.parse(fs.readFileSync(file, 'utf-8'));
+    return {};
   },
 
   async notifyChangedStats(stats) {

packages/api/src/main.js

@@ -20,6 +20,7 @@ const runners = require('./controllers/runners');
 const jsldata = require('./controllers/jsldata');
 const config = require('./controllers/config');
 const files = require('./controllers/files');
+const archive = require('./controllers/archive');
 
 const { rundir } = require('./utility/directories');
 
@@ -55,6 +56,7 @@ function start(argument = null) {
   useController(app, '/jsldata', jsldata);
   useController(app, '/config', config);
   useController(app, '/files', files);
+  useController(app, '/archive', archive);
 
   if (process.env.PAGES_DIRECTORY) {
     app.use('/pages', express.static(process.env.PAGES_DIRECTORY));

packages/api/src/shell/archiveWriter.js

@@ -0,0 +1,14 @@
+const path = require('path');
+const { archivedir, ensureDirectory } = require('../utility/directories');
+// const socket = require('../utility/socket');
+const jsonLinesWriter = require('./jsonLinesWriter');
+
+function archiveWriter({ folderName, fileName }) {
+  if (folderName == 'default') ensureDirectory(path.join(archivedir(), folderName));
+  const jsonlFile = path.join(archivedir(), folderName, `${fileName}.jsonl`);
+  const res = jsonLinesWriter({ fileName: jsonlFile });
+  // socket.emitChanged(`archive-files-changed-${folderName}`);
+  return res;
+}
+
+module.exports = archiveWriter;

packages/api/src/shell/index.js

@@ -11,6 +11,7 @@ const excelSheetReader = require('./excelSheetReader');
 const jsonLinesWriter = require('./jsonLinesWriter');
 const jsonLinesReader = require('./jsonLinesReader');
 const jslDataReader = require('./jslDataReader');
+const archiveWriter = require('./archiveWriter');
 
 module.exports = {
   queryReader,
@@ -26,4 +27,5 @@ module.exports = {
   fakeObjectReader,
   consoleObjectWriter,
   jslDataReader,
+  archiveWriter,
 };

packages/api/src/utility/directories.js

@@ -2,31 +2,28 @@ const os = require('os');
 const path = require('path');
 const fs = require('fs');
 
-let createdDatadir = false;
 const createDirectories = {};
 
+const ensureDirectory = (dir) => {
+  if (!createDirectories[dir]) {
+    if (!fs.existsSync(dir)) {
+      console.log(`Creating directory ${dir}`);
+      fs.mkdirSync(dir);
+    }
+    createDirectories[dir] = true;
+  }
+};
+
 function datadir() {
   const dir = path.join(os.homedir(), 'dbgate-data');
-  if (!createdDatadir) {
-    if (!fs.existsSync(dir)) {
-      console.log(`Creating data directory ${dir}`);
-      fs.mkdirSync(dir);
-    }
-    createdDatadir = true;
-  }
+  ensureDirectory(dir);
 
   return dir;
 }
 
 const dirFunc = (dirname) => () => {
   const dir = path.join(datadir(), dirname);
-  if (!createDirectories[dirname]) {
-    if (!fs.existsSync(dir)) {
-      console.log(`Creating jsl directory ${dir}`);
-      fs.mkdirSync(dir);
-    }
-    createDirectories[dirname] = true;
-  }
+  ensureDirectory(dir);
 
   return dir;
 };
@@ -34,10 +31,13 @@ const dirFunc = (dirname) => () => {
 const jsldir = dirFunc('jsl');
 const rundir = dirFunc('run');
 const uploadsdir = dirFunc('uploads');
+const archivedir = dirFunc('archive');
 
 module.exports = {
   datadir,
   jsldir,
   rundir,
   uploadsdir,
+  archivedir,
+  ensureDirectory,
 };