diff --git a/.github/workflows/build-app-beta.yaml b/.github/workflows/build-app-beta.yaml index adc7ce9e8..e5948beb2 100644 --- a/.github/workflows/build-app-beta.yaml +++ b/.github/workflows/build-app-beta.yaml @@ -6,9 +6,13 @@ name: Electron app BETA push: tags: - v[0-9]+.[0-9]+.[0-9]+-beta.[0-9]+ +permissions: + id-token: write + contents: write jobs: build: runs-on: ${{ matrix.os }} + environment: dbgate-app strategy: fail-fast: false matrix: @@ -53,12 +57,6 @@ jobs: run: | yarn setCurrentVersion - - name: printSecrets - run: | - - yarn printSecrets - env: - GIST_UPLOAD_SECRET: ${{secrets.GIST_UPLOAD_SECRET}} - name: fillPackagedPlugins run: | @@ -66,21 +64,53 @@ jobs: - name: Install Snapcraft if: matrix.os == 'ubuntu-22.04' uses: samuelmeuli/action-snapcraft@v1 - - name: Publish + - name: Publish Windows + if: matrix.os == 'windows-2022' + run: | + + yarn run build:app + - name: Publish MacOS + if: matrix.os == 'macos-14' run: | yarn run build:app env: - GH_TOKEN: ${{ secrets.GH_TOKEN }} - WIN_CSC_LINK: ${{ secrets.WINCERT_2025 }} - WIN_CSC_KEY_PASSWORD: ${{ secrets.WINCERT_2025_PASSWORD }} CSC_LINK: ${{ secrets.APPLECERT_CERTIFICATE }} CSC_KEY_PASSWORD: ${{ secrets.APPLECERT_PASSWORD }} APPLE_ID: ${{ secrets.APPLE_ID }} APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }} APPLE_ID_PASSWORD: ${{ secrets.APPLE_ID_PASSWORD }} - SNAPCRAFT_STORE_CREDENTIALS: ${{secrets.SNAPCRAFT_LOGIN}} APPLE_APP_SPECIFIC_PASSWORD: ${{secrets.APPLE_APP_SPECIFIC_PASSWORD}} + - name: Publish Linux + if: matrix.os == 'ubuntu-22.04' + run: | + + yarn run build:app + env: + SNAPCRAFT_STORE_CREDENTIALS: ${{secrets.SNAPCRAFT_LOGIN}} + - name: Azure login (OIDC) + uses: azure/login@v2 + if: matrix.os == 'windows-2022' + with: + client-id: ${{ secrets.AZURE_TC_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TC_TENANT_ID }} + allow-no-subscriptions: true + - name: Sign Windows artifacts with Azure Trusted Signing + uses: azure/trusted-signing-action@v0 + if: matrix.os == 'windows-2022' + with: + endpoint: https://wus3.codesigning.azure.net/ + trusted-signing-account-name: DbGate + certificate-profile-name: DbGate-Release + files-folder: app/dist + files-folder-filter: exe + timestamp-rfc3161: http://timestamp.acs.microsoft.com + timestamp-digest: SHA256 + - name: Fix YML hashes + if: matrix.os == 'windows-2022' + run: | + + yarn run fixYmlHashes - name: Copy artifacts run: | mkdir artifacts diff --git a/.github/workflows/build-app-check.yaml b/.github/workflows/build-app-check.yaml index d8962ba3d..173a21fad 100644 --- a/.github/workflows/build-app-check.yaml +++ b/.github/workflows/build-app-check.yaml @@ -6,9 +6,13 @@ name: Electron app check build push: tags: - check-[0-9]+-[0-9]+-[0-9]+.[0-9]+ +permissions: + id-token: write + contents: write jobs: build: runs-on: ${{ matrix.os }} + environment: dbgate-app strategy: fail-fast: false matrix: @@ -49,12 +53,6 @@ jobs: run: | yarn setCurrentVersion - - name: printSecrets - run: | - - yarn printSecrets - env: - GIST_UPLOAD_SECRET: ${{secrets.GIST_UPLOAD_SECRET}} - name: fillPackagedPlugins run: | @@ -62,21 +60,53 @@ jobs: - name: Install Snapcraft if: matrix.os == 'ubuntu-22.04' uses: samuelmeuli/action-snapcraft@v1 - - name: Publish + - name: Publish Windows + if: matrix.os == 'windows-2022' + run: | + + yarn run build:app + - name: Publish MacOS + if: matrix.os == 'macos-14' run: | yarn run build:app env: - GH_TOKEN: ${{ secrets.GH_TOKEN }} - WIN_CSC_LINK: ${{ secrets.WINCERT_2025 }} - WIN_CSC_KEY_PASSWORD: ${{ secrets.WINCERT_2025_PASSWORD }} CSC_LINK: ${{ secrets.APPLECERT_CERTIFICATE }} CSC_KEY_PASSWORD: ${{ secrets.APPLECERT_PASSWORD }} APPLE_ID: ${{ secrets.APPLE_ID }} APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }} APPLE_ID_PASSWORD: ${{ secrets.APPLE_ID_PASSWORD }} - SNAPCRAFT_STORE_CREDENTIALS: ${{secrets.SNAPCRAFT_LOGIN}} APPLE_APP_SPECIFIC_PASSWORD: ${{secrets.APPLE_APP_SPECIFIC_PASSWORD}} + - name: Publish Linux + if: matrix.os == 'ubuntu-22.04' + run: | + + yarn run build:app + env: + SNAPCRAFT_STORE_CREDENTIALS: ${{secrets.SNAPCRAFT_LOGIN}} + - name: Azure login (OIDC) + uses: azure/login@v2 + if: matrix.os == 'windows-2022' + with: + client-id: ${{ secrets.AZURE_TC_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TC_TENANT_ID }} + allow-no-subscriptions: true + - name: Sign Windows artifacts with Azure Trusted Signing + uses: azure/trusted-signing-action@v0 + if: matrix.os == 'windows-2022' + with: + endpoint: https://wus3.codesigning.azure.net/ + trusted-signing-account-name: DbGate + certificate-profile-name: DbGate-Release + files-folder: app/dist + files-folder-filter: exe + timestamp-rfc3161: http://timestamp.acs.microsoft.com + timestamp-digest: SHA256 + - name: Fix YML hashes + if: matrix.os == 'windows-2022' + run: | + + yarn run fixYmlHashes - name: Copy artifacts run: | mkdir artifacts diff --git a/.github/workflows/build-app-pro-beta.yaml b/.github/workflows/build-app-pro-beta.yaml index 04e6e6539..7d926bb14 100644 --- a/.github/workflows/build-app-pro-beta.yaml +++ b/.github/workflows/build-app-pro-beta.yaml @@ -6,9 +6,13 @@ name: Electron app PREMIUM BETA push: tags: - v[0-9]+.[0-9]+.[0-9]+-premium-beta.[0-9]+ +permissions: + id-token: write + contents: write jobs: build: runs-on: ${{ matrix.os }} + environment: dbgate-app strategy: fail-fast: false matrix: @@ -39,7 +43,7 @@ jobs: repository: dbgate/dbgate-pro token: ${{ secrets.GH_TOKEN }} path: dbgate-pro - ref: 4b28757ade169ac0a1696351519bbaa4bbba5db9 + ref: 2a71bec538f8e2cf6c1cd1322d89e64346a139fd - name: Merge dbgate/dbgate-pro run: | mkdir ../dbgate-pro @@ -81,37 +85,67 @@ jobs: cd dbgate-merged yarn setCurrentVersion - - name: printSecrets - run: | - cd .. - cd dbgate-merged - - yarn printSecrets - env: - GIST_UPLOAD_SECRET: ${{secrets.GIST_UPLOAD_SECRET}} - name: fillPackagedPlugins run: | cd .. cd dbgate-merged yarn fillPackagedPlugins - - name: Publish + - name: Publish Windows + if: matrix.os == 'windows-2022' + run: | + cd .. + cd dbgate-merged + + yarn run build:app + - name: Publish MacOS + if: matrix.os == 'macos-14' run: | cd .. cd dbgate-merged yarn run build:app env: - GH_TOKEN: ${{ secrets.GH_TOKEN }} - WIN_CSC_LINK: ${{ secrets.WINCERT_2025 }} - WIN_CSC_KEY_PASSWORD: ${{ secrets.WINCERT_2025_PASSWORD }} CSC_LINK: ${{ secrets.APPLECERT_CERTIFICATE }} CSC_KEY_PASSWORD: ${{ secrets.APPLECERT_PASSWORD }} APPLE_ID: ${{ secrets.APPLE_ID }} APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }} APPLE_ID_PASSWORD: ${{ secrets.APPLE_ID_PASSWORD }} - SNAPCRAFT_STORE_CREDENTIALS: ${{secrets.SNAPCRAFT_LOGIN}} APPLE_APP_SPECIFIC_PASSWORD: ${{secrets.APPLE_APP_SPECIFIC_PASSWORD}} + - name: Publish Linux + if: matrix.os == 'ubuntu-22.04' + run: | + cd .. + cd dbgate-merged + + yarn run build:app + env: + SNAPCRAFT_STORE_CREDENTIALS: ${{secrets.SNAPCRAFT_LOGIN}} + - name: Azure login (OIDC) + uses: azure/login@v2 + if: matrix.os == 'windows-2022' + with: + client-id: ${{ secrets.AZURE_TC_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TC_TENANT_ID }} + allow-no-subscriptions: true + - name: Sign Windows artifacts with Azure Trusted Signing + uses: azure/trusted-signing-action@v0 + if: matrix.os == 'windows-2022' + with: + endpoint: https://wus3.codesigning.azure.net/ + trusted-signing-account-name: DbGate + certificate-profile-name: DbGate-Release + files-folder: ../dbgate-merged/app/dist + files-folder-filter: exe + timestamp-rfc3161: http://timestamp.acs.microsoft.com + timestamp-digest: SHA256 + - name: Fix YML hashes + if: matrix.os == 'windows-2022' + run: | + cd .. + cd dbgate-merged + + yarn run fixYmlHashes - name: Copy artifacts run: | mkdir artifacts diff --git a/.github/workflows/build-app-pro.yaml b/.github/workflows/build-app-pro.yaml index fb8ac8101..a88bf9bbc 100644 --- a/.github/workflows/build-app-pro.yaml +++ b/.github/workflows/build-app-pro.yaml @@ -6,9 +6,13 @@ name: Electron app PREMIUM push: tags: - v[0-9]+.[0-9]+.[0-9]+ +permissions: + id-token: write + contents: write jobs: build: runs-on: ${{ matrix.os }} + environment: dbgate-app strategy: fail-fast: false matrix: @@ -39,7 +43,7 @@ jobs: repository: dbgate/dbgate-pro token: ${{ secrets.GH_TOKEN }} path: dbgate-pro - ref: 4b28757ade169ac0a1696351519bbaa4bbba5db9 + ref: 2a71bec538f8e2cf6c1cd1322d89e64346a139fd - name: Merge dbgate/dbgate-pro run: | mkdir ../dbgate-pro @@ -81,37 +85,67 @@ jobs: cd dbgate-merged yarn setCurrentVersion - - name: printSecrets - run: | - cd .. - cd dbgate-merged - - yarn printSecrets - env: - GIST_UPLOAD_SECRET: ${{secrets.GIST_UPLOAD_SECRET}} - name: fillPackagedPlugins run: | cd .. cd dbgate-merged yarn fillPackagedPlugins - - name: Publish + - name: Publish Windows + if: matrix.os == 'windows-2022' + run: | + cd .. + cd dbgate-merged + + yarn run build:app + - name: Publish MacOS + if: matrix.os == 'macos-14' run: | cd .. cd dbgate-merged yarn run build:app env: - GH_TOKEN: ${{ secrets.GH_TOKEN }} - WIN_CSC_LINK: ${{ secrets.WINCERT_2025 }} - WIN_CSC_KEY_PASSWORD: ${{ secrets.WINCERT_2025_PASSWORD }} CSC_LINK: ${{ secrets.APPLECERT_CERTIFICATE }} CSC_KEY_PASSWORD: ${{ secrets.APPLECERT_PASSWORD }} APPLE_ID: ${{ secrets.APPLE_ID }} APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }} APPLE_ID_PASSWORD: ${{ secrets.APPLE_ID_PASSWORD }} - SNAPCRAFT_STORE_CREDENTIALS: ${{secrets.SNAPCRAFT_LOGIN}} APPLE_APP_SPECIFIC_PASSWORD: ${{secrets.APPLE_APP_SPECIFIC_PASSWORD}} + - name: Publish Linux + if: matrix.os == 'ubuntu-22.04' + run: | + cd .. + cd dbgate-merged + + yarn run build:app + env: + SNAPCRAFT_STORE_CREDENTIALS: ${{secrets.SNAPCRAFT_LOGIN}} + - name: Azure login (OIDC) + uses: azure/login@v2 + if: matrix.os == 'windows-2022' + with: + client-id: ${{ secrets.AZURE_TC_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TC_TENANT_ID }} + allow-no-subscriptions: true + - name: Sign Windows artifacts with Azure Trusted Signing + uses: azure/trusted-signing-action@v0 + if: matrix.os == 'windows-2022' + with: + endpoint: https://wus3.codesigning.azure.net/ + trusted-signing-account-name: DbGate + certificate-profile-name: DbGate-Release + files-folder: ../dbgate-merged/app/dist + files-folder-filter: exe + timestamp-rfc3161: http://timestamp.acs.microsoft.com + timestamp-digest: SHA256 + - name: Fix YML hashes + if: matrix.os == 'windows-2022' + run: | + cd .. + cd dbgate-merged + + yarn run fixYmlHashes - name: Copy artifacts run: | mkdir artifacts diff --git a/.github/workflows/build-app.yaml b/.github/workflows/build-app.yaml index 7e2225365..212f72a2b 100644 --- a/.github/workflows/build-app.yaml +++ b/.github/workflows/build-app.yaml @@ -6,9 +6,13 @@ name: Electron app push: tags: - v[0-9]+.[0-9]+.[0-9]+ +permissions: + id-token: write + contents: write jobs: build: runs-on: ${{ matrix.os }} + environment: dbgate-app strategy: fail-fast: false matrix: @@ -49,12 +53,6 @@ jobs: run: | yarn setCurrentVersion - - name: printSecrets - run: | - - yarn printSecrets - env: - GIST_UPLOAD_SECRET: ${{secrets.GIST_UPLOAD_SECRET}} - name: fillPackagedPlugins run: | @@ -62,24 +60,56 @@ jobs: - name: Install Snapcraft if: matrix.os == 'ubuntu-22.04' uses: samuelmeuli/action-snapcraft@v1 - - name: Publish + - name: Publish Windows + if: matrix.os == 'windows-2022' + run: | + + yarn run build:app + - name: Publish MacOS + if: matrix.os == 'macos-14' run: | yarn run build:app env: - GH_TOKEN: ${{ secrets.GH_TOKEN }} - WIN_CSC_LINK: ${{ secrets.WINCERT_2025 }} - WIN_CSC_KEY_PASSWORD: ${{ secrets.WINCERT_2025_PASSWORD }} CSC_LINK: ${{ secrets.APPLECERT_CERTIFICATE }} CSC_KEY_PASSWORD: ${{ secrets.APPLECERT_PASSWORD }} APPLE_ID: ${{ secrets.APPLE_ID }} APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }} APPLE_ID_PASSWORD: ${{ secrets.APPLE_ID_PASSWORD }} - SNAPCRAFT_STORE_CREDENTIALS: ${{secrets.SNAPCRAFT_LOGIN}} APPLE_APP_SPECIFIC_PASSWORD: ${{secrets.APPLE_APP_SPECIFIC_PASSWORD}} + - name: Publish Linux + if: matrix.os == 'ubuntu-22.04' + run: | + + yarn run build:app + env: + SNAPCRAFT_STORE_CREDENTIALS: ${{secrets.SNAPCRAFT_LOGIN}} - name: generatePadFile run: | yarn generatePadFile + - name: Azure login (OIDC) + uses: azure/login@v2 + if: matrix.os == 'windows-2022' + with: + client-id: ${{ secrets.AZURE_TC_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TC_TENANT_ID }} + allow-no-subscriptions: true + - name: Sign Windows artifacts with Azure Trusted Signing + uses: azure/trusted-signing-action@v0 + if: matrix.os == 'windows-2022' + with: + endpoint: https://wus3.codesigning.azure.net/ + trusted-signing-account-name: DbGate + certificate-profile-name: DbGate-Release + files-folder: app/dist + files-folder-filter: exe + timestamp-rfc3161: http://timestamp.acs.microsoft.com + timestamp-digest: SHA256 + - name: Fix YML hashes + if: matrix.os == 'windows-2022' + run: | + + yarn run fixYmlHashes - name: Copy artifacts run: | mkdir artifacts diff --git a/.github/workflows/build-cloud-pro.yaml b/.github/workflows/build-cloud-pro.yaml index 09cde083e..b12c05d08 100644 --- a/.github/workflows/build-cloud-pro.yaml +++ b/.github/workflows/build-cloud-pro.yaml @@ -39,7 +39,7 @@ jobs: repository: dbgate/dbgate-pro token: ${{ secrets.GH_TOKEN }} path: dbgate-pro - ref: 4b28757ade169ac0a1696351519bbaa4bbba5db9 + ref: 2a71bec538f8e2cf6c1cd1322d89e64346a139fd - name: Merge dbgate/dbgate-pro run: | mkdir ../dbgate-pro @@ -66,13 +66,6 @@ jobs: cd .. cd dbgate-merged yarn setCurrentVersion - - name: printSecrets - run: | - cd .. - cd dbgate-merged - yarn printSecrets - env: - GIST_UPLOAD_SECRET: ${{secrets.GIST_UPLOAD_SECRET}} - name: Prepare packer build run: | cd .. diff --git a/.github/workflows/build-docker-pro.yaml b/.github/workflows/build-docker-pro.yaml index 2b772cf56..0ed677792 100644 --- a/.github/workflows/build-docker-pro.yaml +++ b/.github/workflows/build-docker-pro.yaml @@ -44,7 +44,7 @@ jobs: repository: dbgate/dbgate-pro token: ${{ secrets.GH_TOKEN }} path: dbgate-pro - ref: 4b28757ade169ac0a1696351519bbaa4bbba5db9 + ref: 2a71bec538f8e2cf6c1cd1322d89e64346a139fd - name: Merge dbgate/dbgate-pro run: | mkdir ../dbgate-pro @@ -76,14 +76,6 @@ jobs: cd dbgate-merged yarn setCurrentVersion - - name: printSecrets - run: | - cd .. - cd dbgate-merged - - yarn printSecrets - env: - GIST_UPLOAD_SECRET: ${{secrets.GIST_UPLOAD_SECRET}} - name: Prepare docker image run: | cd .. diff --git a/.github/workflows/build-docker.yaml b/.github/workflows/build-docker.yaml index 644335b1d..3316dc4a5 100644 --- a/.github/workflows/build-docker.yaml +++ b/.github/workflows/build-docker.yaml @@ -65,12 +65,6 @@ jobs: run: | yarn setCurrentVersion - - name: printSecrets - run: | - - yarn printSecrets - env: - GIST_UPLOAD_SECRET: ${{secrets.GIST_UPLOAD_SECRET}} - name: Prepare docker image run: | diff --git a/.github/workflows/build-npm-pro.yaml b/.github/workflows/build-npm-pro.yaml index f9fd9aec3..6f3bd13c3 100644 --- a/.github/workflows/build-npm-pro.yaml +++ b/.github/workflows/build-npm-pro.yaml @@ -7,6 +7,9 @@ name: NPM packages PREMIUM tags: - v[0-9]+.[0-9]+.[0-9]+ - v[0-9]+.[0-9]+.[0-9]+-alpha.[0-9]+ +permissions: + id-token: write + contents: write jobs: build: runs-on: ${{ matrix.os }} @@ -32,7 +35,7 @@ jobs: repository: dbgate/dbgate-pro token: ${{ secrets.GH_TOKEN }} path: dbgate-pro - ref: 4b28757ade169ac0a1696351519bbaa4bbba5db9 + ref: 2a71bec538f8e2cf6c1cd1322d89e64346a139fd - name: Merge dbgate/dbgate-pro run: | mkdir ../dbgate-pro @@ -49,13 +52,8 @@ jobs: cd .. cd dbgate-merged node adjustNpmPackageJsonPremium - - name: Configure NPM token - env: - NPM_TOKEN: ${{ secrets.NPM_TOKEN }} - run: | - cd .. - cd dbgate-merged - npm config set '//registry.npmjs.org/:_authToken' "${NPM_TOKEN}" + - name: Update npm + run: npm install -g npm@latest - name: Remove dbmodel - should be not published run: | cd .. @@ -71,35 +69,35 @@ jobs: cd .. cd dbgate-merged yarn setCurrentVersion - - name: printSecrets + - name: Compute npm dist-tag run: | - cd .. - cd dbgate-merged - yarn printSecrets - env: - GIST_UPLOAD_SECRET: ${{secrets.GIST_UPLOAD_SECRET}} + if [[ "${GITHUB_REF_NAME}" =~ -alpha\. ]]; then + echo "NPM_TAG=alpha" >> $GITHUB_ENV + else + echo "NPM_TAG=latest" >> $GITHUB_ENV + fi - name: Publish dbgate-api-premium run: | cd .. cd dbgate-merged/packages/api - npm publish + npm publish --tag "$NPM_TAG" - name: Publish dbgate-web-premium run: | cd .. cd dbgate-merged/packages/web - npm publish + npm publish --tag "$NPM_TAG" - name: Publish dbgate-serve-premium run: | cd .. cd dbgate-merged/packages/serve - npm publish + npm publish --tag "$NPM_TAG" - name: Publish dbgate-plugin-cosmosdb run: | cd .. cd dbgate-merged/plugins/dbgate-plugin-cosmosdb - npm publish + npm publish --tag "$NPM_TAG" - name: Publish dbgate-plugin-firestore run: | cd .. cd dbgate-merged/plugins/dbgate-plugin-firestore - npm publish + npm publish --tag "$NPM_TAG" diff --git a/.github/workflows/build-npm.yaml b/.github/workflows/build-npm.yaml index c20a26cff..079c25f8a 100644 --- a/.github/workflows/build-npm.yaml +++ b/.github/workflows/build-npm.yaml @@ -7,6 +7,9 @@ name: NPM packages tags: - v[0-9]+.[0-9]+.[0-9]+ - v[0-9]+.[0-9]+.[0-9]+-alpha.[0-9]+ +permissions: + id-token: write + contents: write jobs: build: runs-on: ${{ matrix.os }} @@ -26,108 +29,107 @@ jobs: uses: actions/setup-node@v1 with: node-version: 22.x - - name: Configure NPM token - env: - NPM_TOKEN: ${{ secrets.NPM_TOKEN }} - run: | - npm config set '//registry.npmjs.org/:_authToken' "${NPM_TOKEN}" + - name: Update npm + run: npm install -g npm@latest - name: yarn install run: | yarn install - name: setCurrentVersion run: | yarn setCurrentVersion - - name: printSecrets + - name: Compute npm dist-tag run: | - yarn printSecrets - env: - GIST_UPLOAD_SECRET: ${{secrets.GIST_UPLOAD_SECRET}} + if [[ "${GITHUB_REF_NAME}" =~ -alpha\. ]]; then + echo "NPM_TAG=alpha" >> $GITHUB_ENV + else + echo "NPM_TAG=latest" >> $GITHUB_ENV + fi - name: Publish types working-directory: packages/types run: | - npm publish + npm publish --tag "$NPM_TAG" - name: Publish tools working-directory: packages/tools run: | - npm publish + npm publish --tag "$NPM_TAG" - name: Publish sqltree working-directory: packages/sqltree run: | - npm publish + npm publish --tag "$NPM_TAG" - name: Publish api working-directory: packages/api run: | - npm publish + npm publish --tag "$NPM_TAG" - name: Publish datalib working-directory: packages/datalib run: | - npm publish + npm publish --tag "$NPM_TAG" - name: Publish filterparser working-directory: packages/filterparser run: | - npm publish + npm publish --tag "$NPM_TAG" - name: Publish web working-directory: packages/web run: | - npm publish + npm publish --tag "$NPM_TAG" - name: Publish dbgate-serve working-directory: packages/serve run: | - npm publish + npm publish --tag "$NPM_TAG" - name: Publish dbmodel working-directory: packages/dbmodel run: | - npm publish + npm publish --tag "$NPM_TAG" - name: Publish dbgate-plugin-csv working-directory: plugins/dbgate-plugin-csv run: | - npm publish + npm publish --tag "$NPM_TAG" - name: Publish dbgate-plugin-xml working-directory: plugins/dbgate-plugin-xml run: | - npm publish + npm publish --tag "$NPM_TAG" - name: Publish dbgate-plugin-excel working-directory: plugins/dbgate-plugin-excel run: | - npm publish + npm publish --tag "$NPM_TAG" - name: Publish dbgate-plugin-mssql working-directory: plugins/dbgate-plugin-mssql run: | - npm publish + npm publish --tag "$NPM_TAG" - name: Publish dbgate-plugin-mysql working-directory: plugins/dbgate-plugin-mysql run: | - npm publish + npm publish --tag "$NPM_TAG" - name: Publish dbgate-plugin-mongo working-directory: plugins/dbgate-plugin-mongo run: | - npm publish + npm publish --tag "$NPM_TAG" - name: Publish dbgate-plugin-postgres working-directory: plugins/dbgate-plugin-postgres run: | - npm publish + npm publish --tag "$NPM_TAG" - name: Publish dbgate-plugin-sqlite working-directory: plugins/dbgate-plugin-sqlite run: | - npm publish + npm publish --tag "$NPM_TAG" - name: Publish dbgate-plugin-redis working-directory: plugins/dbgate-plugin-redis run: | - npm publish + npm publish --tag "$NPM_TAG" - name: Publish dbgate-plugin-oracle working-directory: plugins/dbgate-plugin-oracle run: | - npm publish + npm publish --tag "$NPM_TAG" - name: Publish dbgate-plugin-clickhouse working-directory: plugins/dbgate-plugin-clickhouse run: | - npm publish + npm publish --tag "$NPM_TAG" - name: Publish dbgate-plugin-dbf working-directory: plugins/dbgate-plugin-dbf run: | - npm publish + npm publish --tag "$NPM_TAG" - name: Publish dbgate-plugin-cassandra working-directory: plugins/dbgate-plugin-cassandra run: | - npm publish + npm publish --tag "$NPM_TAG" diff --git a/.github/workflows/e2e-pro.yaml b/.github/workflows/e2e-pro.yaml index a6e86a76a..7d1f63fe8 100644 --- a/.github/workflows/e2e-pro.yaml +++ b/.github/workflows/e2e-pro.yaml @@ -9,6 +9,9 @@ name: Cypress tests with screenshots PREMIUM - develop - feature/** - hotfix/** +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: e2e-tests: runs-on: ubuntu-latest @@ -26,7 +29,7 @@ jobs: repository: dbgate/dbgate-pro token: ${{ secrets.GH_TOKEN }} path: dbgate-pro - ref: 4b28757ade169ac0a1696351519bbaa4bbba5db9 + ref: 2a71bec538f8e2cf6c1cd1322d89e64346a139fd - name: Merge dbgate/dbgate-pro run: | mkdir ../dbgate-pro diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index 14f2cd751..80ddfbe7d 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -9,6 +9,9 @@ name: Integration and unit tests - develop - feature/** - hotfix/** +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: all-tests: runs-on: ubuntu-latest @@ -42,24 +45,6 @@ jobs: run: | cd packages/tools yarn test:ci - - uses: tanmen/jest-reporter@v1 - if: always() - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - result-file: integration-tests/result.json - action-name: Integration tests - - uses: tanmen/jest-reporter@v1 - if: always() - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - result-file: packages/filterparser/result.json - action-name: Filter parser test results - - uses: tanmen/jest-reporter@v1 - if: always() - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - result-file: packages/datalib/result.json - action-name: Datalib (perspectives) test results services: postgres-integr: image: postgres @@ -83,7 +68,7 @@ jobs: ports: - '15002:1433' clickhouse-integr: - image: bitnami/clickhouse:24.8.4 + image: bitnamilegacy/clickhouse:24.8.4 env: CLICKHOUSE_ADMIN_PASSWORD: Pwd2020Db ports: diff --git a/.gitignore b/.gitignore index a979e82c8..7f3e7b4c4 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,7 @@ docker/plugins .env.development.local .env.test.local .env.production.local +.env.translation npm-debug.log* yarn-debug.log* diff --git a/CHANGELOG.md b/CHANGELOG.md index 915b1cb47..19d08072e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,126 @@ Builds: - linux - application for linux - win - application for Windows +## 6.8.0 +- ADDED: Form cell view for detailed data inspection and editing in data grids, with multi-row bulk editing support +- CHANGED: Cell data sidebar moved to right side, now is part of data grid +- FIXED: Improved widget resizing algorithm +- FIXED: Word wrap feature in SQL editor +- CHANGED: Data grid keyboard navigation improvements +- CHANGED: Improved PostgreSQL decimal type support in data grid #1214 +- ADDED: Retrieve number of databases from Redis configuration #1278 +- ADDED: Run macro context menu (Premium) +- ADDED: Support for skip update columns in replicator +- FIXED: UTF-8 BOM handling in CSV input +- CHANGED: Advanced export is now part of Community edition +- FIXED: SQLite foreign key constraint types +- FIXED: Double drop constraint issue +- CHANGED: Improved map view lat/lon field autodetection +- FIXED: Alter table operations and constraint sanitization +- ADDED: Import connections from environment variables (Team Premium) + +## 6.7.3 +- FIXED: Fixed problem in analyser core - in PostgreSQL, after dropping table, dropped table still appeared in structure +- FIXED: PostgreSQL numeric columns do not align right #1254 +- ADDED: Custom thousands separator #1213 + +## 6.7.2 +- CHANGED: Settings modal redesign - now is settings opened in tab instead of modal, similarily as in VSCode +- FIXED: Fixed search in table shortcuts #1273 +- CHANGED: Improved foreign key editor UX +- FIXED: Fixed incremental DB structure refresh for PostgreSQL, optimalized slow loading primary keys in PostgreSQL +- CHANGED: You could now choose, how to refresh structure, added ability to disconnect or reconnect +- ADDED: Better processing of table backups, generate table restore script #1274 +- CHANGED: Improved storage of settings, especially for Team Premium edition + +## 6.7.1 +- ADDED: LANGUAGE environment variable for the web version. #1266 +- ADDED: New localizations (Italian, Portugese (Brazil), Japanese) +- ADDED: Option to detect language from browser settings in web version +- FIXED: Check updates option no longer available in 6.7.0 #1263 +- FIXED: A MERGE statement must be terminated by a semi-colon (;), but dbgate stripped it. #1257 +- ADDED: Show table size #552 +- ADDED: Sort tables by size and by row count +- ADDED: Connect to Legacy MongoDB (Premium) #540 +- FIXED: Fixed problems in saving team files in Team Premium edition +- CHANGED: Files are by default saved to team folders in Team Premium edition +- ADDED: Other files types supported in Team Premium edition (diagrams, query design, perspectives, import/export jobs, shell scripts, database compare jobs) + +## 6.7.0 +- ADDED: Added localization support, now you can use DbGate in multiple languages (French, Spanish, German, Czech, Slovak, Simplified Chinese) #347 #705 #939 #1079 +- CHANGED: Solved many issues with binary fields, huge performance improvements in binary fields processing +- FIXED: Export to CSV produces empty file #1247 +- CHANGED: Upgraded electron to version 38 #1243 +- FIXED: PostgreSQL export to SQL and XML doesn't include bytea field contents #1228 +- FIXED: Export CSV broken #1080 +- FIXED: Inconsistent handling of hex-like strings #680 +- FIXED: Export mongodb binary cell as binary file #292 +- CHANGED: SSL is used automatically for connections to Azure databases +- ADDED: New export formats CSV for Excel, TSV +- FIXED: Horizontal scrolling on macOS trackpad/Magic Mouse #1250 + +## 6.6.12 +- FIXED: Cannot paste license key on Mac (special commands like copy/paste were disabled on license screen) + +## 6.6.11 +- FIXED: Fixed theming on application startup +- CHANGED: Improved licensing page + +## 6.6.10 +- FIXED: License from environment variable is not refreshed #1245 +- FIXED: connection closing / reconnecting #1237 +- ADDED: retain history across multiple queries #1236 +- ADDED: load CSVs to temp tables #1235 +- FIXED: Not possible to scroll the data view horizontally by pressing shift and scroll mouse middle button on Mac #453 +- FIXED: Expired trial workflow (Premium) +- ADDED: Column name collision resolving #1234 (MySQL) + +## 6.6.8 +- CHANGED: Windows executable now uses Azure trusted signing certificate +- CHANGED: NPM packages now use GitHub OIDC provenance signing for better security +- CHANGED: Some features moved to Premium edition (master/detail views, FK lookups, column expansion, split view, advanced export/import, data archives, grouping, macros) + +## 6.6.6 +- ADDED: Allow disable/re-enable filter #1174 +- ADDED: Close right side tabs #1219 +- ADDED: Ability disable execute current line in query editor #1209 +- ADDED: Support for Redis Cluster #1204 (Premium) + +## 6.6.5 +- ADDED: SQL AI assistant - powered by database chat, could help you to write SQL queries (Premium) +- ADDED: Explain SQL error (powered by AI) (Premium) +- ADDED: Database chat (and SQL AI Assistant) now supports showing charts (Premium) +- FIXED: Fixed editing new files and roles (Team Premium) +- FIXED: Connection to standalone database could be now pinned +- FIXED: Cannot open up large JSON file #1215 + +## 6.6.4 +- ADDED: AI Database chat now supports much more LLM models. (Premium) +- ADDED: Possibility to use your own API key with OPENAI-compatible providers (OpenRouter, Antropic...) +- ADDED: Possibility to use self-hosted own LLM (eg. Llama) +- ADDED: Team files - save SQL files and define shared charts, assign roles and users to these objects (Team Premium) +- FIXED: BUG: does no longer work with Cockroach DB #1202 +- FIXED: DbGate Web UI Connections do not display 'Databases' #1199 +- CHANGED: Redesign fof applications. Applications are now storted in single JSON file +- ADDED: Application editor (Premium) +- ADDED: Posibility to filter only tables with rows +- FIXED: Fixed several issues with large Firebird databases +- CHANGED: Community edition now supports shared folders in read-only mode + +## 6.6.3 +- FIXED: Error “db.getCollection(…).renameCollection is not a function” when renaming collection in dbGate #1198 +- FIXED: Can't list databases from Azure SQL SERVER #1197 +- ADDED: Save zoom level in electron apps + +## 6.6.2 +- ADDED: List of processes, ability to kill process (Server summary) #1178 +- ADDED: Database and table permissions (Team Premium edition) +- ADDED: Redis search box - Scan all #1191 +- FIXED: Optimalized loading SQL server with descriptions #1187 +- CHANGED: Allow a much greater page size #1185 +- FIXED: Optimalized loading SQL server with descriptions #1187 +- FIXED: Executing queries for SQLite crash #1195 + ## 6.6.1 - ADDED: Support for Mongo shell (Premium) - #1114 - FIXED: Support for BLOB in Oracle #1181 diff --git a/README.md b/README.md index 0f3e839f7..639af9c4c 100644 --- a/README.md +++ b/README.md @@ -15,13 +15,10 @@ But there are also many advanced features like schema compare, visual query desi DbGate is licensed under GPL-3.0 license and is free to use for any purpose. * Try it online - [demo.dbgate.org](https://demo.dbgate.org) - online demo application -* **Download** application for Windows, Linux or Mac from [dbgate.io](https://dbgate.io/download/) -* Looking for DbGate Community? **Download** from [dbgate.org](https://dbgate.org/download/) +* **Download** application for Windows, Linux or Mac from [dbgate.io](https://www.dbgate.io/download/) +* Looking for DbGate Community? **Download** from [dbgate.io](https://www.dbgate.io/download-community/) * Run web version as [NPM package](https://www.npmjs.com/package/dbgate-serve) or as [docker image](https://hub.docker.com/r/dbgate/dbgate) * Use nodeJs [scripting interface](https://docs.dbgate.io/scripting) ([API documentation](https://docs.dbgate.io/apidoc)) -* [Recommend DbGate](https://testimonial.to/dbgate) | [Rate on G2](https://www.g2.com/products/dbgate/reviews) -* [Give us feedback](https://dbgate.org/feedback) - it will help us to decide, how to improve DbGate in future -* We [offer 2-year PREMIUM license](https://dbgate.org/review/) for any honest review on these platforms (time-limited offer) ## Supported databases * MySQL @@ -92,8 +89,7 @@ DbGate is licensed under GPL-3.0 license and is free to use for any purpose. Any contributions are welcome. If you want to contribute without coding, consider following: * Tell your friends about DbGate or share on social networks - when more people will use DbGate, it will grow to be better -* Purchase a [DbGate Premium](https://dbgate.io/purchase/premium/) liocense -* Write review on [Product Hunt](https://www.producthunt.com/products/dbgate) or [G2](https://www.g2.com/products/dbgate/reviews) - we offer [2-year PREMIUM license](https://dbgate.org/review/) for reviewers (time limited offer) +* Purchase a [DbGate Premium](https://www.dbgate.io/purchase/premium/) license * Create issue, if you find problem in app, or you have idea to new feature. If issue already exists, you could leave comment on it, to prioritise most wanted issues * Create some tutorial video on [youtube](https://www.youtube.com/playlist?list=PLCo7KjCVXhr0RfUSjM9wJMsp_ShL1q61A) * Become a backer on [GitHub sponsors](https://github.com/sponsors/dbgate) or [Open collective](https://opencollective.com/dbgate) diff --git a/app/package.json b/app/package.json index 8db96276c..1fe96dde0 100644 --- a/app/package.json +++ b/app/package.json @@ -117,7 +117,7 @@ "scripts": { "start": "cross-env ELECTRON_START_URL=http://localhost:5001 DEVMODE=1 electron .", "start:local": "cross-env electron .", - "dist": "electron-builder", + "dist": "electron-builder --publish never", "build": "cd ../packages/api && yarn build && cd ../web && yarn build && cd ../../app && yarn dist", "build:local": "cd ../packages/api && yarn build && cd ../web && yarn build && cd ../../app && yarn predist", "postinstall": "yarn rebuild && patch-package", @@ -128,7 +128,7 @@ "devDependencies": { "copyfiles": "^2.2.0", "cross-env": "^6.0.3", - "electron": "30.0.2", + "electron": "38.6.0", "electron-builder": "25.1.8" } } diff --git a/app/src/electron.js b/app/src/electron.js index 6340a2b74..08fbf12af 100644 --- a/app/src/electron.js +++ b/app/src/electron.js @@ -31,6 +31,16 @@ let mainModule; let appUpdateStatus = ''; let settingsJson = {}; +function getTranslated(key) { + if (typeof key === 'string' && global.TRANSLATION_DATA?.[key]) { + return global.TRANSLATION_DATA?.[key]; + } + if (typeof key?._transKey === 'string') { + return global.TRANSLATION_DATA?.[key._transKey] ?? key._transOptions?.defaultMessage; + } + return key; +} + process.on('uncaughtException', function (error) { console.error('uncaughtException', error); }); @@ -63,6 +73,7 @@ try { let mainWindow; let mainMenu; let runCommandOnLoad = null; +let mainWindowMenuSet = false; log.transports.file.level = 'debug'; autoUpdater.logger = log; @@ -85,17 +96,22 @@ function formatKeyText(keyText) { return keyText.replace('CtrlOrCommand+', 'Ctrl+'); } -function commandItem(item) { +function commandItem(item, disableAll = false) { const id = item.command; const command = commands[id]; if (item.skipInApp) { return { skip: true }; } + if (!command) { + return { skip: true }; + } return { id, - label: command ? command.menuName || command.toolbarName || command.name : id, + label: command + ? getTranslated(command.menuName) || getTranslated(command.toolbarName) || getTranslated(command.name) + : id, accelerator: formatKeyText(command ? command.keyText : undefined), - enabled: command ? command.enabled : false, + enabled: command ? command.enabled && (!disableAll || command.systemCommand) : false, click() { if (mainWindow) { mainWindow.webContents.send('run-command', id); @@ -107,14 +123,14 @@ function commandItem(item) { }; } -function buildMenu() { +function buildMenu(disableAll = false) { let template = _cloneDeepWith(mainMenuDefinition({ editMenu: true, isMac: isMac() }), item => { if (item.divider) { return { type: 'separator' }; } if (item.command) { - return commandItem(item); + return commandItem(item, disableAll); } }); @@ -129,7 +145,7 @@ function buildMenu() { { label: 'DbGate', submenu: [ - commandItem({ command: 'about.show' }), + commandItem({ command: 'about.show' }, disableAll), { role: 'services' }, { role: 'hide' }, { role: 'hideOthers' }, @@ -145,22 +161,28 @@ function buildMenu() { } ipcMain.on('update-commands', async (event, arg) => { - commands = JSON.parse(arg); + const parsed = JSON.parse(arg); + commands = parsed.commands; + const isModalOpened = parsed.isModalOpened; + const dbgatePage = parsed.dbgatePage; for (const key of Object.keys(commands)) { const menu = mainMenu.getMenuItemById(key); if (!menu) continue; const command = commands[key]; // rebuild menu - if (menu.label != command.text || menu.accelerator != command.keyText) { - mainMenu = buildMenu(); + if (global.TRANSLATION_DATA && (menu.label != command.text || menu.accelerator != command.keyText)) { + mainMenu = buildMenu(isModalOpened || !!dbgatePage); Menu.setApplicationMenu(mainMenu); - // mainWindow.setMenu(mainMenu); + if (!mainWindowMenuSet) { + mainWindow.setMenu(mainMenu); + mainWindowMenuSet = true; + } return; } - menu.enabled = command.enabled; + menu.enabled = command.enabled && !isModalOpened && !dbgatePage; } }); ipcMain.on('quit-app', async (event, arg) => { @@ -212,6 +234,10 @@ ipcMain.on('app-started', async (event, arg) => { autoUpdater.checkForUpdates(); } } + + if (initialConfig['winZoomLevel'] != null) { + mainWindow.webContents.zoomLevel = initialConfig['winZoomLevel']; + } }); ipcMain.on('window-action', async (event, arg) => { if (!mainWindow) { @@ -299,6 +325,12 @@ ipcMain.on('check-for-updates', async (event, url) => { autoUpdater.autoDownload = false; autoUpdater.checkForUpdates(); }); +ipcMain.on('translation-data', async (event, arg) => { + global.TRANSLATION_DATA = JSON.parse(arg); + mainMenu = buildMenu(); + Menu.setApplicationMenu(mainMenu); + mainWindow.setMenu(mainMenu); +}); function fillMissingSettings(value) { const res = { @@ -375,8 +407,8 @@ function createWindow() { mainWindow.setFullScreen(true); } - mainMenu = buildMenu(); - mainWindow.setMenu(mainMenu); + // mainMenu = buildMenu(); + // mainWindow.setMenu(mainMenu); function loadMainWindow() { const startUrl = @@ -394,6 +426,7 @@ function createWindow() { JSON.stringify({ winBounds: mainWindow.getBounds(), winIsMaximized: mainWindow.isMaximized(), + winZoomLevel: mainWindow.webContents.zoomLevel, }), 'utf-8' ); diff --git a/app/src/mainMenuDefinition.js b/app/src/mainMenuDefinition.js index c463af4bf..bb3696c94 100644 --- a/app/src/mainMenuDefinition.js +++ b/app/src/mainMenuDefinition.js @@ -1,6 +1,10 @@ -module.exports = ({ editMenu, isMac }) => [ +function _t(key, { defaultMessage, currentTranslations } = {}) { + return (currentTranslations || global.TRANSLATION_DATA)?.[key] || defaultMessage; +} + +module.exports = ({ editMenu, isMac }, currentTranslations = null) => [ { - label: 'File', + label: _t('menu.file', { defaultMessage: 'File', currentTranslations }), submenu: [ { command: 'new.connection', hideDisabled: true }, { command: 'new.sqliteDatabase', hideDisabled: true }, @@ -10,6 +14,7 @@ module.exports = ({ editMenu, isMac }) => [ { command: 'new.queryDesign', hideDisabled: true }, { command: 'new.diagram', hideDisabled: true }, { command: 'new.perspective', hideDisabled: true }, + { command: 'new.application', hideDisabled: true }, { command: 'new.shell', hideDisabled: true }, { command: 'new.jsonl', hideDisabled: true }, { command: 'new.modelTransform', hideDisabled: true }, @@ -27,7 +32,7 @@ module.exports = ({ editMenu, isMac }) => [ }, editMenu ? { - label: 'Edit', + label: _t('menu.edit', { defaultMessage: 'Edit', currentTranslations }), submenu: [ { command: 'edit.undo' }, { command: 'edit.redo' }, @@ -52,7 +57,7 @@ module.exports = ({ editMenu, isMac }) => [ // ], // }, { - label: 'View', + label: _t('menu.view', { defaultMessage: 'View', currentTranslations }), submenu: [ { command: 'app.reload', hideDisabled: true }, { command: 'app.toggleDevTools', hideDisabled: true }, @@ -71,10 +76,12 @@ module.exports = ({ editMenu, isMac }) => [ { command: 'app.zoomIn', hideDisabled: true }, { command: 'app.zoomOut', hideDisabled: true }, { command: 'app.zoomReset', hideDisabled: true }, + { divider: true }, + { command: 'app.showLogs', hideDisabled: true }, ], }, { - label: 'Tools', + label: _t('menu.tools', { defaultMessage: 'Tools', currentTranslations }), submenu: [ { command: 'database.search', hideDisabled: true }, { command: 'commandPalette.show', hideDisabled: true }, @@ -86,11 +93,12 @@ module.exports = ({ editMenu, isMac }) => [ { divider: true }, { command: 'folder.showLogs', hideDisabled: true }, { command: 'folder.showData', hideDisabled: true }, - { command: 'new.gist', hideDisabled: true }, { command: 'app.resetSettings', hideDisabled: true }, { divider: true }, { command: 'app.exportConnections', hideDisabled: true }, { command: 'app.importConnections', hideDisabled: true }, + { divider: true }, + { command: 'app.managePlugins', hideDisabled: true }, ], }, ...(isMac @@ -102,19 +110,19 @@ module.exports = ({ editMenu, isMac }) => [ ] : []), { - label: 'Help', + label: _t('menu.help', { defaultMessage: 'Help', currentTranslations }), submenu: [ { command: 'app.openDocs', hideDisabled: true }, { command: 'app.openWeb', hideDisabled: true }, { command: 'app.openIssue', hideDisabled: true }, { command: 'app.openSponsoring', hideDisabled: true }, - { command: 'app.giveFeedback', hideDisabled: true }, + // { command: 'app.giveFeedback', hideDisabled: true }, { divider: true }, { command: 'settings.commands', hideDisabled: true }, { command: 'tabs.changelog', hideDisabled: true }, { command: 'about.show', hideDisabled: true }, { divider: true }, - { command: 'file.checkForUpdates', hideDisabled: true }, + { command: 'app.checkForUpdates', hideDisabled: true }, ], }, ]; diff --git a/app/yarn.lock b/app/yarn.lock index fc23b421a..9b55f6bc6 100644 --- a/app/yarn.lock +++ b/app/yarn.lock @@ -16,9 +16,9 @@ ajv-keywords "^3.4.1" "@electron/asar@^3.2.7": - version "3.2.17" - resolved "https://registry.yarnpkg.com/@electron/asar/-/asar-3.2.17.tgz#91d28087aad80d1a1c8cc4e667c6476edf50f949" - integrity sha512-OcWImUI686w8LkghQj9R2ynZ2ME693Ek6L1SiaAgqGKzBaTIZw3fHDqN82Rcl+EU1Gm9EgkJ5KLIY/q5DCRbbA== + version "3.4.1" + resolved "https://registry.yarnpkg.com/@electron/asar/-/asar-3.4.1.tgz#4e9196a4b54fba18c56cd8d5cac67c5bdc588065" + integrity sha512-i4/rNPRS84t0vSRa2HorerGRXWyF4vThfHesw0dmcWHp+cspK743UanA0suA5Q5y8kzY2y6YKrvbIUn69BCAiA== dependencies: commander "^5.0.0" glob "^7.1.6" @@ -98,6 +98,18 @@ resolved "https://registry.yarnpkg.com/@gar/promisify/-/promisify-1.1.3.tgz#555193ab2e3bb3b6adc3d551c9c030d9e860daf6" integrity sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw== +"@isaacs/balanced-match@^4.0.1": + version "4.0.1" + resolved "https://registry.yarnpkg.com/@isaacs/balanced-match/-/balanced-match-4.0.1.tgz#3081dadbc3460661b751e7591d7faea5df39dd29" + integrity sha512-yzMTt9lEb8Gv7zRioUilSglI0c0smZ9k5D65677DLWLtWJaXIS3CqcGyUFByYKlnUj6TkjLVs54fBl6+TiGQDQ== + +"@isaacs/brace-expansion@^5.0.0": + version "5.0.0" + resolved "https://registry.yarnpkg.com/@isaacs/brace-expansion/-/brace-expansion-5.0.0.tgz#4b3dabab7d8e75a429414a96bd67bf4c1d13e0f3" + integrity sha512-ZT55BDLV0yv0RBm2czMiZ+SqCGO7AvmOM3G/w2xhVPH+te0aKgFjmBvGlL1dH+ql2tgGO3MVrbb3jCKyvpgnxA== + dependencies: + "@isaacs/balanced-match" "^4.0.1" + "@isaacs/cliui@^8.0.2": version "8.0.2" resolved "https://registry.yarnpkg.com/@isaacs/cliui/-/cliui-8.0.2.tgz#b37667b7bc181c168782259bab42474fbf52b550" @@ -202,16 +214,23 @@ "@types/node" "*" "@types/ms@*": - version "0.7.34" - resolved "https://registry.yarnpkg.com/@types/ms/-/ms-0.7.34.tgz#10964ba0dee6ac4cd462e2795b6bebd407303433" - integrity sha512-nG96G3Wp6acyAgJqGasjODb+acrI7KltPiRxzHPXnP3NgI28bpQDRv53olbqGXbfcgF5aiiHmO3xpwEpS5Ld9g== + version "2.1.0" + resolved "https://registry.yarnpkg.com/@types/ms/-/ms-2.1.0.tgz#052aa67a48eccc4309d7f0191b7e41434b90bb78" + integrity sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA== -"@types/node@*", "@types/node@^20.9.0": - version "20.12.10" - resolved "https://registry.yarnpkg.com/@types/node/-/node-20.12.10.tgz#8f0c3f12b0f075eee1fe20c1afb417e9765bef76" - integrity sha512-Eem5pH9pmWBHoGAT8Dr5fdc5rYA+4NAovdM4EktRPVAAiJhmWWfQrA0cFhAbOsQdSfIHjAud6YdkbL69+zSKjw== +"@types/node@*": + version "24.10.0" + resolved "https://registry.yarnpkg.com/@types/node/-/node-24.10.0.tgz#6b79086b0dfc54e775a34ba8114dcc4e0221f31f" + integrity sha512-qzQZRBqkFsYyaSWXuEHc2WR9c0a0CXwiE5FWUvn7ZM+vdy1uZLfCunD38UzhuB7YN/J11ndbDBcTmOdxJo9Q7A== dependencies: - undici-types "~5.26.4" + undici-types "~7.16.0" + +"@types/node@^22.7.7": + version "22.19.0" + resolved "https://registry.yarnpkg.com/@types/node/-/node-22.19.0.tgz#849606ef3920850583a4e7ee0930987c35ad80be" + integrity sha512-xpr/lmLPQEj+TUnHmR+Ab91/glhJvsqcjB+yY0Ix9GO70H6Lb4FHH5GeqdOE5btAx7eIMwuHkp4H2MSkLcqWbA== + dependencies: + undici-types "~6.21.0" "@types/plist@^3.0.1": version "3.0.5" @@ -229,9 +248,9 @@ "@types/node" "*" "@types/verror@^1.10.3": - version "1.10.10" - resolved "https://registry.yarnpkg.com/@types/verror/-/verror-1.10.10.tgz#d5a4b56abac169bfbc8b23d291363a682e6fa087" - integrity sha512-l4MM0Jppn18hb9xmM6wwD1uTdShpf9Pn80aXTStnK1C94gtPvJcV2FrDmbOQUAQfJ1cKZHktkQUDwEqaAKXMMg== + version "1.10.11" + resolved "https://registry.yarnpkg.com/@types/verror/-/verror-1.10.11.tgz#d3d6b418978c8aa202d41e5bb3483227b6ecc1bb" + integrity sha512-RlDm9K7+o5stv0Co8i8ZRGxDbrTxhJtgjqjFyVh/tXQyl/rYtTKlnTvZ88oSTeYREWurwx20Js4kTuKCsFkUtg== "@types/yauzl@^2.9.1": version "2.10.3" @@ -241,9 +260,9 @@ "@types/node" "*" "@xmldom/xmldom@^0.8.8": - version "0.8.10" - resolved "https://registry.yarnpkg.com/@xmldom/xmldom/-/xmldom-0.8.10.tgz#a1337ca426aa61cef9fe15b5b28e340a72f6fa99" - integrity sha512-2WALfTl4xo2SkGCYRt6rDTFfk9R1czmBvUQy12gK2KuRKIpWEhcbbzy8EZXtz/jkRqHX8bFEc6FC1HjX4TUWYw== + version "0.8.11" + resolved "https://registry.yarnpkg.com/@xmldom/xmldom/-/xmldom-0.8.11.tgz#b79de2d67389734c57c52595f7a7305e30c2d608" + integrity sha512-cQzWCtO6C8TQiYl1ruKNn2U6Ao4o4WBBcbL61yJl84x+j5sOWWFU9X7DpND8XZG3daDppSsigMdfAIl2upQBRw== "@yarnpkg/lockfile@^1.1.0": version "1.1.0" @@ -263,14 +282,14 @@ agent-base@6, agent-base@^6.0.2: debug "4" agent-base@^7.1.0, agent-base@^7.1.2: - version "7.1.3" - resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-7.1.3.tgz#29435eb821bc4194633a5b89e5bc4703bafc25a1" - integrity sha512-jRR5wdylq8CkOe6hei19GGZnxM6rBGwFl3Bg0YItGDimvjGtAvdZk4Pu6Cl4u4Igsws4a1fd1Vq3ezrhn4KmFw== + version "7.1.4" + resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-7.1.4.tgz#e3cd76d4c548ee895d3c3fd8dc1f6c5b9032e7a8" + integrity sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ== agentkeepalive@^4.2.1: - version "4.5.0" - resolved "https://registry.yarnpkg.com/agentkeepalive/-/agentkeepalive-4.5.0.tgz#2673ad1389b3c418c5a20c5d7364f93ca04be923" - integrity sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew== + version "4.6.0" + resolved "https://registry.yarnpkg.com/agentkeepalive/-/agentkeepalive-4.6.0.tgz#35f73e94b3f40bf65f105219c623ad19c136ea6a" + integrity sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ== dependencies: humanize-ms "^1.2.1" @@ -303,9 +322,9 @@ ansi-regex@^5.0.1: integrity sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ== ansi-regex@^6.0.1: - version "6.1.0" - resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-6.1.0.tgz#95ec409c69619d6cb1b8b34f14b660ef28ebd654" - integrity sha512-7HSX4QQb4CspciLpVFwyRe79O3xsIZDDLER21kERQ71oaPodF8jL725AgJMFAYbooIqolJoRLuM81SpeUkpkvA== + version "6.2.2" + resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-6.2.2.tgz#60216eea464d864597ce2832000738a0589650c1" + integrity sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg== ansi-styles@^4.0.0, ansi-styles@^4.1.0: version "4.3.0" @@ -315,9 +334,9 @@ ansi-styles@^4.0.0, ansi-styles@^4.1.0: color-convert "^2.0.1" ansi-styles@^6.1.0: - version "6.2.1" - resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-6.2.1.tgz#0e62320cf99c21afff3b3012192546aacbfb05c5" - integrity sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug== + version "6.2.3" + resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-6.2.3.tgz#c044d5dcc521a076413472597a1acb1f103c4041" + integrity sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg== app-builder-bin@5.0.0-alpha.10: version "5.0.0-alpha.10" @@ -363,9 +382,9 @@ app-builder-lib@25.1.8: temp-file "^3.4.0" "aproba@^1.0.3 || ^2.0.0": - version "2.0.0" - resolved "https://registry.yarnpkg.com/aproba/-/aproba-2.0.0.tgz#52520b8ae5b569215b354efc0caa3fe1e45a8adc" - integrity sha512-lYe4Gx7QT+MKGbDsA+Z+he/Wtef0BiwDOlK/XkBrdfsh9J/jPPXbX0tE9x9cl27Tmu5gg3QUbUrQYa/y+KOHPQ== + version "2.1.0" + resolved "https://registry.yarnpkg.com/aproba/-/aproba-2.1.0.tgz#75500a190313d95c64e871e7e4284c6ac219f0b1" + integrity sha512-tLIEcj5GuR2RSTnxNKdkK0dJ/GrC7P38sUkiDmDuHfsHmbagTFAxDVIBltoklXEVIQ/f14IL8IMJ5pn9Hez1Ew== are-we-there-yet@^3.0.0: version "3.0.1" @@ -395,10 +414,10 @@ async-exit-hook@^2.0.1: resolved "https://registry.yarnpkg.com/async-exit-hook/-/async-exit-hook-2.0.1.tgz#8bd8b024b0ec9b1c01cccb9af9db29bd717dfaf3" integrity sha512-NW2cX8m1Q7KPA7a5M2ULQeZ2wR5qI5PAbw5L0UOMxdioVk9PMZ0h1TmyZEkPYrCvYjDlFICusOu1dlEKAAeXBw== -async@^3.2.3: - version "3.2.5" - resolved "https://registry.yarnpkg.com/async/-/async-3.2.5.tgz#ebd52a8fdaf7a2289a24df399f8d8485c8a46b66" - integrity sha512-baNZyqaaLhyLVKm/DlvdW051MSgO6b8eVfIezl9E5PqWxFgzLm/wQntEW4zOytVburDEr0JlALEpdOFwvErLsg== +async@^3.2.6: + version "3.2.6" + resolved "https://registry.yarnpkg.com/async/-/async-3.2.6.tgz#1b0728e14929d51b85b449b7f06e27c1145e38ce" + integrity sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA== asynckit@^0.4.0: version "0.4.0" @@ -447,33 +466,33 @@ boolean@^3.0.1: integrity sha512-d0II/GO9uf9lfUHH2BQsjxzRJZBdsjgsBiW4BvhWk/3qoKwQFjIDVN19PfX8F2D/r9PCMTtLWjYVCFrpeYUzsw== brace-expansion@^1.1.7: - version "1.1.11" - resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd" - integrity sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA== + version "1.1.12" + resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.12.tgz#ab9b454466e5a8cc3a187beaad580412a9c5b843" + integrity sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg== dependencies: balanced-match "^1.0.0" concat-map "0.0.1" brace-expansion@^2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-2.0.1.tgz#1edc459e0f0c548486ecf9fc99f2221364b9a0ae" - integrity sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA== + version "2.0.2" + resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-2.0.2.tgz#54fc53237a613d854c7bd37463aad17df87214e7" + integrity sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ== dependencies: balanced-match "^1.0.0" -braces@^3.0.2: - version "3.0.2" - resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.2.tgz#3454e1a462ee8d599e236df336cd9ea4f8afe107" - integrity sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A== +braces@^3.0.3: + version "3.0.3" + resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.3.tgz#490332f40919452272d55a8480adc0c441358789" + integrity sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA== dependencies: - fill-range "^7.0.1" + fill-range "^7.1.1" buffer-crc32@~0.2.3: version "0.2.13" resolved "https://registry.yarnpkg.com/buffer-crc32/-/buffer-crc32-0.2.13.tgz#0d333e3f00eac50aa1454abd30ef8c2a5d9a7242" integrity sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ== -buffer-equal-constant-time@1.0.1: +buffer-equal-constant-time@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz#f8e71132f7ffe6e01a5c9697a4c6f3e48d5cc819" integrity sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA== @@ -499,10 +518,10 @@ builder-util-runtime@9.2.10: debug "^4.3.4" sax "^1.2.4" -builder-util-runtime@9.2.5: - version "9.2.5" - resolved "https://registry.yarnpkg.com/builder-util-runtime/-/builder-util-runtime-9.2.5.tgz#0afdffa0adb5c84c14926c7dd2cf3c6e96e9be83" - integrity sha512-HjIDfhvqx/8B3TDN4GbABQcgpewTU4LMRTQPkVpKYV3lsuxEJoIfvg09GyWTNmfVNSUAYf+fbTN//JX4TH20pg== +builder-util-runtime@9.3.1: + version "9.3.1" + resolved "https://registry.yarnpkg.com/builder-util-runtime/-/builder-util-runtime-9.3.1.tgz#0daedde0f6d381f2a00a50a407b166fe7dca1a67" + integrity sha512-2/egrNDDnRaxVwK3A+cJq6UOlqOdedGA7JPqCeJjN2Zjk1/QB/6QUi3b714ScIGS7HafFXTyzJEOr5b44I3kvQ== dependencies: debug "^4.3.4" sax "^1.2.4" @@ -571,7 +590,15 @@ cacheable-request@^7.0.2: normalize-url "^6.0.1" responselike "^2.0.0" -chalk@^4.0.0, chalk@^4.0.2, chalk@^4.1.0, chalk@^4.1.2: +call-bind-apply-helpers@^1.0.1, call-bind-apply-helpers@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz#4b5428c222be985d79c3d82657479dbe0b59b2d6" + integrity sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ== + dependencies: + es-errors "^1.3.0" + function-bind "^1.1.2" + +chalk@^4.0.0, chalk@^4.1.0, chalk@^4.1.2: version "4.1.2" resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.2.tgz#aac4e2b7734a740867aeb16bf02aad556a1e7a01" integrity sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA== @@ -744,9 +771,9 @@ cross-env@^6.0.3: cross-spawn "^7.0.0" cross-spawn@^6.0.5: - version "6.0.5" - resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-6.0.5.tgz#4a5ec7c64dfae22c3a14124dbacdee846d80cbc4" - integrity sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ== + version "6.0.6" + resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-6.0.6.tgz#30d0efa0712ddb7eb5a76e1e8721bffafa6b5d57" + integrity sha512-VqCUuhcd1iB+dsv8gxPttb5iZh/D0iubSP21g36KXdEuf6I5JiioesUVjpCdHV9MZRUfVFlvwtIUyPfxo5trtw== dependencies: nice-try "^1.0.4" path-key "^2.0.1" @@ -754,26 +781,19 @@ cross-spawn@^6.0.5: shebang-command "^1.2.0" which "^1.2.9" -cross-spawn@^7.0.0, cross-spawn@^7.0.1, cross-spawn@^7.0.3: - version "7.0.3" - resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6" - integrity sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w== +cross-spawn@^7.0.0, cross-spawn@^7.0.1, cross-spawn@^7.0.3, cross-spawn@^7.0.6: + version "7.0.6" + resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.6.tgz#8a58fe78f00dcd70c370451759dfbfaf03e8ee9f" + integrity sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA== dependencies: path-key "^3.1.0" shebang-command "^2.0.0" which "^2.0.1" -debug@4, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.4: - version "4.3.4" - resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.4.tgz#1319f6579357f2338d3337d2cdd4914bb5dcc865" - integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ== - dependencies: - ms "2.1.2" - -debug@^4.3.3: - version "4.4.0" - resolved "https://registry.yarnpkg.com/debug/-/debug-4.4.0.tgz#2b3f2aea2ffeb776477460267377dc8710faba8a" - integrity sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA== +debug@4, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.3, debug@^4.3.4: + version "4.4.3" + resolved "https://registry.yarnpkg.com/debug/-/debug-4.4.3.tgz#c6ae432d9bd9662582fce08709b038c58e9e3d6a" + integrity sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA== dependencies: ms "^2.1.3" @@ -825,9 +845,9 @@ delegates@^1.0.0: integrity sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ== detect-libc@^2.0.1: - version "2.0.3" - resolved "https://registry.yarnpkg.com/detect-libc/-/detect-libc-2.0.3.tgz#f0cd503b40f9939b894697d19ad50895e30cf700" - integrity sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw== + version "2.1.2" + resolved "https://registry.yarnpkg.com/detect-libc/-/detect-libc-2.1.2.tgz#689c5dcdc1900ef5583a4cb9f6d7b473742074ad" + integrity sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ== detect-node@^2.0.4: version "2.1.0" @@ -878,9 +898,18 @@ dotenv-expand@^11.0.6: dotenv "^16.4.5" dotenv@^16.4.5: - version "16.4.7" - resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.4.7.tgz#0e20c5b82950140aa99be360a8a5f52335f53c26" - integrity sha512-47qPchRCykZC03FhkYAhrvwU4xDBFIj1QPqaarj6mdM/hgUzfPHcpkHJOn3mJAufFeeAxAzeGsr5X0M4k6fLZQ== + version "16.6.1" + resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.6.1.tgz#773f0e69527a8315c7285d5ee73c4459d20a8020" + integrity sha512-uBq4egWHTcTt33a72vpSG0z3HnPuIl6NqYcTrKEg2azoEyl2hpW0zqlxysq2pK9HlDIHyHyakeYaYnSAwd8bow== + +dunder-proto@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/dunder-proto/-/dunder-proto-1.0.1.tgz#d7ae667e1dc83482f8b70fd0f6eefc50da30f58a" + integrity sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A== + dependencies: + call-bind-apply-helpers "^1.0.1" + es-errors "^1.3.0" + gopd "^1.2.0" eastasianwidth@^0.2.0: version "0.2.0" @@ -936,11 +965,11 @@ electron-publish@25.1.7: mime "^2.5.2" electron-updater@^6.3.4: - version "6.3.4" - resolved "https://registry.yarnpkg.com/electron-updater/-/electron-updater-6.3.4.tgz#3934bc89875bb524c2cbbd11041114e97c0c2496" - integrity sha512-uZUo7p1Y53G4tl6Cgw07X1yF8Jlz6zhaL7CQJDZ1fVVkOaBfE2cWtx80avwDVi8jHp+I/FWawrMgTAeCCNIfAg== + version "6.6.2" + resolved "https://registry.yarnpkg.com/electron-updater/-/electron-updater-6.6.2.tgz#3e65e044f1a99b00d61e200e24de8e709c69ce99" + integrity sha512-Cr4GDOkbAUqRHP5/oeOmH/L2Bn6+FQPxVLZtPbcmKZC63a1F3uu5EefYOssgZXG3u/zBlubbJ5PJdITdMVggbw== dependencies: - builder-util-runtime "9.2.5" + builder-util-runtime "9.3.1" fs-extra "^10.1.0" js-yaml "^4.1.0" lazy-val "^1.0.5" @@ -949,13 +978,13 @@ electron-updater@^6.3.4: semver "^7.6.3" tiny-typed-emitter "^2.1.0" -electron@30.0.2: - version "30.0.2" - resolved "https://registry.yarnpkg.com/electron/-/electron-30.0.2.tgz#95ba019216bf8be9f3097580123e33ea37497733" - integrity sha512-zv7T+GG89J/hyWVkQsLH4Y/rVEfqJG5M/wOBIGNaDdqd8UV9/YZPdS7CuFeaIj0H9LhCt95xkIQNpYB/3svOkQ== +electron@38.6.0: + version "38.6.0" + resolved "https://registry.yarnpkg.com/electron/-/electron-38.6.0.tgz#c862bff41d42776e307bf5cc92503dda23612339" + integrity sha512-68OFNxJlrEStA+t8k5atzf4frJddvRR1N1oalr49Ll8YZ0+0nEsDhw4UNhTCoZKTjSYcxFF/4rt+sco+OlnB3g== dependencies: "@electron/get" "^2.0.0" - "@types/node" "^20.9.0" + "@types/node" "^22.7.7" extract-zip "^2.0.1" emoji-regex@^8.0.0: @@ -976,9 +1005,9 @@ encoding@^0.1.13: iconv-lite "^0.6.2" end-of-stream@^1.1.0: - version "1.4.4" - resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.4.tgz#5ae64a5f45057baf3626ec14da0ca5e4b2431eb0" - integrity sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q== + version "1.4.5" + resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.5.tgz#7344d711dea40e0b74abc2ed49778743ccedb08c" + integrity sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg== dependencies: once "^1.4.0" @@ -992,27 +1021,42 @@ err-code@^2.0.2: resolved "https://registry.yarnpkg.com/err-code/-/err-code-2.0.3.tgz#23c2f3b756ffdfc608d30e27c9a941024807e7f9" integrity sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA== -es-define-property@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/es-define-property/-/es-define-property-1.0.0.tgz#c7faefbdff8b2696cf5f46921edfb77cc4ba3845" - integrity sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ== - dependencies: - get-intrinsic "^1.2.4" +es-define-property@^1.0.0, es-define-property@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/es-define-property/-/es-define-property-1.0.1.tgz#983eb2f9a6724e9303f61addf011c72e09e0b0fa" + integrity sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g== es-errors@^1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/es-errors/-/es-errors-1.3.0.tgz#05f75a25dab98e4fb1dcd5e1472c0546d5057c8f" integrity sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw== +es-object-atoms@^1.0.0, es-object-atoms@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/es-object-atoms/-/es-object-atoms-1.1.1.tgz#1c4f2c4837327597ce69d2ca190a7fdd172338c1" + integrity sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA== + dependencies: + es-errors "^1.3.0" + +es-set-tostringtag@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz#f31dbbe0c183b00a6d26eb6325c810c0fd18bd4d" + integrity sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA== + dependencies: + es-errors "^1.3.0" + get-intrinsic "^1.2.6" + has-tostringtag "^1.0.2" + hasown "^2.0.2" + es6-error@^4.1.1: version "4.1.1" resolved "https://registry.yarnpkg.com/es6-error/-/es6-error-4.1.1.tgz#9e3af407459deed47e9a91f9b885a84eb05c561d" integrity sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg== escalade@^3.1.1: - version "3.1.2" - resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.2.tgz#54076e9ab29ea5bf3d8f1ed62acffbb88272df27" - integrity sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA== + version "3.2.0" + resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.2.0.tgz#011a3f69856ba189dffa7dc8fcce99d2a87903e5" + integrity sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA== escape-string-regexp@^4.0.0: version "4.0.0" @@ -1020,9 +1064,9 @@ escape-string-regexp@^4.0.0: integrity sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA== exponential-backoff@^3.1.1: - version "3.1.1" - resolved "https://registry.yarnpkg.com/exponential-backoff/-/exponential-backoff-3.1.1.tgz#64ac7526fe341ab18a39016cd22c787d01e00bf6" - integrity sha512-dX7e/LHVJ6W3DE1MHWi9S1EYzDESENfLrYohG2G++ovZrYOkm4Knwa0mc1cn84xJOR4KEU0WSchhLbd0UklbHw== + version "3.1.3" + resolved "https://registry.yarnpkg.com/exponential-backoff/-/exponential-backoff-3.1.3.tgz#51cf92c1c0493c766053f9d3abee4434c244d2f6" + integrity sha512-ZgEeZXj30q+I0EN+CbSSpIyPaJ5HVQD18Z1m+u1FXbAeT94mr1zw50q4q6jiiC447Nl/YTcIYSAftiGqetwXCA== extract-zip@^2.0.1: version "2.0.1" @@ -1064,10 +1108,10 @@ filelist@^1.0.4: dependencies: minimatch "^5.0.1" -fill-range@^7.0.1: - version "7.0.1" - resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.0.1.tgz#1919a6a7c75fe38b2c7c77e5198535da9acdda40" - integrity sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ== +fill-range@^7.1.1: + version "7.1.1" + resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.1.1.tgz#44265d3cac07e3ea7dc247516380643754a05292" + integrity sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg== dependencies: to-regex-range "^5.0.1" @@ -1079,20 +1123,22 @@ find-yarn-workspace-root@^2.0.0: micromatch "^4.0.2" foreground-child@^3.1.0: - version "3.3.0" - resolved "https://registry.yarnpkg.com/foreground-child/-/foreground-child-3.3.0.tgz#0ac8644c06e431439f8561db8ecf29a7b5519c77" - integrity sha512-Ld2g8rrAyMYFXBhEqMz8ZAHBi4J4uS1i/CxGMDnjyFWddMXLVcDp051DZfu+t7+ab7Wv6SMqpWmyFIj5UbfFvg== + version "3.3.1" + resolved "https://registry.yarnpkg.com/foreground-child/-/foreground-child-3.3.1.tgz#32e8e9ed1b68a3497befb9ac2b6adf92a638576f" + integrity sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw== dependencies: - cross-spawn "^7.0.0" + cross-spawn "^7.0.6" signal-exit "^4.0.1" form-data@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.0.tgz#93919daeaf361ee529584b9b31664dc12c9fa452" - integrity sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww== + version "4.0.4" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.4.tgz#784cdcce0669a9d68e94d11ac4eea98088edd2c4" + integrity sha512-KrGhL9Q4zjj0kiUt5OO4Mr/A/jlI2jDYs5eHBpYHPcBEVSiipAvn2Ko2HnPe20rmcuuvMHNdZFp+4IlGTMF0Ow== dependencies: asynckit "^0.4.0" combined-stream "^1.0.8" + es-set-tostringtag "^2.1.0" + hasown "^2.0.2" mime-types "^2.1.12" fs-extra@^10.0.0, fs-extra@^10.1.0: @@ -1105,9 +1151,9 @@ fs-extra@^10.0.0, fs-extra@^10.1.0: universalify "^2.0.0" fs-extra@^11.1.1: - version "11.2.0" - resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-11.2.0.tgz#e70e17dfad64232287d01929399e0ea7c86b0e5b" - integrity sha512-PmDi3uwK5nFuXh7XDTlVnS17xJS7vW36is2+w3xcv8SVxiB4NyATf4ctkVY5bkSjX0Y4nbvZCq1/EjtEyr9ktw== + version "11.3.2" + resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-11.3.2.tgz#c838aeddc6f4a8c74dd15f85e11fe5511bfe02a4" + integrity sha512-Xr9F6z6up6Ws+NjzMCZc6WXg2YFRlrLP9NQDO3VQrWrfiojdhS56TzueT88ze0uBdCTwEIhQ3ptnmKeWGFAe0A== dependencies: graceful-fs "^4.2.0" jsonfile "^6.0.1" @@ -1168,16 +1214,29 @@ get-caller-file@^2.0.5: resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-2.0.5.tgz#4f94412a82db32f36e3b0b9741f8a97feb031f7e" integrity sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg== -get-intrinsic@^1.1.3, get-intrinsic@^1.2.4: - version "1.2.4" - resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.2.4.tgz#e385f5a4b5227d449c3eabbad05494ef0abbeadd" - integrity sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ== +get-intrinsic@^1.2.6: + version "1.3.0" + resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.3.0.tgz#743f0e3b6964a93a5491ed1bffaae054d7f98d01" + integrity sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ== dependencies: + call-bind-apply-helpers "^1.0.2" + es-define-property "^1.0.1" es-errors "^1.3.0" + es-object-atoms "^1.1.1" function-bind "^1.1.2" - has-proto "^1.0.1" - has-symbols "^1.0.3" - hasown "^2.0.0" + get-proto "^1.0.1" + gopd "^1.2.0" + has-symbols "^1.1.0" + hasown "^2.0.2" + math-intrinsics "^1.1.0" + +get-proto@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/get-proto/-/get-proto-1.0.1.tgz#150b3f2743869ef3e851ec0c49d15b1d14d00ee1" + integrity sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g== + dependencies: + dunder-proto "^1.0.1" + es-object-atoms "^1.0.0" get-stream@^5.1.0: version "5.2.0" @@ -1241,12 +1300,10 @@ globalthis@^1.0.1: define-properties "^1.2.1" gopd "^1.0.1" -gopd@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/gopd/-/gopd-1.0.1.tgz#29ff76de69dac7489b7c0918a5788e56477c332c" - integrity sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA== - dependencies: - get-intrinsic "^1.1.3" +gopd@^1.0.1, gopd@^1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/gopd/-/gopd-1.2.0.tgz#89f56b8217bdbc8802bd299df6d7f1081d7e51a1" + integrity sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg== got@^11.7.0, got@^11.8.5: version "11.8.6" @@ -1282,22 +1339,24 @@ has-property-descriptors@^1.0.0: dependencies: es-define-property "^1.0.0" -has-proto@^1.0.1: - version "1.0.3" - resolved "https://registry.yarnpkg.com/has-proto/-/has-proto-1.0.3.tgz#b31ddfe9b0e6e9914536a6ab286426d0214f77fd" - integrity sha512-SJ1amZAJUiZS+PhsVLf5tGydlaVB8EdFpaSO4gmiUKUOxk8qzn5AIy4ZeJUmh22znIdk/uMAUT2pl3FxzVUH+Q== +has-symbols@^1.0.3, has-symbols@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/has-symbols/-/has-symbols-1.1.0.tgz#fc9c6a783a084951d0b971fe1018de813707a338" + integrity sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ== -has-symbols@^1.0.3: - version "1.0.3" - resolved "https://registry.yarnpkg.com/has-symbols/-/has-symbols-1.0.3.tgz#bb7b2c4349251dce87b125f7bdf874aa7c8b39f8" - integrity sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A== +has-tostringtag@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/has-tostringtag/-/has-tostringtag-1.0.2.tgz#2cdc42d40bef2e5b4eeab7c01a73c54ce7ab5abc" + integrity sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw== + dependencies: + has-symbols "^1.0.3" has-unicode@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/has-unicode/-/has-unicode-2.0.1.tgz#e0e6fe6a28cf51138855e086d1691e771de2a8b9" integrity sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ== -hasown@^2.0.0: +hasown@^2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/hasown/-/hasown-2.0.2.tgz#003eaf91be7adc372e84ec59dc37252cedb80003" integrity sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ== @@ -1312,9 +1371,9 @@ hosted-git-info@^4.1.0: lru-cache "^6.0.0" http-cache-semantics@^4.0.0, http-cache-semantics@^4.1.0: - version "4.1.1" - resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz#abe02fcb2985460bf0323be664436ec3476a6d5a" - integrity sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ== + version "4.2.0" + resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-4.2.0.tgz#205f4db64f8562b76a4ff9235aa5279839a09dd5" + integrity sha512-dTxcvPXqPvXBQpq5dUr6mEMJX4oIEFv6bwom3FDwKRDsuIjjJGANqhBuoAn9c1RQJIdAKav33ED65E2ys+87QQ== http-proxy-agent@^5.0.0: version "5.0.0" @@ -1412,13 +1471,10 @@ inherits@2, inherits@^2.0.1, inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.1, resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== -ip-address@^9.0.5: - version "9.0.5" - resolved "https://registry.yarnpkg.com/ip-address/-/ip-address-9.0.5.tgz#117a960819b08780c3bd1f14ef3c1cc1d3f3ea5a" - integrity sha512-zHtQzGojZXTwZTHQqra+ETKd4Sn3vgi7uBmlPoXVWZqYvuKmtI0l/VZTjqGmJY9x88GGOaZ9+G9ES8hC4T4X8g== - dependencies: - jsbn "1.1.0" - sprintf-js "^1.1.3" +ip-address@^10.0.1: + version "10.1.0" + resolved "https://registry.yarnpkg.com/ip-address/-/ip-address-10.1.0.tgz#d8dcffb34d0e02eb241427444a6e23f5b0595aa4" + integrity sha512-XXADHxXmvT9+CRxhXg56LJovE+bmWnEWB78LB83VZTprKTmaC5QfruXocxzTZ2Kl0DNwKuBdlIhjL8LeY8Sf8Q== is-ci@^2.0.0: version "2.0.0" @@ -1487,9 +1543,9 @@ isbinaryfile@^4.0.8: integrity sha512-iHrqe5shvBUcFbmZq9zOQHBoeOhZJu6RQGrDpBgenUm/Am+F3JM2MgQj+rK3Z601fzrL5gLZWtAPH2OBaSVcyw== isbinaryfile@^5.0.0: - version "5.0.4" - resolved "https://registry.yarnpkg.com/isbinaryfile/-/isbinaryfile-5.0.4.tgz#2a2edefa76cafa66613fe4c1ea52f7f031017bdf" - integrity sha512-YKBKVkKhty7s8rxddb40oOkuP0NbaeXrQvLin6QMHL7Ypiy2RW9LwOVrVgZRyOrhQlayMd9t+D8yDy8MKFTSDQ== + version "5.0.6" + resolved "https://registry.yarnpkg.com/isbinaryfile/-/isbinaryfile-5.0.6.tgz#01eac28867aeffaebaee7eaf21d1dd3a67d7c0c7" + integrity sha512-I+NmIfBHUl+r2wcDd6JwE9yWje/PIVY/R5/CmV8dXLZd5K+L9X2klAOwfAHNnondLXkbHyTAleQAWonpTJBTtw== isexe@^2.0.0: version "2.0.0" @@ -1506,14 +1562,13 @@ jackspeak@^3.1.2: "@pkgjs/parseargs" "^0.11.0" jake@^10.8.5: - version "10.9.1" - resolved "https://registry.yarnpkg.com/jake/-/jake-10.9.1.tgz#8dc96b7fcc41cb19aa502af506da4e1d56f5e62b" - integrity sha512-61btcOHNnLnsOdtLgA5efqQWjnSi/vow5HbI7HMdKKWqvrKR1bLK3BPlJn9gcSaP2ewuamUSMB5XEy76KUIS2w== + version "10.9.4" + resolved "https://registry.yarnpkg.com/jake/-/jake-10.9.4.tgz#d626da108c63d5cfb00ab5c25fadc7e0084af8e6" + integrity sha512-wpHYzhxiVQL+IV05BLE2Xn34zW1S223hvjtqk0+gsPrwd/8JNLXJgZZM/iPFsYc1xyphF+6M6EvdE5E9MBGkDA== dependencies: - async "^3.2.3" - chalk "^4.0.2" + async "^3.2.6" filelist "^1.0.4" - minimatch "^3.1.2" + picocolors "^1.1.1" js-yaml@^4.1.0: version "4.1.0" @@ -1522,11 +1577,6 @@ js-yaml@^4.1.0: dependencies: argparse "^2.0.1" -jsbn@1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-1.1.0.tgz#b01307cb29b618a1ed26ec79e911f803c4da0040" - integrity sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A== - json-buffer@3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/json-buffer/-/json-buffer-3.0.1.tgz#9338802a30d3b6605fbe0613e094008ca8c05a13" @@ -1555,9 +1605,9 @@ jsonfile@^4.0.0: graceful-fs "^4.1.6" jsonfile@^6.0.1: - version "6.1.0" - resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-6.1.0.tgz#bc55b2634793c679ec6403094eb13698a6ec0aae" - integrity sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ== + version "6.2.0" + resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-6.2.0.tgz#7c265bd1b65de6977478300087c99f1c84383f62" + integrity sha512-FGuPw30AdOIUTRMC2OMRtQV+jkVj2cfPqSeWXv1NEAJ1qZ5zb1X6z1mFhbfOB/iy3ssJCD+3KuZ8r8C3uVFlAg== dependencies: universalify "^2.0.0" optionalDependencies: @@ -1580,11 +1630,11 @@ jsonwebtoken@^9.0.2: semver "^7.5.4" jwa@^1.4.1: - version "1.4.1" - resolved "https://registry.yarnpkg.com/jwa/-/jwa-1.4.1.tgz#743c32985cb9e98655530d53641b66c8645b039a" - integrity sha512-qiLX/xhEEFKUAJ6FiBMbes3w9ATzyk5W7Hvzpa/SLYdxNtng+gcurvrI7TbACjIXlsJyr05/S1oUhZrc63evQA== + version "1.4.2" + resolved "https://registry.yarnpkg.com/jwa/-/jwa-1.4.2.tgz#16011ac6db48de7b102777e57897901520eec7b9" + integrity sha512-eeH5JO+21J78qMvTIDdBXidBd6nG2kZjg5Ohz/1fpa28Z4CcsWUzJ1ZZyFq/3z3N17aZy+ZuBoHljASbL1WfOw== dependencies: - buffer-equal-constant-time "1.0.1" + buffer-equal-constant-time "^1.0.1" ecdsa-sig-formatter "1.0.11" safe-buffer "^5.0.1" @@ -1729,12 +1779,17 @@ matcher@^3.0.0: dependencies: escape-string-regexp "^4.0.0" +math-intrinsics@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/math-intrinsics/-/math-intrinsics-1.1.0.tgz#a0dd74be81e2aa5c2f27e65ce283605ee4e2b7f9" + integrity sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g== + micromatch@^4.0.2: - version "4.0.5" - resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.5.tgz#bc8999a7cbbf77cdc89f132f6e467051b49090c6" - integrity sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA== + version "4.0.8" + resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.8.tgz#d66fa18f3a47076789320b9b1af32bd86d9fa202" + integrity sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA== dependencies: - braces "^3.0.2" + braces "^3.0.3" picomatch "^2.3.1" mime-db@1.52.0: @@ -1770,13 +1825,13 @@ mimic-response@^3.1.0: integrity sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ== minimatch@^10.0.0: - version "10.0.1" - resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-10.0.1.tgz#ce0521856b453c86e25f2c4c0d03e6ff7ddc440b" - integrity sha512-ethXTt3SGGR+95gudmqJ1eNhRO7eGEGIgYA9vnPatK4/etz2MEVDno5GMCibdMTuBMyElzIlgxMna3K94XDIDQ== + version "10.1.1" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-10.1.1.tgz#e6e61b9b0c1dcab116b5a7d1458e8b6ae9e73a55" + integrity sha512-enIvLvRAFZYXJzkCYG5RKmPfrFArdLv+R+lbQ53BmIMLIry74bjKzX6iHAm8WYamJkhSSEabrWN5D97XnKObjQ== dependencies: - brace-expansion "^2.0.1" + "@isaacs/brace-expansion" "^5.0.0" -minimatch@^3.0.3, minimatch@^3.0.4, minimatch@^3.0.5, minimatch@^3.1.1, minimatch@^3.1.2: +minimatch@^3.0.3, minimatch@^3.0.4, minimatch@^3.0.5, minimatch@^3.1.1: version "3.1.2" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.2.tgz#19cd194bfd3e428f049a70817c038d89ab4be35b" integrity sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw== @@ -1871,11 +1926,6 @@ mkdirp@^1.0.3, mkdirp@^1.0.4: resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e" integrity sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw== -ms@2.1.2: - version "2.1.2" - resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" - integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== - ms@^2.0.0, ms@^2.1.1, ms@^2.1.3: version "2.1.3" resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" @@ -1892,9 +1942,9 @@ nice-try@^1.0.4: integrity sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ== node-abi@^3.45.0: - version "3.71.0" - resolved "https://registry.yarnpkg.com/node-abi/-/node-abi-3.71.0.tgz#52d84bbcd8575efb71468fbaa1f9a49b2c242038" - integrity sha512-SZ40vRiy/+wRTf21hxkkEjPJZpARzUMVcJoQse2EF8qkUWbbO2z7vd5oA/H6bVH6SZQ5STGcu0KRDS7biNRfxw== + version "3.80.0" + resolved "https://registry.yarnpkg.com/node-abi/-/node-abi-3.80.0.tgz#d7390951f27caa129cceeec01e1c20fc9f07581c" + integrity sha512-LyPuZJcI9HVwzXK1GPxWNzrr+vr8Hp/3UqlmWxxh8p54U1ZbclOqbSog9lWHaCX+dBaiGi6n/hIX+mKu74GmPA== dependencies: semver "^7.3.5" @@ -1904,9 +1954,9 @@ node-addon-api@^1.6.3: integrity sha512-ibPK3iA+vaY1eEjESkQkM0BbCqFOaZMiXRTtdB0u7b4djtY6JnsjvPdUHVMg6xQt3B8fpTTWHI9A+ADjM9frzg== node-api-version@^0.2.0: - version "0.2.0" - resolved "https://registry.yarnpkg.com/node-api-version/-/node-api-version-0.2.0.tgz#5177441da2b1046a4d4547ab9e0972eed7b1ac1d" - integrity sha512-fthTTsi8CxaBXMaBAD7ST2uylwvsnYxh2PfaScwpMhos6KlSFajXQPcM4ogNE1q2s3Lbz9GCGqeIHC+C6OZnKg== + version "0.2.1" + resolved "https://registry.yarnpkg.com/node-api-version/-/node-api-version-0.2.1.tgz#19bad54f6d65628cbee4e607a325e4488ace2de9" + integrity sha512-2xP/IGGMmmSQpI1+O/k72jF/ykvZ89JeuKX3TLJAYPDVLUalrshrLHkeVcCCZqG/eEa635cr8IBYzgnDvM2O8Q== dependencies: semver "^7.3.5" @@ -2081,6 +2131,11 @@ pend@~1.2.0: resolved "https://registry.yarnpkg.com/pend/-/pend-1.2.0.tgz#7a57eb550a6783f9115331fcf4663d5c8e007a50" integrity sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg== +picocolors@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.1.1.tgz#3d321af3eab939b083c8f929a1d12cda81c26b6b" + integrity sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA== + picomatch@^2.3.1: version "2.3.1" resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.1.tgz#3ba3833733646d9d3e4995946c1365a67fb07a42" @@ -2119,9 +2174,9 @@ promise-retry@^2.0.1: retry "^0.12.0" pump@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/pump/-/pump-3.0.0.tgz#b4a2116815bde2f4e1ea602354e8c75565107a64" - integrity sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww== + version "3.0.3" + resolved "https://registry.yarnpkg.com/pump/-/pump-3.0.3.tgz#151d979f1a29668dc0025ec589a455b53282268d" + integrity sha512-todwxLMY7/heScKmntwQG8CXVkWUOdYxIvY2s0VWAAMh/nd8SoYiRaKjlr7+iCs984f2P8zvrfWcDDYVb73NfA== dependencies: end-of-stream "^1.1.0" once "^1.3.1" @@ -2261,9 +2316,9 @@ sanitize-filename@^1.6.3: truncate-utf8-bytes "^1.0.0" sax@^1.2.4: - version "1.3.0" - resolved "https://registry.yarnpkg.com/sax/-/sax-1.3.0.tgz#a5dbe77db3be05c9d1ee7785dbd3ea9de51593d0" - integrity sha512-0s+oAmw9zLl1V1cS9BtZN7JAd0cW5e0QH4W3LWEK6a4LaLEA2OTpGYWDY+6XasBLtz6wkm3u1xRw95mRuJ59WA== + version "1.4.3" + resolved "https://registry.yarnpkg.com/sax/-/sax-1.4.3.tgz#fcebae3b756cdc8428321805f4b70f16ec0ab5db" + integrity sha512-yqYn1JhPczigF94DMS+shiDMjDowYO6y9+wB/4WgO0Y19jWYk0lQ4tuG5KI7kj4FTp1wxPj5IFfcrz/s1c3jjQ== semver-compare@^1.0.0: version "1.0.0" @@ -2280,15 +2335,10 @@ semver@^6.2.0: resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4" integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA== -semver@^7.3.2: - version "7.6.1" - resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.1.tgz#60bfe090bf907a25aa8119a72b9f90ef7ca281b2" - integrity sha512-f/vbBsu+fOiYt+lmwZV0rVwJScl46HppnOA1ZvIuBWKOTlllpyJ3bfVax76/OrhCH38dyxoDIA8K7uB963IYgA== - -semver@^7.3.5, semver@^7.3.8, semver@^7.5.3, semver@^7.5.4, semver@^7.6.3: - version "7.6.3" - resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.3.tgz#980f7b5550bc175fb4dc09403085627f9eb33143" - integrity sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A== +semver@^7.3.2, semver@^7.3.5, semver@^7.3.8, semver@^7.5.3, semver@^7.5.4, semver@^7.6.3: + version "7.7.3" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.7.3.tgz#4b5f4143d007633a8dc671cd0a6ef9147b8bb946" + integrity sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q== serialize-error@^7.0.1: version "7.0.1" @@ -2372,11 +2422,11 @@ socks-proxy-agent@^7.0.0: socks "^2.6.2" socks@^2.6.2: - version "2.8.3" - resolved "https://registry.yarnpkg.com/socks/-/socks-2.8.3.tgz#1ebd0f09c52ba95a09750afe3f3f9f724a800cb5" - integrity sha512-l5x7VUUWbjVFbafGLxPWkYsHIhEvmF85tbIeFZWc8ZPtoMyybuEhL7Jye/ooC4/d48FgOjSJXgsF/AJPYCW8Zw== + version "2.8.7" + resolved "https://registry.yarnpkg.com/socks/-/socks-2.8.7.tgz#e2fb1d9a603add75050a2067db8c381a0b5669ea" + integrity sha512-HLpt+uLy/pxB+bum/9DzAgiKS8CX1EvbWxI4zlmgGCExImLdiad2iCwXT5Z4c9c3Eq8rP2318mPW2c+QbtjK8A== dependencies: - ip-address "^9.0.5" + ip-address "^10.0.1" smart-buffer "^4.2.0" source-map-support@^0.5.19: @@ -2392,7 +2442,7 @@ source-map@^0.6.0: resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263" integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g== -sprintf-js@^1.1.2, sprintf-js@^1.1.3: +sprintf-js@^1.1.2: version "1.1.3" resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.1.3.tgz#4914b903a2f8b685d17fdf78a70e917e872e444a" integrity sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA== @@ -2454,9 +2504,9 @@ string_decoder@~1.1.1: ansi-regex "^5.0.1" strip-ansi@^7.0.1: - version "7.1.0" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.1.0.tgz#d5b6568ca689d8561370b0707685d22434faff45" - integrity sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ== + version "7.1.2" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.1.2.tgz#132875abde678c7ea8d691533f2e7e22bb744dba" + integrity sha512-gmBGslpoQJtgnMAvOVqGZpEz9dyoKTCzy2nfz/n8aIFhN/jCE/rCmcxabB6jOOHV+0WNnylOxaxBQPSvcWklhA== dependencies: ansi-regex "^6.0.1" @@ -2522,9 +2572,9 @@ tmp@^0.0.33: os-tmpdir "~1.0.2" tmp@^0.2.0: - version "0.2.3" - resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.2.3.tgz#eb783cc22bc1e8bebd0671476d46ea4eb32a79ae" - integrity sha512-nZD7m9iCPC5g0pYmcaxogYKggSfLsdxl8of3Q/oIbqCqLLIO9IAF0GWjX1z9NZRHPiXv8Wex4yDCaZsgEw0Y8w== + version "0.2.5" + resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.2.5.tgz#b06bcd23f0f3c8357b426891726d16015abfd8f8" + integrity sha512-voyz6MApa1rQGUxT3E+BK7/ROe8itEx7vD8/HEvt4xwXucvQ5G5oeEiHkmHZJuBO21RpOf+YYm9MOivj709jow== to-regex-range@^5.0.1: version "5.0.1" @@ -2546,14 +2596,19 @@ type-fest@^0.13.1: integrity sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg== typescript@^5.4.3: - version "5.7.2" - resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.7.2.tgz#3169cf8c4c8a828cde53ba9ecb3d2b1d5dd67be6" - integrity sha512-i5t66RHxDvVN40HfDd1PsEThGNnlMCMT3jMUuoh9/0TaqWevNontacunWyN02LA9/fIbEWlcHZcgTKb9QoaLfg== + version "5.9.3" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.9.3.tgz#5b4f59e15310ab17a216f5d6cf53ee476ede670f" + integrity sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw== -undici-types@~5.26.4: - version "5.26.5" - resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617" - integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA== +undici-types@~6.21.0: + version "6.21.0" + resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-6.21.0.tgz#691d00af3909be93a7faa13be61b3a5b50ef12cb" + integrity sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ== + +undici-types@~7.16.0: + version "7.16.0" + resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-7.16.0.tgz#ffccdff36aea4884cbfce9a750a0580224f58a46" + integrity sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw== unique-filename@^2.0.0: version "2.0.1" @@ -2592,9 +2647,9 @@ uri-js@^4.2.2: punycode "^2.1.0" utf8-byte-length@^1.0.1: - version "1.0.4" - resolved "https://registry.yarnpkg.com/utf8-byte-length/-/utf8-byte-length-1.0.4.tgz#f45f150c4c66eee968186505ab93fcbb8ad6bf61" - integrity sha512-4+wkEYLBbWxqTahEsWrhxepcoVOJ+1z5PGIjPZxRkytcdSUaNjIjBM7Xn8E+pdSuV7SzvWovBFA54FO0JSoqhA== + version "1.0.5" + resolved "https://registry.yarnpkg.com/utf8-byte-length/-/utf8-byte-length-1.0.5.tgz#f9f63910d15536ee2b2d5dd4665389715eac5c1e" + integrity sha512-Xn0w3MtiQ6zoz2vFyUVruaCL53O/DwUvkEeOvj+uulMm0BkUGYWmBYVyElqZaSLhY6ZD0ulfU3aBra2aVT4xfA== util-deprecate@^1.0.1, util-deprecate@~1.0.1: version "1.0.2" diff --git a/common/fixYmlHashes.js b/common/fixYmlHashes.js new file mode 100644 index 000000000..bf317ef23 --- /dev/null +++ b/common/fixYmlHashes.js @@ -0,0 +1,110 @@ +import fs from 'node:fs/promises'; +import { createHash } from 'node:crypto'; +import path from 'node:path'; +import process from 'node:process'; +import { fileURLToPath } from 'node:url'; +import YAML from 'yaml'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +async function sha512Base64(filePath) { + const buf = await fs.readFile(filePath); + const h = createHash('sha512'); + h.update(buf); + return h.digest('base64'); +} + +async function fileSize(filePath) { + const st = await fs.stat(filePath); + return st.size; +} + +async function fixOneYaml(ymlPath, distDir) { + let raw; + try { + raw = await fs.readFile(ymlPath, 'utf8'); + } catch (e) { + console.error(`✗ Cannot read ${ymlPath}:`, e.message); + return; + } + + let doc; + try { + doc = YAML.parse(raw); + } catch (e) { + console.error(`✗ Cannot parse YAML ${ymlPath}:`, e.message); + return; + } + + if (!doc || !Array.isArray(doc.files)) { + console.warn(`! ${path.basename(ymlPath)} has no 'files' array — skipped.`); + return; + } + + let changed = false; + + // Update each files[i].sha512 and files[i].size based on files[i].url + for (const entry of doc.files) { + if (!entry?.url) continue; + + const target = path.resolve(distDir, entry.url); + try { + const [hash, size] = await Promise.all([sha512Base64(target), fileSize(target)]); + if (entry.sha512 !== hash || entry.size !== size) { + console.log(`• ${path.basename(ymlPath)}: refresh ${entry.url}`); + entry.sha512 = hash; + entry.size = size; + changed = true; + } + } catch (e) { + console.warn( + `! Missing or unreadable file for ${entry.url} (referenced by ${path.basename(ymlPath)}): ${e.message}` + ); + } + } + + // Update top-level sha512 for the primary "path" file if present + if (doc.path) { + const primary = path.resolve(distDir, doc.path); + try { + const hash = await sha512Base64(primary); + if (doc.sha512 !== hash) { + console.log(`• ${path.basename(ymlPath)}: refresh top-level sha512 for path=${doc.path}`); + doc.sha512 = hash; + changed = true; + } + } catch (e) { + console.warn(`! Primary 'path' file not found for ${path.basename(ymlPath)}: ${doc.path} (${e.message})`); + } + } + + if (changed) { + const out = YAML.stringify(doc); + await fs.writeFile(ymlPath, out, 'utf8'); + console.log(`✓ Updated ${path.basename(ymlPath)}`); + } else { + console.log(`= No changes for ${path.basename(ymlPath)}`); + } +} + +async function main() { + const distDir = path.resolve(process.argv[2] ?? path.join(__dirname, '..', 'app', 'dist')); + const entries = await fs.readdir(distDir); + const ymls = entries.filter(f => f.toLowerCase().endsWith('.yml')); + + if (ymls.length === 0) { + console.warn(`No .yml files found in ${distDir}`); + return; + } + + console.log(`Scanning ${distDir}`); + for (const y of ymls) { + await fixOneYaml(path.join(distDir, y), distDir); + } +} + +main().catch(err => { + console.error(err); + process.exit(1); +}); diff --git a/common/translations-cli/extract.js b/common/translations-cli/extract.js index 9c6374594..618e1dddb 100644 --- a/common/translations-cli/extract.js +++ b/common/translations-cli/extract.js @@ -6,7 +6,7 @@ const { getFiles } = require('./helpers'); const readFilePromise = promisify(fs.readFile); -const translationRegex = /_t\(\s*['"]([^'"]+)['"]\s*,\s*\{\s*defaultMessage\s*:\s*['"]([^'"]+)['"]\s*\}/g; +const translationRegex = /_t\(\s*['"]([^'"]+)['"]\s*,\s*\{\s*defaultMessage\s*:\s*(?:'([^'\\]*(?:\\.[^'\\]*)*)'|"([^"\\]*(?:\\.[^"\\]*)*)"|\`([^`\\]*(?:\\.[^`\\]*)*(?:\{[^}]*\}[^`\\]*(?:\\.[^`\\]*)*)*)\`)(?:\s*,\s*[^}]*)*\s*\}/g; /** * @param {string} file @@ -20,7 +20,8 @@ async function extractTranslationsFromFile(file) { let match; while ((match = translationRegex.exec(content)) !== null) { - const [_, key, defaultText] = match; + const [_, key, singleQuotedText, doubleQuotedText, templateLiteral] = match; + const defaultText = singleQuotedText || doubleQuotedText || templateLiteral; translations[key] = defaultText; } diff --git a/common/translations-cli/program.js b/common/translations-cli/program.js index 7233641b7..24e4b287a 100644 --- a/common/translations-cli/program.js +++ b/common/translations-cli/program.js @@ -160,4 +160,31 @@ program } }); +program + .command('sort') + .description('Sort translation files by keys') + .action(() => { + try { + const languages = getAllNonDefaultLanguages(); + for (const language of languages) { + const filePath = `./translations/${language}.json`; + const content = fs.readFileSync(filePath, 'utf-8'); + const translations = JSON.parse(content); + const sortedTranslations = {}; + Object.keys(translations) + .sort() + .forEach(key => { + // @ts-ignore + sortedTranslations[key] = translations[key]; + }); + fs.writeFileSync(filePath, JSON.stringify(sortedTranslations, null, 2), 'utf-8'); + console.log(`Sorted translations for language: ${language}`); + } + } catch (error) { + console.error(error); + console.error('Error during sort:', error.message); + process.exit(1); + } + }); + module.exports = { program }; diff --git a/common/translations-cli/translate.js b/common/translations-cli/translate.js new file mode 100644 index 000000000..fd6613102 --- /dev/null +++ b/common/translations-cli/translate.js @@ -0,0 +1,132 @@ +require('dotenv').config({ path: '.env.translation' }); +const fs = require('fs'); +const path = require('path'); +const OpenAI = require('openai'); + +const client = new OpenAI({ apiKey: process.env.OPENAI_API_KEY }); + +const translationsDir = path.join(__dirname, '../../translations'); +const enFilePath = path.join(translationsDir, 'en.json'); + +const languageNames = { + 'cs.json': 'Czech', + 'de.json': 'German', + 'es.json': 'Spanish', + 'fr.json': 'French', + 'it.json': 'Italian', + 'ja.json': 'Japanese', + 'pt.json': 'Portuguese', + 'sk.json': 'Slovak', + 'zh.json': 'Chinese' +}; + +// Read source (english) +const enTranslations = JSON.parse(fs.readFileSync(enFilePath, 'utf8')); +const enKeys = Object.keys(enTranslations); + +// Get all translation files +const translationFiles = fs.readdirSync(translationsDir) + .filter(file => file.endsWith('.json') && file !== 'en.json') + .sort(); + +console.log(`Found ${enKeys.length} keys in en.json\n`); +console.log('='.repeat(80)); + +async function translateMissingIds({file, translations, missingIds}){ + const languageName = languageNames[file]; + if (!languageName) { + console.log(`No language name mapping for file: ${file}`); + return; + } + + // Build object with only missing translations + const needed = {}; + missingIds.forEach(key => { + needed[key] = enTranslations[key]; + }); + + // Get all existing translations as style examples + const existingTranslations = {}; + Object.keys(translations).forEach(key => { + if (translations[key] && !translations[key].startsWith('***')) { + existingTranslations[key] = { + en: enTranslations[key], + translated: translations[key] + }; + } + }); + + const prompt = `You are a professional translator for DbGate, a database management application. + +Translate the following English UI strings to ${languageName}. + +IMPORTANT RULES: +1. Preserve ALL placeholders exactly as they appear: {plugin}, {columnNumber}, {0}, {1}, etc. +2. Maintain technical terminology appropriately for database software +3. Match the translation style, tone, and formality of the existing translations shown below +4. Keep the same level of brevity or verbosity as the existing translations +5. Return ONLY valid JSON - no markdown, no explanations, no code blocks +6. Use the same keys as provided + +EXISTING TRANSLATIONS (for style reference): +${JSON.stringify(existingTranslations, null, 2)} + +STRINGS TO TRANSLATE: +${JSON.stringify(needed, null, 2)} + +Return format: {"key": "translated value", ...}`; + + const response = await client.chat.completions.create({ + model: 'gpt-5.1', + messages: [ + { role: 'system', content: 'You are a professional translator specializing in software localization. Match the style and tone of existing translations. Return only valid JSON.' }, + { role: 'user', content: prompt } + ], + temperature: 0.2 + }); + + let translatedJson = response.choices[0].message.content.trim(); + + // Remove markdown code blocks if present + translatedJson = translatedJson.replace(/^```json\n?/, '').replace(/\n?```$/, ''); + + return JSON.parse(translatedJson); +} + +(async () => { + for (const file of translationFiles) { + const filePath = path.join(translationsDir, file); + const translations = JSON.parse(fs.readFileSync(filePath, 'utf8')); + + const missingIds = enKeys.filter(key => !translations.hasOwnProperty(key) || (typeof translations[key] === 'string' && translations[key].startsWith('***'))); + + + console.log(`\n${file.toUpperCase()}`); + console.log('-'.repeat(80)); + + if (missingIds.length === 0) { + console.log('✓ All translations complete!'); + continue; + } else { + console.log(`Found ${missingIds.length} untranslated IDs\n`); + } + + const newTranslations = await translateMissingIds({file, translations, missingIds}); + + if (!newTranslations) { + console.log(`Skipping file due to translation error: ${file}`); + continue; + } + + for (const [key, value] of Object.entries(newTranslations)) { + translations[key] = value; + console.log(`Translated: ${key} => ${value}`); + } + + fs.writeFileSync(filePath, JSON.stringify(translations, null, 2) + '\n', 'utf8'); + console.log(`\n✓ Updated translations written to ${file}`); + } + + console.log('\n' + '='.repeat(80)); + console.log('Translation complete!\n'); +})(); diff --git a/common/volatilePackages.js b/common/volatilePackages.js index 78c375f18..40d4a9207 100644 --- a/common/volatilePackages.js +++ b/common/volatilePackages.js @@ -4,6 +4,7 @@ const volatilePackages = [ '@clickhouse/client', 'bson', // this package is already bundled and is used in mongodb 'mongodb', + 'mongodb-old', 'mongodb-client-encryption', 'tedious', 'msnodesqlv8', diff --git a/e2e-tests/cypress.config.js b/e2e-tests/cypress.config.js index ef2d17717..28fa48e60 100644 --- a/e2e-tests/cypress.config.js +++ b/e2e-tests/cypress.config.js @@ -10,6 +10,7 @@ module.exports = defineConfig({ // baseUrl: 'http://localhost:3000', // trashAssetsBeforeRuns: false, chromeWebSecurity: false, + reporter: process.env.CI ? 'mocha-reporter-gha' : 'spec', setupNodeEvents(on, config) { // implement node event listeners here diff --git a/e2e-tests/cypress/e2e/add-connection.cy.js b/e2e-tests/cypress/e2e/add-connection.cy.js index d8dad00f8..cc76c2797 100644 --- a/e2e-tests/cypress/e2e/add-connection.cy.js +++ b/e2e-tests/cypress/e2e/add-connection.cy.js @@ -113,10 +113,35 @@ describe('Add connection', () => { cy.contains('performance_schema'); }); + it('Plugin tab', () => { + cy.testid('WidgetIconPanel_menu').click(); + cy.contains('Tools').click(); + cy.contains('Manage plugins').click(); + cy.contains('dbgate-plugin-theme-total-white').click(); + // text from plugin markdown + cy.contains('Total white theme'); + // wait for load logos + cy.wait(2000); + cy.themeshot('view-plugin-tab'); + }); + it('export connections', () => { cy.testid('WidgetIconPanel_menu').click(); cy.contains('Tools').click(); cy.contains('Export connections').click(); cy.themeshot('export-connections'); }); + + it('configure LLM provider', () => { + cy.testid('WidgetIconPanel_settings').click(); + cy.contains('Settings').click(); + cy.contains('AI').click(); + cy.testid('AiSupportedProvidersInfo_add_OpenRouter').click(); + cy.testid('AiProviderCard_apikey_OpenRouter').clear().type('xxx'); + cy.testid('AiProviderCard_testButton_OpenRouter').click(); + cy.testid('AiProviderCard_statusValid_OpenRouter').should('exist'); + cy.testid('AiProviderCard_editButton_OpenRouter').click(); + cy.wait(1000); + cy.themeshot('llm-providers-settings'); + }); }); diff --git a/e2e-tests/cypress/e2e/browse-data.cy.js b/e2e-tests/cypress/e2e/browse-data.cy.js index 409ff66d2..495e768aa 100644 --- a/e2e-tests/cypress/e2e/browse-data.cy.js +++ b/e2e-tests/cypress/e2e/browse-data.cy.js @@ -60,7 +60,7 @@ describe('Data browser data', () => { cy.contains('MyChinook').click(); cy.testid('SqlObjectList_search').clear().type('album'); cy.contains('Tables (1/11)'); - cy.contains('347 rows, InnoDB'); + cy.contains('347 rows, 65.5 KB, InnoDB'); cy.testid('SqlObjectList_searchMenuDropDown').click(); cy.contains('Column name').click(); cy.contains('Tables (2/11)'); @@ -202,7 +202,7 @@ describe('Data browser data', () => { cy.themeshot('query-editor-join-wizard'); }); - it('Mongo JSON data view', () => { + it('Mongo query JSON data view', () => { cy.contains('Mongo-connection').click(); cy.contains('MgChinook').click(); cy.contains('Customer').click(); @@ -213,9 +213,10 @@ describe('Data browser data', () => { cy.contains('Open query').click(); cy.wait(1000); cy.contains('Execute').click(); - cy.testid('WidgetIconPanel_cell-data').click(); + cy.testid('TabContent_1').contains('Leonie').rightclick(); + cy.contains('Show cell data').click(); // test JSON view - cy.contains('Country: "Brazil"'); + cy.contains('Country: "Germany"'); cy.themeshot('mongo-query-json-view'); }); @@ -277,6 +278,14 @@ describe('Data browser data', () => { cy.testid('CommandPalette_main').themeshot('command-palette', { padding: 50 }); }); + it('About window', () => { + cy.contains('Connections'); + cy.testid('WidgetIconPanel_menu').click(); + cy.contains('Help').click(); + cy.contains('About').click(); + cy.testid('ModalBase_window').themeshot('about-window', { padding: 50 }); + }); + it('Show map', () => { cy.contains('Postgres-connection').click(); cy.contains('PgGeoData').click(); @@ -285,7 +294,8 @@ describe('Data browser data', () => { // cy.contains('location').click(); cy.contains('14.2').click(); cy.contains('13.9').click({ shiftKey: true }); - cy.testid('WidgetIconPanel_cell-data').click(); + cy.testid('WidgetIconPanel_database').click(); + cy.testid('TableDataTab_toggleCellDataView').click(); cy.wait(2000); cy.themeshot('cell-map-view'); }); @@ -302,17 +312,6 @@ describe('Data browser data', () => { cy.themeshot('search-in-connections'); }); - it('Plugin tab', () => { - cy.testid('WidgetIconPanel_settings').click(); - cy.contains('Manage plugins').click(); - cy.contains('dbgate-plugin-theme-total-white').click(); - // text from plugin markdown - cy.contains('Total white theme'); - // wait for load logos - cy.wait(2000); - cy.themeshot('view-plugin-tab'); - }); - it('Edit mongo data JSON', () => { // TODO FIX: Missing button+ctx menu Revert all changes, missing button+ctx menu add document // TODO: Dark theme - not visible changed and deleted document @@ -340,7 +339,7 @@ describe('Data browser data', () => { cy.themeshot('save-changes-mongodb'); }); - it('Edit mongo data JSON', () => { + it('Mongo JSON cell view', () => { // TODO FIX: Auto expand cell view cy.contains('Mongo-connection').click(); cy.contains('MgRivers').click(); @@ -350,7 +349,8 @@ describe('Data browser data', () => { cy.testid('ColumnManagerRow_checkbox_countries.1').click(); cy.testid('ColumnManagerRow_checkbox__id').click(); cy.testid('DataFilterControl_input_countries.1').type('EXISTS{enter}'); - cy.testid('WidgetIconPanel_cell-data').click(); + cy.contains('Austria').click(); + cy.testid('CollectionDataTab_toggleCellDataView').click(); cy.themeshot('mongodb-json-cell-view'); }); @@ -381,27 +381,6 @@ describe('Data browser data', () => { cy.themeshot('compare-database-settings'); }); - it('Database chat', () => { - cy.contains('MySql-connection').click(); - cy.contains('MyChinook').click(); - cy.testid('TabsPanel_buttonNewObject').click(); - cy.testid('NewObjectModal_databaseChat').click(); - cy.wait(1000); - cy.get('body').realType('find most popular artist'); - cy.get('body').realPress('{enter}'); - cy.testid('DatabaseChatTab_executeAllQueries', { timeout: 20000 }).click(); - cy.wait(20000); - // cy.contains('Iron Maiden'); - cy.themeshot('database-chat'); - - // cy.testid('DatabaseChatTab_promptInput').click(); - // cy.get('body').realType('I need top 10 songs with the biggest income'); - // cy.get('body').realPress('{enter}'); - // cy.contains('Hot Girl', { timeout: 20000 }); - // cy.wait(1000); - // cy.themeshot('database-chat'); - }); - it('Modify data', () => { // TODO FIX: delete references cascade not working cy.contains('MySql-connection').click(); @@ -496,4 +475,36 @@ describe('Data browser data', () => { cy.testid('DataDeployTab_importIntoDb').click(); cy.themeshot('data-replicator'); }); + + it('Form cell view', () => { + cy.contains('MySql-connection').click(); + cy.contains('MyChinook').click(); + cy.contains('Invoice').click(); + cy.get('[data-row="0"][data-col="header"]').click(); + cy.contains('Autodetect - Form'); + cy.themeshot('form-cell-view'); + }); + + it('Group by', () => { + cy.contains('MySql-connection').click(); + cy.contains('MyChinook').click(); + cy.contains('Album').click(); + cy.testid('WidgetIconPanel_database').click(); + cy.testid('ColumnHeaderControl_dropdown_ArtistId').click(); + cy.contains('Group by').click(); + cy.testid('ColumnHeaderControl_dropdown_Title').first().click(); + cy.themeshot('data-browser-group-by'); + }); + + it('Filter by expanded column', () => { + cy.contains('MySql-connection').click(); + cy.contains('MyChinook').click(); + cy.contains('Album').click(); + cy.testid('WidgetIconPanel_database').click(); + cy.testid('ColumnManagerRow_expand_ArtistId').click(); + cy.testid('ColumnManagerRow_checkbox_ArtistId.Name').click(); + cy.testid('ColumnManagerRow_checkbox_ArtistId').click(); + cy.testid('DataFilterControl_input_ArtistId.Name').type('mich{enter}'); + cy.themeshot('data-browser-filter-by-expanded'); + }); }); diff --git a/e2e-tests/cypress/e2e/charts.cy.js b/e2e-tests/cypress/e2e/charts.cy.js index ba7316640..f679d9b31 100644 --- a/e2e-tests/cypress/e2e/charts.cy.js +++ b/e2e-tests/cypress/e2e/charts.cy.js @@ -109,4 +109,163 @@ describe('Charts', () => { cy.contains('Compare database'); cy.themeshot('new-object-window'); }); + + it.skip('Database chat - charts', () => { + cy.contains('MySql-connection').click(); + cy.contains('MyChinook').click(); + cy.testid('TabsPanel_buttonNewObject').click(); + cy.testid('NewObjectModal_databaseChat').click(); + cy.wait(1000); + cy.get('body').realType('show me chart of most popular genres'); + cy.get('body').realPress('{enter}'); + cy.testid('DatabaseChatTab_executeAllQueries', { timeout: 30000 }).click(); + cy.testid('chart-canvas', { timeout: 30000 }).should($c => + expect($c[0].toDataURL()).to.match(/^data:image\/png;base64/) + ); + cy.themeshot('database-chat-chart'); + }); + + it.skip('Database chat', () => { + cy.contains('MySql-connection').click(); + cy.contains('MyChinook').click(); + cy.testid('TabsPanel_buttonNewObject').click(); + cy.testid('NewObjectModal_databaseChat').click(); + cy.wait(1000); + cy.get('body').realType('find most popular artist'); + cy.get('body').realPress('{enter}'); + cy.testid('DatabaseChatTab_executeAllQueries', { timeout: 30000 }).click(); + cy.wait(30000); + // cy.contains('Iron Maiden'); + cy.themeshot('database-chat'); + + // cy.testid('DatabaseChatTab_promptInput').click(); + // cy.get('body').realType('I need top 10 songs with the biggest income'); + // cy.get('body').realPress('{enter}'); + // cy.contains('Hot Girl', { timeout: 20000 }); + // cy.wait(1000); + // cy.themeshot('database-chat'); + }); + + it.skip('Explain query error', () => { + cy.contains('MySql-connection').click(); + cy.contains('MyChinook').click(); + cy.testid('TabsPanel_buttonNewObject').click(); + cy.testid('NewObjectModal_query').click(); + cy.wait(1000); + cy.get('body').realType('select * from Invoice2'); + cy.contains('Execute').click(); + cy.testid('MessageViewRow-explainErrorButton-1').click(); + cy.testid('ChatCodeRenderer_useSqlButton', { timeout: 30000 }); + cy.themeshot('explain-query-error'); + }); + + it('Switch language', () => { + cy.contains('MySql-connection').click(); + cy.contains('MyChinook').click(); + cy.testid('WidgetIconPanel_settings').click(); + + cy.testid('SettingsModal_languageSelect').select('Deutsch'); + cy.testid('ConfirmModal_okButton').click(); + cy.testid('WidgetIconPanel_settings').click(); + cy.contains('Sprache'); + cy.themeshot('switch-language-de'); + + cy.testid('SettingsModal_languageSelect').select('Français'); + cy.testid('ConfirmModal_okButton').click(); + cy.testid('WidgetIconPanel_settings').click(); + cy.contains('Langue'); + cy.themeshot('switch-language-fr'); + + cy.testid('SettingsModal_languageSelect').select('Español'); + cy.testid('ConfirmModal_okButton').click(); + cy.testid('WidgetIconPanel_settings').click(); + cy.contains('Idioma'); + cy.themeshot('switch-language-es'); + + cy.testid('SettingsModal_languageSelect').select('Čeština'); + cy.testid('ConfirmModal_okButton').click(); + cy.testid('WidgetIconPanel_settings').click(); + cy.contains('Jazyk'); + cy.themeshot('switch-language-cs'); + + cy.testid('SettingsModal_languageSelect').select('中文'); + cy.testid('ConfirmModal_okButton').click(); + cy.testid('WidgetIconPanel_settings').click(); + cy.contains('语言'); + cy.themeshot('switch-language-zh'); + + cy.testid('SettingsModal_languageSelect').select('English'); + cy.testid('ConfirmModal_okButton').click(); + cy.testid('WidgetIconPanel_settings'); + }); + + it('Settings', () => { + cy.testid('WidgetIconPanel_settings').click(); + cy.themeshot('app-settings-general'); + + cy.contains('Behaviour').click(); + cy.themeshot('app-settings-behaviour'); + cy.get('[data-testid=BehaviourSettings_useTabPreviewMode]').uncheck(); + + // SQL Editor + cy.contains('SQL Editor').click(); + cy.get('[data-testid=SQLEditorSettings_sqlCommandsCase]').select('lowerCase'); + + cy.contains('MySql-connection').click(); + cy.contains('charts_sample').click(); + cy.contains('employees').click(); + cy.contains('MyChinook').click(); + cy.contains('Customer').rightclick(); + cy.contains('SQL template').click(); + cy.contains('CREATE TABLE').click(); + cy.contains('create table'); + + // Default Actions + cy.testid('WidgetIconPanel_settings').click(); + cy.contains('Default Actions').click(); + cy.get('[data-testid=DefaultActionsSettings_useLastUsedAction]').uncheck(); + + + // Themes + cy.contains('Themes').click(); + cy.themeshot('app-settings-themes'); + cy.contains('Dark').click(); + cy.get('body').find('.theme-dark').should('exist'); + cy.contains('Light').click(); + cy.get('body').find('.theme-light').should('exist'); + + // General + cy.contains(/^General$/).click(); + cy.contains('charts_sample'); + cy.get('[data-testid=GeneralSettings_lockedDatabaseMode]').check(); + cy.contains('Connections').click(); + cy.contains('charts_sample').should('not.exist'); + + // Datagrid + cy.contains('Data grid').click(); + cy.get('[data-testid=DataGridSettings_showHintColumns]').uncheck(); + cy.wait(500); + cy.contains('Album').click(); + cy.contains('AC/DC').should('not.exist'); + + cy.testid('WidgetIconPanel_settings').click(); + cy.contains('Keyboard shortcuts').click(); + cy.themeshot('app-settings-keyboard-shortcuts'); + cy.contains('Chart').click(); + cy.testid('CommandModal_keyboardButton').click(); + cy.realPress(['Control', 'g']); + cy.realPress('Enter'); + cy.contains('OK').click(); + cy.contains('Ctrl+G'); + + + cy.contains('AI').click(); + cy.themeshot('app-settings-ai'); + cy.get('[data-testid=AISettings_addProviderButton]').click(); + cy.contains('Provider 1'); + cy.get('[data-testid=AiProviderCard_removeButton]').click(); + cy.contains('Are you sure you want to remove Provider 1 provider?'); + cy.contains('OK').click(); + cy.contains('Provider 1').should('not.exist'); + }); }); diff --git a/e2e-tests/cypress/e2e/multi-sql.cy.js b/e2e-tests/cypress/e2e/multi-sql.cy.js index 9b4596570..27239dce1 100644 --- a/e2e-tests/cypress/e2e/multi-sql.cy.js +++ b/e2e-tests/cypress/e2e/multi-sql.cy.js @@ -103,13 +103,70 @@ describe('Transactions', () => { describe('Backup table', () => { multiTest({ skipMongo: true }, (connectionName, databaseName, engine, options = {}) => { + const implicitTransactions = options.implicitTransactions ?? false; + cy.contains(connectionName).click(); if (databaseName) cy.contains(databaseName).click(); - cy.contains('customers').rightclick(); + cy.contains('addresses').rightclick(); cy.contains('Create table backup').click(); cy.testid('ConfirmSqlModal_okButton').click(); - cy.contains('_customers').click(); - cy.contains('Rows: 8').should('be.visible'); + cy.testid('app-object-group-items-table-backups').contains('addresses').click(); + cy.contains('Rows: 12').should('be.visible'); + cy.testid('app-object-group-items-tables').contains('addresses').click(); + + cy.contains('Ridgewood').click(); + cy.testid('TableDataTab_deleteSelectedRows').click(); + cy.contains('Rosewood').click(); + cy.testid('TableDataTab_deleteSelectedRows').click(); + + cy.contains('Vermont').click(); + cy.get('body').realType('Wermont{enter}'); + + cy.testid('TableDataTab_insertNewRow').click(); + cy.get('body').realType('Modranska{enter}'); + cy.realPress(['ArrowLeft']); + cy.realPress(['ArrowLeft']); + cy.get('body').realType('13{enter}'); + cy.realPress(['ArrowRight']); + cy.get('body').realType('1{enter}'); + cy.realPress(['ArrowRight']); + cy.realPress(['ArrowRight']); + cy.realPress(['ArrowRight']); + cy.get('body').realType('Prague{enter}'); + cy.realPress(['ArrowRight']); + cy.get('body').realType('CZ{enter}'); + cy.realPress(['ArrowRight']); + cy.get('body').realType('10000{enter}'); + cy.realPress(['ArrowRight']); + cy.get('body').realType('111222333{enter}'); + + cy.testid('TableDataTab_save').click(); + cy.testid('ConfirmSqlModal_okButton', { timeout: 10000 }).click(); + cy.contains('Rows: 11').should('be.visible'); // wait for save + + cy.testid('app-object-group-items-table-backups').contains('addresses').rightclick(); + cy.contains('restore script').click(); + cy.contains('UPDATE'); // wait for query + cy.testid('QueryTab_executeButton').click(); + cy.contains('Query execution finished'); + + if (implicitTransactions) { + cy.testid('QueryTab_commitTransactionButton').click(); + cy.contains('Commit Transaction finished'); + } + + cy.realPress('F1'); + cy.realType('Close all'); + cy.realPress('Enter'); + // cy.testid('CloseTabModal_buttonConfirm').click(); + cy.wait(1000); + + cy.testid('app-object-group-items-tables').contains('addresses', { timeout: 10000 }).click(); + + // check whether data was successfully restored + cy.contains('Rows: 12').should('be.visible'); + cy.contains('Ridgewood'); + cy.contains('Vermont'); }); }); @@ -146,13 +203,15 @@ describe('Import CSV', () => { cy.contains('Import').click(); cy.get('input[type=file]').selectFile('cypress/fixtures/customers-20.csv', { force: true }); - cy.contains('customers-20'); + cy.testid('ImportExportConfigurator_tableMappingSection').contains('customers-20'); cy.testid('ImportExportTab_preview_content').contains('50ddd99fAdF48B3').should('be.visible'); cy.testid('ImportExportTab_executeButton').click(); - cy.contains('20 rows written').should('be.visible'); + cy.testid('ImportExportConfigurator_tableMappingSection').contains('20 rows written').should('be.visible'); cy.testid('SqlObjectList_refreshButton').click(); + cy.testid('DatabasStatusMenu_refreshFull').click(); + // cy.contains('Refresh DB structure (incremental)').click(); cy.testid('SqlObjectList_container').contains('customers-20').click(); cy.contains('Rows: 20').should('be.visible'); @@ -178,7 +237,7 @@ describe('Import CSV - source error', () => { cy.testid('ImportExportTab_preview_content').contains('Invalid Closing Quote').should('be.visible'); cy.testid('ImportExportTab_executeButton').click(); - cy.testid('ImportExportConfigurator_errorInfoIcon_customers-20-err').click(); + cy.testid('ImportExportConfigurator_errorInfoIcon_customers-20-err', { timeout: 10000 }).click(); cy.testid('ErrorMessageModal_message').contains('Invalid Closing Quote').should('be.visible'); }); @@ -197,7 +256,7 @@ describe('Import CSV - target error', () => { cy.contains('customers-20'); cy.testid('ImportExportConfigurator_targetName_customers-20').clear().type('system."]`'); cy.testid('ImportExportTab_executeButton').click(); - cy.testid('ImportExportConfigurator_errorInfoIcon_customers-20').click(); + cy.testid('ImportExportConfigurator_errorInfoIcon_customers-20', { timeout: 10000 }).click(); cy.testid('ErrorMessageModal_message').should('be.visible'); }); }); diff --git a/e2e-tests/cypress/e2e/team.cy.js b/e2e-tests/cypress/e2e/team.cy.js index 074f3b7fc..a2e4caa6e 100644 --- a/e2e-tests/cypress/e2e/team.cy.js +++ b/e2e-tests/cypress/e2e/team.cy.js @@ -21,14 +21,17 @@ describe('Team edition tests', () => { cy.testid('AdminMenuWidget_itemConnections').click(); cy.contains('New connection').click(); cy.testid('ConnectionDriverFields_connectionType').select('PostgreSQL'); + cy.contains('not granted').should('not.exist'); cy.themeshot('connection-administration'); cy.testid('AdminMenuWidget_itemRoles').click(); cy.contains('logged-user').click(); + cy.contains('not granted').should('not.exist'); cy.themeshot('role-administration'); cy.testid('AdminMenuWidget_itemUsers').click(); cy.contains('New user').click(); + cy.contains('not granted').should('not.exist'); cy.themeshot('user-administration'); cy.testid('AdminMenuWidget_itemAuthentication').click(); @@ -36,6 +39,7 @@ describe('Team edition tests', () => { cy.contains('Use database login').click(); cy.contains('Add authentication').click(); cy.contains('OAuth 2.0').click(); + cy.contains('not granted').should('not.exist'); cy.themeshot('authentication-administration'); }); @@ -119,4 +123,29 @@ describe('Team edition tests', () => { cy.contains('Exporting query').click(); cy.themeshot('auditlog'); }); + + it('Edit database permissions', () => { + cy.testid('LoginPage_linkAdmin').click(); + cy.testid('LoginPage_password').type('adminpwd'); + cy.testid('LoginPage_submitLogin').click(); + + cy.testid('AdminMenuWidget_itemRoles').click(); + cy.testid('AdminRolesTab_table').contains('superadmin').click(); + cy.testid('AdminRolesTab_databases').click(); + + cy.testid('AdminDatabasesPermissionsGrid_addButton').click(); + cy.testid('AdminDatabasesPermissionsGrid_addButton').click(); + cy.testid('AdminDatabasesPermissionsGrid_addButton').click(); + + cy.testid('AdminListOrRegexEditor_1_regexInput').type('^Chinook[\\d]*$'); + cy.testid('AdminListOrRegexEditor_2_listSwitch').click(); + cy.testid('AdminListOrRegexEditor_2_listInput').type('Nortwind\nSales'); + cy.testid('AdminDatabasesPermissionsGrid_roleSelect_0').select('-2'); + cy.testid('AdminDatabasesPermissionsGrid_roleSelect_1').select('-3'); + cy.testid('AdminDatabasesPermissionsGrid_roleSelect_2').select('-4'); + + cy.contains('not granted').should('not.exist'); + + cy.themeshot('database-permissions'); + }); }); diff --git a/e2e-tests/package.json b/e2e-tests/package.json index 7f6aeaaf7..5ad1e3008 100644 --- a/e2e-tests/package.json +++ b/e2e-tests/package.json @@ -10,11 +10,11 @@ "cypress-real-events": "^1.13.0", "env-cmd": "^10.1.0", "kill-port": "^2.0.1", + "mocha-reporter-gha": "^1.1.1", "start-server-and-test": "^2.0.8" }, "scripts": { "cy:open": "cypress open --config experimentalInteractiveRunEvents=true", - "cy:run:add-connection": "cypress run --spec cypress/e2e/add-connection.cy.js", "cy:run:portal": "cypress run --spec cypress/e2e/portal.cy.js", "cy:run:oauth": "cypress run --spec cypress/e2e/oauth.cy.js", @@ -23,7 +23,6 @@ "cy:run:multi-sql": "cypress run --spec cypress/e2e/multi-sql.cy.js", "cy:run:cloud": "cypress run --spec cypress/e2e/cloud.cy.js", "cy:run:charts": "cypress run --spec cypress/e2e/charts.cy.js", - "start:add-connection": "node clearTestingData && cd .. && node packer/build/bundle.js --listen-api --run-e2e-tests", "start:portal": "node clearTestingData && cd .. && env-cmd -f e2e-tests/env/portal/.env node e2e-tests/init/portal.js && env-cmd -f e2e-tests/env/portal/.env node packer/build/bundle.js --listen-api --run-e2e-tests", "start:oauth": "node clearTestingData && cd .. && env-cmd -f e2e-tests/env/oauth/.env node packer/build/bundle.js --listen-api --run-e2e-tests", @@ -32,7 +31,6 @@ "start:multi-sql": "node clearTestingData && cd .. && env-cmd -f e2e-tests/env/multi-sql/.env node e2e-tests/init/multi-sql.js && env-cmd -f e2e-tests/env/multi-sql/.env node packer/build/bundle.js --listen-api --run-e2e-tests", "start:cloud": "node clearTestingData && cd .. && env-cmd -f e2e-tests/env/cloud/.env node packer/build/bundle.js --listen-api --run-e2e-tests", "start:charts": "node clearTestingData && cd .. && env-cmd -f e2e-tests/env/charts/.env node e2e-tests/init/charts.js && env-cmd -f e2e-tests/env/charts/.env node packer/build/bundle.js --listen-api --run-e2e-tests", - "test:add-connection": "start-server-and-test start:add-connection http://localhost:3000 cy:run:add-connection", "test:portal": "start-server-and-test start:portal http://localhost:3000 cy:run:portal", "test:oauth": "start-server-and-test start:oauth http://localhost:3000 cy:run:oauth", @@ -41,7 +39,6 @@ "test:multi-sql": "start-server-and-test start:multi-sql http://localhost:3000 cy:run:multi-sql", "test:cloud": "start-server-and-test start:cloud http://localhost:3000 cy:run:cloud", "test:charts": "start-server-and-test start:charts http://localhost:3000 cy:run:charts", - "test": "yarn test:add-connection && yarn test:portal && yarn test:oauth && yarn test:browse-data && yarn test:team && yarn test:multi-sql && yarn test:cloud && yarn test:charts", "test:ci": "yarn test" }, diff --git a/e2e-tests/yarn.lock b/e2e-tests/yarn.lock index 29aa0b706..05a4005e4 100644 --- a/e2e-tests/yarn.lock +++ b/e2e-tests/yarn.lock @@ -2,6 +2,34 @@ # yarn lockfile v1 +"@actions/core@^1.10.1": + version "1.11.1" + resolved "https://registry.yarnpkg.com/@actions/core/-/core-1.11.1.tgz#ae683aac5112438021588030efb53b1adb86f172" + integrity sha512-hXJCSrkwfA46Vd9Z3q4cpEpHB1rL5NG04+/rbqW9d3+CSvtB1tYe8UTpAlixa1vj0m/ULglfEK2UKxMGxCxv5A== + dependencies: + "@actions/exec" "^1.1.1" + "@actions/http-client" "^2.0.1" + +"@actions/exec@^1.1.1": + version "1.1.1" + resolved "https://registry.yarnpkg.com/@actions/exec/-/exec-1.1.1.tgz#2e43f28c54022537172819a7cf886c844221a611" + integrity sha512-+sCcHHbVdk93a0XT19ECtO/gIXoxvdsgQLzb2fE2/5sIZmWQuluYyjPQtrtTHdU1YzTZ7bAPN4sITq2xi1679w== + dependencies: + "@actions/io" "^1.0.1" + +"@actions/http-client@^2.0.1": + version "2.2.3" + resolved "https://registry.yarnpkg.com/@actions/http-client/-/http-client-2.2.3.tgz#31fc0b25c0e665754ed39a9f19a8611fc6dab674" + integrity sha512-mx8hyJi/hjFvbPokCg4uRd4ZX78t+YyRPtnKWwIl+RzNaVuFpQHfmlGVfsKEJN8LwTCvL+DfVgAM04XaHkm6bA== + dependencies: + tunnel "^0.0.6" + undici "^5.25.4" + +"@actions/io@^1.0.1": + version "1.1.3" + resolved "https://registry.yarnpkg.com/@actions/io/-/io-1.1.3.tgz#4cdb6254da7962b07473ff5c335f3da485d94d71" + integrity sha512-wi9JjgKLYS7U/z8PPbco+PvTb/nRWjeoFlJ1Qer83k/3C5PHQi28hiVdeE2kHXmIL99mQFawx8qt/JPjZilJ8Q== + "@colors/colors@1.5.0": version "1.5.0" resolved "https://registry.yarnpkg.com/@colors/colors/-/colors-1.5.0.tgz#bb504579c1cae923e6576a4f5da43d25f97bdbd9" @@ -39,6 +67,11 @@ debug "^3.1.0" lodash.once "^4.1.1" +"@fastify/busboy@^2.0.0": + version "2.1.1" + resolved "https://registry.yarnpkg.com/@fastify/busboy/-/busboy-2.1.1.tgz#b9da6a878a371829a0502c9b6c1c143ef6663f4d" + integrity sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA== + "@hapi/hoek@^9.0.0", "@hapi/hoek@^9.3.0": version "9.3.0" resolved "https://registry.yarnpkg.com/@hapi/hoek/-/hoek-9.3.0.tgz#8368869dcb735be2e7f5cb7647de78e167a251fb" @@ -947,6 +980,13 @@ minimist@^1.2.8: resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.8.tgz#c1a464e7693302e082a075cee0c057741ac4772c" integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA== +mocha-reporter-gha@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/mocha-reporter-gha/-/mocha-reporter-gha-1.1.1.tgz#e1248abd0769f55b57b36ccd7db2b0b6573d5adf" + integrity sha512-CFbcgM56V4yWlbF91XuwrE6a5X/IqjVXTPefO7m8cY8Es8G1UhJ2KKOrk16AcSemRzVWXp2Fdy3bWJ7j45snWw== + dependencies: + "@actions/core" "^1.10.1" + ms@^2.1.1, ms@^2.1.3: version "2.1.3" resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" @@ -1292,6 +1332,11 @@ tunnel-agent@^0.6.0: dependencies: safe-buffer "^5.0.1" +tunnel@^0.0.6: + version "0.0.6" + resolved "https://registry.yarnpkg.com/tunnel/-/tunnel-0.0.6.tgz#72f1314b34a5b192db012324df2cc587ca47f92c" + integrity sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg== + tweetnacl@^0.14.3, tweetnacl@~0.14.0: version "0.14.5" resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64" @@ -1307,6 +1352,13 @@ undici-types@~6.20.0: resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-6.20.0.tgz#8171bf22c1f588d1554d55bf204bc624af388433" integrity sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg== +undici@^5.25.4: + version "5.29.0" + resolved "https://registry.yarnpkg.com/undici/-/undici-5.29.0.tgz#419595449ae3f2cdcba3580a2e8903399bd1f5a3" + integrity sha512-raqeBD6NQK4SkWhQzeYKd1KmIG6dllBOTt55Rmkt4HtI9mwdWtJljnrXjAFUBLTSN67HWrOIZ3EPF4kjUw80Bg== + dependencies: + "@fastify/busboy" "^2.0.0" + universalify@^2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/universalify/-/universalify-2.0.1.tgz#168efc2180964e6386d061e094df61afe239b18d" diff --git a/integration-tests/__tests__/alter-table.spec.js b/integration-tests/__tests__/alter-table.spec.js index 65e1b2d3f..1aec9af5b 100644 --- a/integration-tests/__tests__/alter-table.spec.js +++ b/integration-tests/__tests__/alter-table.spec.js @@ -12,6 +12,7 @@ const { } = require('dbgate-tools'); function pickImportantTableInfo(engine, table) { + if (!table) return table; const props = ['columnName', 'defaultValue']; if (!engine.skipNullability) props.push('notNull'); if (!engine.skipAutoIncrement) props.push('autoIncrement'); @@ -25,6 +26,15 @@ function pickImportantTableInfo(engine, table) { .map(props => _.omitBy(props, (v, k) => k == 'defaultValue' && v == 'NULL' && engine.setNullDefaultInsteadOfDrop) ), + + // TODO: + foreignKeys: table.foreignKeys + .sort((a, b) => a.refTableName.localeCompare(b.refTableName)) + .map(fk => ({ + constraintType: fk.constraintType, + refTableName: fk.refTableName, + columns: fk.columns.map(col => ({ columnName: col.columnName, refColumnName: col.refColumnName })), + })), }; } @@ -33,7 +43,7 @@ function checkTableStructure(engine, t1, t2) { expect(pickImportantTableInfo(engine, t1)).toEqual(pickImportantTableInfo(engine, t2)); } -async function testTableDiff(engine, conn, driver, mangle) { +async function testTableDiff(engine, conn, driver, mangle, changedTable = 't1') { const initQuery = formatQueryWithoutParams(driver, `create table ~t0 (~id int not null primary key)`); await driver.query(conn, transformSqlForEngine(engine, initQuery)); @@ -68,17 +78,39 @@ async function testTableDiff(engine, conn, driver, mangle) { await driver.query(conn, transformSqlForEngine(engine, query)); } - const tget = x => x.tables.find(y => y.pureName == 't1'); - const structure1 = generateDbPairingId(extendDatabaseInfo(await driver.analyseFull(conn))); + if (!engine.skipReferences) { + const query = formatQueryWithoutParams( + driver, + `create table ~t3 (~id int not null primary key, ~fkval int ${ + driver.dialect.implicitNullDeclaration ? '' : 'null' + })` + ); + + await driver.query(conn, transformSqlForEngine(engine, query)); + } + + const tget = x => x?.tables?.find(y => y.pureName == changedTable); + const structure1Source = await driver.analyseFull(conn); + const structure1 = generateDbPairingId(extendDatabaseInfo(structure1Source)); let structure2 = _.cloneDeep(structure1); mangle(tget(structure2)); structure2 = extendDatabaseInfo(structure2); const { sql } = getAlterTableScript(tget(structure1), tget(structure2), {}, structure1, structure2, driver); + + // sleep 1s - some engines have update datetime precision only to seconds + await new Promise(resolve => setTimeout(resolve, 1000)); + console.log('RUNNING ALTER SQL', driver.engine, ':', sql); await driver.script(conn, sql); + // TODO: + // if (!engine.skipIncrementalAnalysis) { + // const structure2RealIncremental = await driver.analyseIncremental(conn, structure1Source); + // checkTableStructure(engine, tget(structure2RealIncremental), tget(structure2)); + // } + const structure2Real = extendDatabaseInfo(await driver.analyseFull(conn)); checkTableStructure(engine, tget(structure2Real), tget(structure2)); @@ -87,6 +119,7 @@ async function testTableDiff(engine, conn, driver, mangle) { const TESTED_COLUMNS = ['col_pk', 'col_std', 'col_def', 'col_fk', 'col_ref', 'col_idx', 'col_uq']; // const TESTED_COLUMNS = ['col_pk']; +// const TESTED_COLUMNS = ['col_fk']; // const TESTED_COLUMNS = ['col_idx']; // const TESTED_COLUMNS = ['col_def']; // const TESTED_COLUMNS = ['col_std']; @@ -150,11 +183,25 @@ describe('Alter table', () => { )( 'Drop column - %s - %s', testWrapper(async (conn, driver, column, engine) => { - await testTableDiff(engine, conn, driver, tbl => (tbl.columns = tbl.columns.filter(x => x.columnName != column))); + await testTableDiff(engine, conn, driver, + tbl => { + tbl.columns = tbl.columns.filter(x => x.columnName != column); + tbl.foreignKeys = tbl.foreignKeys + .map(fk => ({ + ...fk, + columns: fk.columns.filter(col => col.columnName != column) + })) + .filter(fk => fk.columns.length > 0); + } + ); }) ); - test.each(createEnginesColumnsSource(engines.filter(x => !x.skipNullable && !x.skipChangeNullability)))( + test.each( + createEnginesColumnsSource(engines.filter(x => !x.skipNullability && !x.skipChangeNullability)).filter( + ([_label, col]) => !col.endsWith('_pk') + ) + )( 'Change nullability - %s - %s', testWrapper(async (conn, driver, column, engine) => { await testTableDiff( @@ -173,7 +220,11 @@ describe('Alter table', () => { engine, conn, driver, - tbl => (tbl.columns = tbl.columns.map(x => (x.columnName == column ? { ...x, columnName: 'col_renamed' } : x))) + tbl => { + tbl.columns = tbl.columns.map(x => (x.columnName == column ? { ...x, columnName: 'col_renamed' } : x)); + tbl.foreignKeys = tbl.foreignKeys.map(fk => ({...fk, columns: fk.columns.map(col => col.columnName == column ? { ...col, columnName: 'col_renamed' } : col) + })); + } ); }) ); @@ -214,6 +265,48 @@ describe('Alter table', () => { }) ); + test.each(engines.filter(x => !x.skipReferences).map(engine => [engine.label, engine]))( + 'Drop FK - %s', + testWrapper(async (conn, driver, engine) => { + await testTableDiff( + engine, + conn, + driver, + tbl => { + tbl.foreignKeys = []; + }, + 't2' + ); + }) + ); + + test.each(engines.filter(x => !x.skipReferences).map(engine => [engine.label, engine]))( + 'Create FK - %s', + testWrapper(async (conn, driver, engine) => { + await testTableDiff( + engine, + conn, + driver, + tbl => { + tbl.foreignKeys = [ + { + constraintType: 'foreignKey', + pureName: 't3', + refTableName: 't1', + columns: [ + { + columnName: 'fkval', + refColumnName: 'col_ref', + }, + ], + }, + ]; + }, + 't3' + ); + }) + ); + // test.each(engines.map(engine => [engine.label, engine]))( // 'Change autoincrement - %s', // testWrapper(async (conn, driver, engine) => { diff --git a/integration-tests/__tests__/data-replicator.spec.js b/integration-tests/__tests__/data-replicator.spec.js index d9c089c85..95811ea49 100644 --- a/integration-tests/__tests__/data-replicator.spec.js +++ b/integration-tests/__tests__/data-replicator.spec.js @@ -303,4 +303,52 @@ describe('Data replicator', () => { }), 15 * 1000 ); + + test.each(engines.filter(x => !x.skipDataReplicator).map(engine => [engine.label, engine]))( + 'Skip columns for update - %s', + testWrapper(async (conn, driver, engine) => { + runCommandOnDriver(conn, driver, dmp => + dmp.createTable({ + pureName: 't1', + columns: [ + { columnName: 'id', dataType: 'int', autoIncrement: true, notNull: true }, + { columnName: 'key', dataType: 'varchar(50)', notNull: true }, + { columnName: 'val', dataType: 'varchar(50)' }, + ], + primaryKey: { + columns: [{ columnName: 'id' }], + }, + }) + ); + + const getcfg = (v1 = 'v1') => ({ + systemConnection: conn, + driver, + items: [ + { + name: 't1', + matchColumns: ['key'], + skipUpdateColumns: ['val'], + findExisting: true, + updateExisting: true, + createNew: true, + jsonArray: [ + { key: '1', val: v1 }, + { key: '2', val: 'v2' }, + { key: '3', val: 'v3' }, + ], + }, + ], + }); + + await dataReplicator(getcfg('v1')); + + const res1 = await runQueryOnDriver(conn, driver, dmp => dmp.put(`select ~val from ~t1 where ~key='1'`)); + expect(res1.rows[0].val).toEqual('v1'); + + await dataReplicator(getcfg('v2')); + const res2 = await runQueryOnDriver(conn, driver, dmp => dmp.put(`select ~val from ~t1 where ~key='1'`)); + expect(res2.rows[0].val).toEqual('v1'); + }) + ); }); diff --git a/integration-tests/__tests__/query.spec.js b/integration-tests/__tests__/query.spec.js index b8afeadd4..b35f10c91 100644 --- a/integration-tests/__tests__/query.spec.js +++ b/integration-tests/__tests__/query.spec.js @@ -49,6 +49,32 @@ class StreamHandler { } } +class BinaryTestStreamHandler { + constructor(resolve, reject, expectedValue) { + this.resolve = resolve; + this.reject = reject; + this.expectedValue = expectedValue; + this.rowsReceived = []; + } + row(row) { + try { + this.rowsReceived.push(row); + if (this.expectedValue) { + expect(row).toEqual(this.expectedValue); + } + } catch (error) { + this.reject(error); + return; + } + } + recordset(columns) {} + done(result) { + this.resolve(this.rowsReceived); + } + info(msg) {} +} + + function executeStreamItem(driver, conn, sql) { return new Promise(resolve => { const handler = new StreamHandler(resolve); @@ -223,4 +249,51 @@ describe('Query', () => { expect(row[keys[0]] == 1).toBeTruthy(); }) ); + + test.each(engines.filter(x => x.binaryDataType).map(engine => [engine.label, engine]))( + 'Binary - %s', + testWrapper(async (dbhan, driver, engine) => { + await runCommandOnDriver(dbhan, driver, dmp => + dmp.createTable({ + pureName: 't1', + columns: [ + { columnName: 'id', dataType: 'int', notNull: true, autoIncrement: true }, + { columnName: 'val', dataType: engine.binaryDataType }, + ], + primaryKey: { + columns: [{ columnName: 'id' }], + }, + }) + ); + const structure = await driver.analyseFull(dbhan); + const table = structure.tables.find(x => x.pureName == 't1'); + + const dmp = driver.createDumper(); + dmp.putCmd("INSERT INTO ~t1 (~val) VALUES (%v)", { + $binary: { base64: 'iVBORw0KWgo=' }, + }); + await driver.query(dbhan, dmp.s, {discardResult: true}); + + const dmp2 = driver.createDumper(); + dmp2.put('SELECT ~val FROM ~t1'); + const res = await driver.query(dbhan, dmp2.s); + + const row = res.rows[0]; + const keys = Object.keys(row); + expect(keys.length).toEqual(1); + expect(row[keys[0]]).toEqual({$binary: {base64: 'iVBORw0KWgo='}}); + + const res2 = await driver.readQuery(dbhan, dmp2.s); + const rows = await Array.fromAsync(res2); + const rowsVal = rows.filter(r => r.val != null); + + expect(rowsVal.length).toEqual(1); + expect(rowsVal[0].val).toEqual({$binary: {base64: 'iVBORw0KWgo='}}); + + const res3 = await new Promise((resolve, reject) => { + const handler = new BinaryTestStreamHandler(resolve, reject, {val: {$binary: {base64: 'iVBORw0KWgo='}}}); + driver.stream(dbhan, dmp2.s, handler); + }); + }) + ); }); diff --git a/integration-tests/__tests__/schema-tests.spec.js b/integration-tests/__tests__/schema-tests.spec.js index fda045c90..53aa0bfb5 100644 --- a/integration-tests/__tests__/schema-tests.spec.js +++ b/integration-tests/__tests__/schema-tests.spec.js @@ -33,7 +33,9 @@ describe('Schema tests', () => { expect(schemas2.find(x => x.schemaName == 'myschema')).toBeTruthy(); expect(schemas2.length).toEqual(count + 1); expect(schemas2.find(x => x.isDefault).schemaName).toEqual(engine.defaultSchemaName); - expect(structure2).toBeNull(); + if (!engine.skipIncrementalAnalysis) { + expect(structure2).toBeNull(); + } }) ); @@ -51,7 +53,9 @@ describe('Schema tests', () => { const structure2 = await driver.analyseIncremental(conn, structure1); const schemas2 = await driver.listSchemas(conn); expect(schemas2.find(x => x.schemaName == 'myschema')).toBeFalsy(); - expect(structure2).toBeNull(); + if (!engine.skipIncrementalAnalysis) { + expect(structure2).toBeNull(); + } }) ); diff --git a/integration-tests/__tests__/table-analyse.spec.js b/integration-tests/__tests__/table-analyse.spec.js index 570a2e2f2..705adf077 100644 --- a/integration-tests/__tests__/table-analyse.spec.js +++ b/integration-tests/__tests__/table-analyse.spec.js @@ -94,7 +94,7 @@ describe('Table analyse', () => { }) ); - test.each(engines.filter(x => !x.skipIncrementalAnalysis).map(engine => [engine.label, engine]))( + test.each(engines.map(engine => [engine.label, engine]))( 'Table add - incremental analysis - %s', testWrapper(async (conn, driver, engine) => { await runCommandOnDriver(conn, driver, dmp => dmp.put(t2Sql(engine))); @@ -112,7 +112,7 @@ describe('Table analyse', () => { }) ); - test.each(engines.filter(x => !x.skipIncrementalAnalysis).map(engine => [engine.label, engine]))( + test.each(engines.map(engine => [engine.label, engine]))( 'Table remove - incremental analysis - %s', testWrapper(async (conn, driver, engine) => { await runCommandOnDriver(conn, driver, dmp => dmp.put(t1Sql(engine))); @@ -130,7 +130,7 @@ describe('Table analyse', () => { }) ); - test.each(engines.filter(x => !x.skipIncrementalAnalysis).map(engine => [engine.label, engine]))( + test.each(engines.map(engine => [engine.label, engine]))( 'Table change - incremental analysis - %s', testWrapper(async (conn, driver, engine) => { await runCommandOnDriver(conn, driver, dmp => dmp.put(t1Sql(engine))); diff --git a/integration-tests/docker-compose.yaml b/integration-tests/docker-compose.yaml index c64836a4d..e48b5ddb1 100644 --- a/integration-tests/docker-compose.yaml +++ b/integration-tests/docker-compose.yaml @@ -44,7 +44,7 @@ services: # - 15942:9042 # # clickhouse: - # image: bitnami/clickhouse:24.8.4 + # image: bitnamilegacy/clickhouse:24.8.4 # restart: always # ports: # - 15005:8123 diff --git a/integration-tests/engines.js b/integration-tests/engines.js index 6c16040a9..528925e2c 100644 --- a/integration-tests/engines.js +++ b/integration-tests/engines.js @@ -44,6 +44,7 @@ const mysqlEngine = { supportRenameSqlObject: false, dbSnapshotBySeconds: true, dumpFile: 'data/chinook-mysql.sql', + binaryDataType: 'blob', dumpChecks: [ { sql: 'select count(*) as res from genre', @@ -186,6 +187,7 @@ const mariaDbEngine = { /** @type {import('dbgate-types').TestEngineInfo} */ const postgreSqlEngine = { label: 'PostgreSQL', + skipIncrementalAnalysis: true, connection: { engine: 'postgres@dbgate-plugin-postgres', password: 'Pwd2020Db', @@ -216,6 +218,7 @@ const postgreSqlEngine = { supportSchemas: true, supportRenameSqlObject: true, defaultSchemaName: 'public', + binaryDataType: 'bytea', dumpFile: 'data/chinook-postgre.sql', dumpChecks: [ { @@ -446,6 +449,7 @@ const sqlServerEngine = { supportTableComments: true, supportColumnComments: true, // skipSeparateSchemas: true, + binaryDataType: 'varbinary(100)', triggers: [ { testName: 'triggers before each row', @@ -506,6 +510,7 @@ const sqliteEngine = { }, }, ], + binaryDataType: 'blob', }; const libsqlFileEngine = { @@ -619,6 +624,7 @@ const oracleEngine = { }, }, ], + binaryDataType: 'blob', }; /** @type {import('dbgate-types').TestEngineInfo} */ @@ -754,16 +760,16 @@ const enginesOnLocal = [ // cassandraEngine, // mysqlEngine, // mariaDbEngine, - // postgreSqlEngine, - // sqlServerEngine, + postgreSqlEngine, + //sqlServerEngine, // sqliteEngine, // cockroachDbEngine, // clickhouseEngine, // libsqlFileEngine, // libsqlWsEngine, - // oracleEngine, + //oracleEngine, // duckdbEngine, - firebirdEngine, + //firebirdEngine, ]; /** @type {import('dbgate-types').TestEngineInfo[] & Record} */ diff --git a/integration-tests/jest.config.js b/integration-tests/jest.config.js index 83368208f..c3c764a9b 100644 --- a/integration-tests/jest.config.js +++ b/integration-tests/jest.config.js @@ -1,3 +1,4 @@ module.exports = { setupFilesAfterEnv: ['/setupTests.js'], + reporters: ['default', 'github-actions'], }; diff --git a/integration-tests/package.json b/integration-tests/package.json index a8c3e670b..8bf77ab50 100644 --- a/integration-tests/package.json +++ b/integration-tests/package.json @@ -18,7 +18,7 @@ }, "devDependencies": { "cross-env": "^7.0.3", - "jest": "^27.0.1", + "jest": "^28.1.3", "pino-pretty": "^11.2.2", "tmp": "^0.2.3" } diff --git a/integration-tests/tools.js b/integration-tests/tools.js index a1a14dfd9..55b316e8a 100644 --- a/integration-tests/tools.js +++ b/integration-tests/tools.js @@ -22,7 +22,9 @@ async function connect(engine, database) { if (engine.generateDbFile) { const conn = await driver.connect({ ...connection, - databaseFile: (engine.databaseFileLocationOnServer ?? 'dbtemp/') + database, + databaseFile: + (engine.databaseFileLocationOnServer ?? (process.env.CITEST ? 'dbtemp/' : 'integration-tests/dbtemp/')) + + database, }); return conn; } else { diff --git a/package.json b/package.json index 240a4d409..ee6a5b97b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "private": true, - "version": "6.6.1", + "version": "6.8.0", "name": "dbgate-all", "workspaces": [ "packages/*", @@ -22,6 +22,7 @@ "start:api:auth": "yarn workspace dbgate-api start:auth | pino-pretty", "start:api:dblogin": "yarn workspace dbgate-api start:dblogin | pino-pretty", "start:api:storage": "yarn workspace dbgate-api start:storage | pino-pretty", + "start:api:sfill": "yarn workspace dbgate-api start:sfill | pino-pretty", "start:api:storage:built": "yarn workspace dbgate-api start:storage:built | pino-pretty", "start:api:azure": "yarn workspace dbgate-api start:azure | pino-pretty", "start:api:e2e:team": "yarn workspace dbgate-api start:e2e:team | pino-pretty", @@ -52,6 +53,7 @@ "generatePadFile": "node generatePadFile", "fillPackagedPlugins": "node fillPackagedPlugins", "resetPackagedPlugins": "node resetPackagedPlugins", + "fixYmlHashes": "cd common && yarn init -y && yarn add yaml -W && cd .. && node common/fixYmlHashes.js app/dist", "prettier": "prettier --write packages/api/src && prettier --write packages/datalib/src && prettier --write packages/filterparser/src && prettier --write packages/sqltree/src && prettier --write packages/tools/src && prettier --write packages/types && prettier --write packages/web/src && prettier --write app/src", "copy:docker:build": "copyfiles packages/api/dist/* docker -f && copyfiles packages/web/public/* docker -u 2 && copyfiles \"packages/web/public/**/*\" docker -u 2 && copyfiles \"plugins/dist/**/*\" docker/plugins -u 2", "copy:packer:build": "copyfiles packages/api/dist/* packer/build -f && copyfiles packages/web/public/* packer/build -u 2 && copyfiles \"packages/web/public/**/*\" packer/build -u 2 && copyfiles \"plugins/dist/**/*\" packer/build/plugins -u 2 && copyfiles packer/install-packages.sh packer/build -f && yarn install:drivers:packer", @@ -73,6 +75,8 @@ "translations:add-missing": "node common/translations-cli/index.js add-missing", "translations:remove-unused": "node common/translations-cli/index.js remove-unused", "translations:check": "node common/translations-cli/index.js check", + "translations:sort": "node common/translations-cli/index.js sort", + "translations:translate": "node common/translations-cli/translate.js", "errors": "node common/assign-dbgm-codes.mjs ." }, "dependencies": { diff --git a/packages/api/.env b/packages/api/.env index 9ab8fc3d2..2e7ce8a5b 100644 --- a/packages/api/.env +++ b/packages/api/.env @@ -2,6 +2,7 @@ DEVMODE=1 SHELL_SCRIPTING=1 ALLOW_DBGATE_PRIVATE_CLOUD=1 DEVWEB=1 +# LOCAL_AUTH_PROXY=1 # LOCAL_AI_GATEWAY=true # REDIRECT_TO_DBGATE_CLOUD_LOGIN=1 diff --git a/packages/api/env/sfill/.env b/packages/api/env/sfill/.env new file mode 100644 index 000000000..68462836c --- /dev/null +++ b/packages/api/env/sfill/.env @@ -0,0 +1,46 @@ +DEVMODE=1 +DEVWEB=1 + +STORAGE_SERVER=localhost +STORAGE_USER=root +STORAGE_PASSWORD=Pwd2020Db +STORAGE_PORT=3306 +STORAGE_DATABASE=dbgate-filled +STORAGE_ENGINE=mysql@dbgate-plugin-mysql + +CONNECTIONS=mysql,postgres,mongo,redis + +LABEL_mysql=MySql +SERVER_mysql=dbgatedckstage1.sprinx.cz +USER_mysql=root +PASSWORD_mysql=Pwd2020Db +PORT_mysql=3306 +ENGINE_mysql=mysql@dbgate-plugin-mysql + +LABEL_postgres=Postgres +SERVER_postgres=dbgatedckstage1.sprinx.cz +USER_postgres=postgres +PASSWORD_postgres=Pwd2020Db +PORT_postgres=5432 +ENGINE_postgres=postgres@dbgate-plugin-postgres + +LABEL_mongo=Mongo +SERVER_mongo=dbgatedckstage1.sprinx.cz +USER_mongo=root +PASSWORD_mongo=Pwd2020Db +PORT_mongo=27017 +ENGINE_mongo=mongo@dbgate-plugin-mongo + +LABEL_redis=Redis +SERVER_redis=dbgatedckstage1.sprinx.cz +ENGINE_redis=redis@dbgate-plugin-redis +PORT_redis=6379 + +ROLE_test1_CONNECTIONS=mysql +ROLE_test1_PERMISSIONS=widgets/* +ROLE_test1_DATABASES_db1_CONNECTION=mysql +ROLE_test1_DATABASES_db1_PERMISSION=run_script +ROLE_test1_DATABASES_db1_DATABASES=db1 +ROLE_test1_DATABASES_db2_CONNECTION=redis +ROLE_test1_DATABASES_db2_PERMISSION=run_script +ROLE_test1_DATABASES_db2_DATABASES=db2 diff --git a/packages/api/package.json b/packages/api/package.json index 7eeda8fe4..c347bd5bb 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -31,7 +31,7 @@ "cors": "^2.8.5", "cross-env": "^6.0.3", "dbgate-datalib": "^6.0.0-alpha.1", - "dbgate-query-splitter": "^4.11.5", + "dbgate-query-splitter": "^4.11.9", "dbgate-sqltree": "^6.0.0-alpha.1", "dbgate-tools": "^6.0.0-alpha.1", "debug": "^4.3.4", @@ -75,6 +75,7 @@ "start:dblogin": "env-cmd -f env/dblogin/.env node src/index.js --listen-api", "start:filedb": "env-cmd node src/index.js /home/jena/test/chinook/Chinook.db --listen-api", "start:storage": "env-cmd -f env/storage/.env node src/index.js --listen-api", + "start:sfill": "env-cmd -f env/sfill/.env node src/index.js --listen-api", "start:storage:built": "env-cmd -f env/storage/.env cross-env DEVMODE= BUILTWEBMODE=1 node dist/bundle.js --listen-api", "start:singleconn": "env-cmd node src/index.js --server localhost --user root --port 3307 --engine mysql@dbgate-plugin-mysql --password test --listen-api", "start:azure": "env-cmd -f env/azure/.env node src/index.js --listen-api", diff --git a/packages/api/src/auth/authCommon.js b/packages/api/src/auth/authCommon.js index 824b9baa8..9863dfefe 100644 --- a/packages/api/src/auth/authCommon.js +++ b/packages/api/src/auth/authCommon.js @@ -10,7 +10,13 @@ function getTokenSecret() { return tokenSecret; } +function getStaticTokenSecret() { + // TODO static not fixed + return '14813c43-a91b-4ad1-9dcd-a81bd7dbb05f'; +} + module.exports = { getTokenLifetime, getTokenSecret, + getStaticTokenSecret, }; diff --git a/packages/api/src/auth/authProvider.js b/packages/api/src/auth/authProvider.js index 8e65b6129..af57d0cd4 100644 --- a/packages/api/src/auth/authProvider.js +++ b/packages/api/src/auth/authProvider.js @@ -10,6 +10,7 @@ const logger = getLogger('authProvider'); class AuthProviderBase { amoid = 'none'; + skipInList = false; async login(login, password, options = undefined, req = undefined) { return { @@ -36,12 +37,28 @@ class AuthProviderBase { return !!req?.user || !!req?.auth; } - getCurrentPermissions(req) { + async getCurrentPermissions(req) { const login = this.getCurrentLogin(req); const permissions = process.env[`LOGIN_PERMISSIONS_${login}`]; return permissions || process.env.PERMISSIONS; } + async checkCurrentConnectionPermission(req, conid) { + return true; + } + + async getCurrentDatabasePermissions(req) { + return []; + } + + async getCurrentTablePermissions(req) { + return []; + } + + async getCurrentFilePermissions(req) { + return []; + } + getLoginPageConnections() { return null; } diff --git a/packages/api/src/controllers/apps.js b/packages/api/src/controllers/apps.js index 43c2d6e28..9425013e4 100644 --- a/packages/api/src/controllers/apps.js +++ b/packages/api/src/controllers/apps.js @@ -1,233 +1,99 @@ const fs = require('fs-extra'); const _ = require('lodash'); const path = require('path'); -const { appdir } = require('../utility/directories'); +const { appdir, filesdir } = require('../utility/directories'); const socket = require('../utility/socket'); const connections = require('./connections'); +const { + loadPermissionsFromRequest, + loadFilePermissionsFromRequest, + hasPermission, + getFilePermissionRole, +} = require('../utility/hasPermission'); module.exports = { - folders_meta: true, - async folders() { - const folders = await fs.readdir(appdir()); - return [ - ...folders.map(name => ({ - name, - })), - ]; - }, - - createFolder_meta: true, - async createFolder({ folder }) { - const name = await this.getNewAppFolder({ name: folder }); - await fs.mkdir(path.join(appdir(), name)); - socket.emitChanged('app-folders-changed'); - this.emitChangedDbApp(folder); - return name; - }, - - files_meta: true, - async files({ folder }) { - if (!folder) return []; - const dir = path.join(appdir(), folder); + getAllApps_meta: true, + async getAllApps({}, req) { + const dir = path.join(filesdir(), 'apps'); if (!(await fs.exists(dir))) return []; - const files = await fs.readdir(dir); - - function fileType(ext, type) { - return files - .filter(name => name.endsWith(ext)) - .map(name => ({ - name: name.slice(0, -ext.length), - label: path.parse(name.slice(0, -ext.length)).base, - type, - })); - } - - return [ - ...fileType('.command.sql', 'command.sql'), - ...fileType('.query.sql', 'query.sql'), - ...fileType('.config.json', 'config.json'), - ]; - }, - - async emitChangedDbApp(folder) { - const used = await this.getUsedAppFolders(); - if (used.includes(folder)) { - socket.emitChanged('used-apps-changed'); - } - }, - - refreshFiles_meta: true, - async refreshFiles({ folder }) { - socket.emitChanged('app-files-changed', { app: folder }); - }, - - refreshFolders_meta: true, - async refreshFolders() { - socket.emitChanged(`app-folders-changed`); - }, - - deleteFile_meta: true, - async deleteFile({ folder, file, fileType }) { - await fs.unlink(path.join(appdir(), folder, `${file}.${fileType}`)); - socket.emitChanged('app-files-changed', { app: folder }); - this.emitChangedDbApp(folder); - }, - - renameFile_meta: true, - async renameFile({ folder, file, newFile, fileType }) { - await fs.rename( - path.join(path.join(appdir(), folder), `${file}.${fileType}`), - path.join(path.join(appdir(), folder), `${newFile}.${fileType}`) - ); - socket.emitChanged('app-files-changed', { app: folder }); - this.emitChangedDbApp(folder); - }, - - renameFolder_meta: true, - async renameFolder({ folder, newFolder }) { - const uniqueName = await this.getNewAppFolder({ name: newFolder }); - await fs.rename(path.join(appdir(), folder), path.join(appdir(), uniqueName)); - socket.emitChanged(`app-folders-changed`); - }, - - deleteFolder_meta: true, - async deleteFolder({ folder }) { - if (!folder) throw new Error('Missing folder parameter'); - await fs.rmdir(path.join(appdir(), folder), { recursive: true }); - socket.emitChanged(`app-folders-changed`); - socket.emitChanged('app-files-changed', { app: folder }); - socket.emitChanged('used-apps-changed'); - }, - - async getNewAppFolder({ name }) { - if (!(await fs.exists(path.join(appdir(), name)))) return name; - let index = 2; - while (await fs.exists(path.join(appdir(), `${name}${index}`))) { - index += 1; - } - return `${name}${index}`; - }, - - getUsedAppFolders_meta: true, - async getUsedAppFolders() { - const list = await connections.list(); - const apps = []; - - for (const connection of list) { - for (const db of connection.databases || []) { - for (const key of _.keys(db || {})) { - if (key.startsWith('useApp:') && db[key]) { - apps.push(key.substring('useApp:'.length)); - } - } - } - } - - return _.intersection(_.uniq(apps), await fs.readdir(appdir())); - }, - - getUsedApps_meta: true, - async getUsedApps() { - const apps = await this.getUsedAppFolders(); const res = []; + const loadedPermissions = await loadPermissionsFromRequest(req); + const filePermissions = await loadFilePermissionsFromRequest(req); - for (const folder of apps) { - res.push(await this.loadApp({ folder })); - } - return res; - }, - - // getAppsForDb_meta: true, - // async getAppsForDb({ conid, database }) { - // const connection = await connections.get({ conid }); - // if (!connection) return []; - // const db = (connection.databases || []).find(x => x.name == database); - // const apps = []; - // const res = []; - // if (db) { - // for (const key of _.keys(db || {})) { - // if (key.startsWith('useApp:') && db[key]) { - // apps.push(key.substring('useApp:'.length)); - // } - // } - // } - // for (const folder of apps) { - // res.push(await this.loadApp({ folder })); - // } - // return res; - // }, - - loadApp_meta: true, - async loadApp({ folder }) { - const res = { - queries: [], - commands: [], - name: folder, - }; - const dir = path.join(appdir(), folder); - if (await fs.exists(dir)) { - const files = await fs.readdir(dir); - - async function processType(ext, field) { - for (const file of files) { - if (file.endsWith(ext)) { - res[field].push({ - name: file.slice(0, -ext.length), - sql: await fs.readFile(path.join(dir, file), { encoding: 'utf-8' }), - }); - } - } + for (const file of await fs.readdir(dir)) { + if (!hasPermission(`all-disk-files`, loadedPermissions)) { + const role = getFilePermissionRole('apps', file, filePermissions); + if (role == 'deny') continue; } + const content = await fs.readFile(path.join(dir, file), { encoding: 'utf-8' }); + const appJson = JSON.parse(content); + // const app = { + // appid: file, + // name: appJson.applicationName, + // usageRules: appJson.usageRules || [], + // icon: appJson.applicationIcon || 'img app', + // color: appJson.applicationColor, + // queries: Object.values(appJson.files || {}) + // .filter(x => x.type == 'query') + // .map(x => ({ + // name: x.label, + // sql: x.sql, + // })), + // commands: Object.values(appJson.files || {}) + // .filter(x => x.type == 'command') + // .map(x => ({ + // name: x.label, + // sql: x.sql, + // })), + // virtualReferences: appJson.virtualReferences, + // dictionaryDescriptions: appJson.dictionaryDescriptions, + // }; + const app = { + ...appJson, + appid: file, + }; - await processType('.command.sql', 'commands'); - await processType('.query.sql', 'queries'); + res.push(app); } - - try { - res.virtualReferences = JSON.parse( - await fs.readFile(path.join(dir, 'virtual-references.config.json'), { encoding: 'utf-8' }) - ); - } catch (err) { - res.virtualReferences = []; - } - try { - res.dictionaryDescriptions = JSON.parse( - await fs.readFile(path.join(dir, 'dictionary-descriptions.config.json'), { encoding: 'utf-8' }) - ); - } catch (err) { - res.dictionaryDescriptions = []; - } - return res; }, - async saveConfigFile(appFolder, filename, filterFunc, newItem) { - const file = path.join(appdir(), appFolder, filename); - - let json; - try { - json = JSON.parse(await fs.readFile(file, { encoding: 'utf-8' })); - } catch (err) { - json = []; + createAppFromDb_meta: true, + async createAppFromDb({ appName, server, database }, req) { + const appdir = path.join(filesdir(), 'apps'); + if (!fs.existsSync(appdir)) { + await fs.mkdir(appdir); } - - if (filterFunc) { - json = json.filter(filterFunc); + const appId = _.kebabCase(appName); + let suffix = undefined; + while (fs.existsSync(path.join(appdir, `${appId}${suffix || ''}`))) { + if (!suffix) suffix = 2; + else suffix++; } + const finalAppId = `${appId}${suffix || ''}`; - json = [...json, newItem]; + const appJson = { + applicationName: appName, + usageRules: [ + { + serverHostsList: server, + databaseNamesList: database, + }, + ], + }; - await fs.writeFile(file, JSON.stringify(json, undefined, 2)); + await fs.writeFile(path.join(appdir, `${finalAppId}`), JSON.stringify(appJson, undefined, 2)); - socket.emitChanged('app-files-changed', { app: appFolder }); - socket.emitChanged('used-apps-changed'); + socket.emitChanged(`files-changed`, { folder: 'apps' }); + + return finalAppId; }, saveVirtualReference_meta: true, - async saveVirtualReference({ appFolder, schemaName, pureName, refSchemaName, refTableName, columns }) { - await this.saveConfigFile( - appFolder, - 'virtual-references.config.json', + async saveVirtualReference({ appid, schemaName, pureName, refSchemaName, refTableName, columns }) { + await this.saveConfigItem( + appid, + 'virtualReferences', columns.length == 1 ? x => !( @@ -245,14 +111,17 @@ module.exports = { columns, } ); + + socket.emitChanged(`files-changed`, { folder: 'apps' }); + return true; }, saveDictionaryDescription_meta: true, - async saveDictionaryDescription({ appFolder, pureName, schemaName, expression, columns, delimiter }) { - await this.saveConfigFile( - appFolder, - 'dictionary-descriptions.config.json', + async saveDictionaryDescription({ appid, pureName, schemaName, expression, columns, delimiter }) { + await this.saveConfigItem( + appid, + 'dictionaryDescriptions', x => !(x.schemaName == schemaName && x.pureName == pureName), { schemaName, @@ -263,18 +132,271 @@ module.exports = { } ); + socket.emitChanged(`files-changed`, { folder: 'apps' }); + return true; }, - createConfigFile_meta: true, - async createConfigFile({ appFolder, fileName, content }) { - const file = path.join(appdir(), appFolder, fileName); - if (!(await fs.exists(file))) { - await fs.writeFile(file, JSON.stringify(content, undefined, 2)); - socket.emitChanged('app-files-changed', { app: appFolder }); - socket.emitChanged('used-apps-changed'); - return true; + async saveConfigItem(appid, fieldName, filterFunc, newItem) { + const file = path.join(filesdir(), 'apps', appid); + + const appJson = JSON.parse(await fs.readFile(file, { encoding: 'utf-8' })); + let json = appJson[fieldName] || []; + + if (filterFunc) { + json = json.filter(filterFunc); } - return false; + + json = [...json, newItem]; + + await fs.writeFile( + file, + JSON.stringify( + { + ...appJson, + [fieldName]: json, + }, + undefined, + 2 + ) + ); + + socket.emitChanged('files-changed', { folder: 'apps' }); }, + + // folders_meta: true, + // async folders() { + // const folders = await fs.readdir(appdir()); + // return [ + // ...folders.map(name => ({ + // name, + // })), + // ]; + // }, + + // createFolder_meta: true, + // async createFolder({ folder }) { + // const name = await this.getNewAppFolder({ name: folder }); + // await fs.mkdir(path.join(appdir(), name)); + // socket.emitChanged('app-folders-changed'); + // this.emitChangedDbApp(folder); + // return name; + // }, + + // files_meta: true, + // async files({ folder }) { + // if (!folder) return []; + // const dir = path.join(appdir(), folder); + // if (!(await fs.exists(dir))) return []; + // const files = await fs.readdir(dir); + + // function fileType(ext, type) { + // return files + // .filter(name => name.endsWith(ext)) + // .map(name => ({ + // name: name.slice(0, -ext.length), + // label: path.parse(name.slice(0, -ext.length)).base, + // type, + // })); + // } + + // return [ + // ...fileType('.command.sql', 'command.sql'), + // ...fileType('.query.sql', 'query.sql'), + // ...fileType('.config.json', 'config.json'), + // ]; + // }, + + // async emitChangedDbApp(folder) { + // const used = await this.getUsedAppFolders(); + // if (used.includes(folder)) { + // socket.emitChanged('used-apps-changed'); + // } + // }, + + // refreshFiles_meta: true, + // async refreshFiles({ folder }) { + // socket.emitChanged('app-files-changed', { app: folder }); + // }, + + // refreshFolders_meta: true, + // async refreshFolders() { + // socket.emitChanged(`app-folders-changed`); + // }, + + // deleteFile_meta: true, + // async deleteFile({ folder, file, fileType }) { + // await fs.unlink(path.join(appdir(), folder, `${file}.${fileType}`)); + // socket.emitChanged('app-files-changed', { app: folder }); + // this.emitChangedDbApp(folder); + // }, + + // renameFile_meta: true, + // async renameFile({ folder, file, newFile, fileType }) { + // await fs.rename( + // path.join(path.join(appdir(), folder), `${file}.${fileType}`), + // path.join(path.join(appdir(), folder), `${newFile}.${fileType}`) + // ); + // socket.emitChanged('app-files-changed', { app: folder }); + // this.emitChangedDbApp(folder); + // }, + + // renameFolder_meta: true, + // async renameFolder({ folder, newFolder }) { + // const uniqueName = await this.getNewAppFolder({ name: newFolder }); + // await fs.rename(path.join(appdir(), folder), path.join(appdir(), uniqueName)); + // socket.emitChanged(`app-folders-changed`); + // }, + + // deleteFolder_meta: true, + // async deleteFolder({ folder }) { + // if (!folder) throw new Error('Missing folder parameter'); + // await fs.rmdir(path.join(appdir(), folder), { recursive: true }); + // socket.emitChanged(`app-folders-changed`); + // socket.emitChanged('app-files-changed', { app: folder }); + // socket.emitChanged('used-apps-changed'); + // }, + + // async getNewAppFolder({ name }) { + // if (!(await fs.exists(path.join(appdir(), name)))) return name; + // let index = 2; + // while (await fs.exists(path.join(appdir(), `${name}${index}`))) { + // index += 1; + // } + // return `${name}${index}`; + // }, + + // getUsedAppFolders_meta: true, + // async getUsedAppFolders() { + // const list = await connections.list(); + // const apps = []; + + // for (const connection of list) { + // for (const db of connection.databases || []) { + // for (const key of _.keys(db || {})) { + // if (key.startsWith('useApp:') && db[key]) { + // apps.push(key.substring('useApp:'.length)); + // } + // } + // } + // } + + // return _.intersection(_.uniq(apps), await fs.readdir(appdir())); + // }, + + // // getAppsForDb_meta: true, + // // async getAppsForDb({ conid, database }) { + // // const connection = await connections.get({ conid }); + // // if (!connection) return []; + // // const db = (connection.databases || []).find(x => x.name == database); + // // const apps = []; + // // const res = []; + // // if (db) { + // // for (const key of _.keys(db || {})) { + // // if (key.startsWith('useApp:') && db[key]) { + // // apps.push(key.substring('useApp:'.length)); + // // } + // // } + // // } + // // for (const folder of apps) { + // // res.push(await this.loadApp({ folder })); + // // } + // // return res; + // // }, + + // loadApp_meta: true, + // async loadApp({ folder }) { + // const res = { + // queries: [], + // commands: [], + // name: folder, + // }; + // const dir = path.join(appdir(), folder); + // if (await fs.exists(dir)) { + // const files = await fs.readdir(dir); + + // async function processType(ext, field) { + // for (const file of files) { + // if (file.endsWith(ext)) { + // res[field].push({ + // name: file.slice(0, -ext.length), + // sql: await fs.readFile(path.join(dir, file), { encoding: 'utf-8' }), + // }); + // } + // } + // } + + // await processType('.command.sql', 'commands'); + // await processType('.query.sql', 'queries'); + // } + + // try { + // res.virtualReferences = JSON.parse( + // await fs.readFile(path.join(dir, 'virtual-references.config.json'), { encoding: 'utf-8' }) + // ); + // } catch (err) { + // res.virtualReferences = []; + // } + // try { + // res.dictionaryDescriptions = JSON.parse( + // await fs.readFile(path.join(dir, 'dictionary-descriptions.config.json'), { encoding: 'utf-8' }) + // ); + // } catch (err) { + // res.dictionaryDescriptions = []; + // } + + // return res; + // }, + + // async saveConfigFile(appFolder, filename, filterFunc, newItem) { + // const file = path.join(appdir(), appFolder, filename); + + // let json; + // try { + // json = JSON.parse(await fs.readFile(file, { encoding: 'utf-8' })); + // } catch (err) { + // json = []; + // } + + // if (filterFunc) { + // json = json.filter(filterFunc); + // } + + // json = [...json, newItem]; + + // await fs.writeFile(file, JSON.stringify(json, undefined, 2)); + + // socket.emitChanged('app-files-changed', { app: appFolder }); + // socket.emitChanged('used-apps-changed'); + // }, + + // saveDictionaryDescription_meta: true, + // async saveDictionaryDescription({ appFolder, pureName, schemaName, expression, columns, delimiter }) { + // await this.saveConfigFile( + // appFolder, + // 'dictionary-descriptions.config.json', + // x => !(x.schemaName == schemaName && x.pureName == pureName), + // { + // schemaName, + // pureName, + // expression, + // columns, + // delimiter, + // } + // ); + + // return true; + // }, + + // createConfigFile_meta: true, + // async createConfigFile({ appFolder, fileName, content }) { + // const file = path.join(appdir(), appFolder, fileName); + // if (!(await fs.exists(file))) { + // await fs.writeFile(file, JSON.stringify(content, undefined, 2)); + // socket.emitChanged('app-files-changed', { app: appFolder }); + // socket.emitChanged('used-apps-changed'); + // return true; + // } + // return false; + // }, }; diff --git a/packages/api/src/controllers/auth.js b/packages/api/src/controllers/auth.js index ae6e144b3..bd5053dda 100644 --- a/packages/api/src/controllers/auth.js +++ b/packages/api/src/controllers/auth.js @@ -51,6 +51,7 @@ function authMiddleware(req, res, next) { '/auth/oauth-token', '/auth/login', '/auth/redirect', + '/redirect', '/stream', '/storage/get-connections-for-login-page', '/storage/set-admin-password', @@ -139,9 +140,9 @@ module.exports = { const accessToken = jwt.sign( { login: 'superadmin', - permissions: await storage.loadSuperadminPermissions(), roleId: -3, licenseUid, + amoid: 'superadmin', }, getTokenSecret(), { @@ -173,7 +174,9 @@ module.exports = { getProviders_meta: true, getProviders() { return { - providers: getAuthProviders().map(x => x.toJson()), + providers: getAuthProviders() + .filter(x => !x.skipInList) + .map(x => x.toJson()), default: getDefaultAuthProvider()?.amoid, }; }, diff --git a/packages/api/src/controllers/cloud.js b/packages/api/src/controllers/cloud.js index d1cbc00dd..38228272e 100644 --- a/packages/api/src/controllers/cloud.js +++ b/packages/api/src/controllers/cloud.js @@ -8,6 +8,9 @@ const { getCloudContent, putCloudContent, removeCloudCachedConnection, + getPromoWidgetData, + getPromoWidgetList, + getPromoWidgetPreview, } = require('../utility/cloudIntf'); const connections = require('./connections'); const socket = require('../utility/socket'); @@ -32,8 +35,8 @@ module.exports = { }, refreshPublicFiles_meta: true, - async refreshPublicFiles({ isRefresh }) { - await refreshPublicFiles(isRefresh); + async refreshPublicFiles({ isRefresh }, req) { + await refreshPublicFiles(isRefresh, req?.headers?.['x-ui-language']); return { status: 'ok', }; @@ -283,6 +286,28 @@ module.exports = { return getAiGatewayServer(); }, + premiumPromoWidget_meta: true, + async premiumPromoWidget() { + const data = await getPromoWidgetData(); + if (data?.state != 'data') { + return null; + } + if (data.validTo && new Date().getTime() > new Date(data.validTo).getTime()) { + return null; + } + return data; + }, + + promoWidgetList_meta: true, + async promoWidgetList() { + return getPromoWidgetList(); + }, + + promoWidgetPreview_meta: true, + async promoWidgetPreview({ campaign, variant }) { + return getPromoWidgetPreview(campaign, variant); + }, + // chatStream_meta: { // raw: true, // method: 'post', diff --git a/packages/api/src/controllers/config.js b/packages/api/src/controllers/config.js index 0706c320c..a7425ab72 100644 --- a/packages/api/src/controllers/config.js +++ b/packages/api/src/controllers/config.js @@ -3,7 +3,7 @@ const os = require('os'); const path = require('path'); const axios = require('axios'); const { datadir, getLogsFilePath } = require('../utility/directories'); -const { hasPermission } = require('../utility/hasPermission'); +const { hasPermission, loadPermissionsFromRequest } = require('../utility/hasPermission'); const socket = require('../utility/socket'); const _ = require('lodash'); const AsyncLock = require('async-lock'); @@ -46,7 +46,7 @@ module.exports = { async get(_params, req) { const authProvider = getAuthProviderFromReq(req); const login = authProvider.getCurrentLogin(req); - const permissions = authProvider.getCurrentPermissions(req); + const permissions = await authProvider.getCurrentPermissions(req); const isUserLoggedIn = authProvider.isUserLoggedIn(req); const singleConid = authProvider.getSingleConnectionId(req); @@ -71,6 +71,7 @@ module.exports = { const isLicenseValid = checkedLicense?.status == 'ok'; const logoutUrl = storageConnectionError ? null : await authProvider.getLogoutUrl(); const adminConfig = storageConnectionError ? null : await storage.readConfig({ group: 'admin' }); + const settingsConfig = storageConnectionError ? null : await storage.readConfig({ group: 'settings' }); storage.startRefreshLicense(); @@ -121,6 +122,7 @@ module.exports = { allowPrivateCloud: platformInfo.isElectron || !!process.env.ALLOW_DBGATE_PRIVATE_CLOUD, ...currentVersion, redirectToDbGateCloudLogin: !!process.env.REDIRECT_TO_DBGATE_CLOUD_LOGIN, + preferrendLanguage: settingsConfig?.['storage.language'] || process.env.LANGUAGE || null, }; return configResult; @@ -280,22 +282,18 @@ module.exports = { updateSettings_meta: true, async updateSettings(values, req) { - if (!hasPermission(`settings/change`, req)) return false; + const loadedPermissions = await loadPermissionsFromRequest(req); + if (!hasPermission(`settings/change`, loadedPermissions)) return false; cachedSettingsValue = null; const res = await lock.acquire('settings', async () => { const currentValue = await this.loadSettings(); try { - let updated = currentValue; + let updated = { + ...currentValue, + ...values, + }; if (process.env.STORAGE_DATABASE) { - updated = { - ...currentValue, - ..._.mapValues(values, v => { - if (v === true) return 'true'; - if (v === false) return 'false'; - return v; - }), - }; await storage.writeConfig({ group: 'settings', config: updated, @@ -392,7 +390,8 @@ module.exports = { exportConnectionsAndSettings_meta: true, async exportConnectionsAndSettings(_params, req) { - if (!hasPermission(`admin/config`, req)) { + const loadedPermissions = await loadPermissionsFromRequest(req); + if (!hasPermission(`admin/config`, loadedPermissions)) { throw new Error('Permission denied: admin/config'); } @@ -416,7 +415,8 @@ module.exports = { importConnectionsAndSettings_meta: true, async importConnectionsAndSettings({ db }, req) { - if (!hasPermission(`admin/config`, req)) { + const loadedPermissions = await loadPermissionsFromRequest(req); + if (!hasPermission(`admin/config`, loadedPermissions)) { throw new Error('Permission denied: admin/config'); } diff --git a/packages/api/src/controllers/connections.js b/packages/api/src/controllers/connections.js index f29f96e64..4260c4b11 100644 --- a/packages/api/src/controllers/connections.js +++ b/packages/api/src/controllers/connections.js @@ -14,11 +14,16 @@ const JsonLinesDatabase = require('../utility/JsonLinesDatabase'); const processArgs = require('../utility/processArgs'); const { safeJsonParse, getLogger, extractErrorLogData } = require('dbgate-tools'); const platformInfo = require('../utility/platformInfo'); -const { connectionHasPermission, testConnectionPermission } = require('../utility/hasPermission'); +const { + connectionHasPermission, + testConnectionPermission, + loadPermissionsFromRequest, +} = require('../utility/hasPermission'); const pipeForkLogs = require('../utility/pipeForkLogs'); const requireEngineDriver = require('../utility/requireEngineDriver'); const { getAuthProviderById } = require('../auth/authProvider'); const { startTokenChecking } = require('../utility/authProxy'); +const { extractConnectionsFromEnv } = require('../utility/envtools'); const logger = getLogger('connections'); @@ -57,55 +62,7 @@ function getDatabaseFileLabel(databaseFile) { function getPortalCollections() { if (process.env.CONNECTIONS) { - const connections = _.compact(process.env.CONNECTIONS.split(',')).map(id => ({ - _id: id, - engine: process.env[`ENGINE_${id}`], - server: process.env[`SERVER_${id}`], - user: process.env[`USER_${id}`], - password: process.env[`PASSWORD_${id}`], - passwordMode: process.env[`PASSWORD_MODE_${id}`], - port: process.env[`PORT_${id}`], - databaseUrl: process.env[`URL_${id}`], - useDatabaseUrl: !!process.env[`URL_${id}`], - databaseFile: process.env[`FILE_${id}`]?.replace( - '%%E2E_TEST_DATA_DIRECTORY%%', - path.join(path.dirname(path.dirname(__dirname)), 'e2e-tests', 'tmpdata') - ), - socketPath: process.env[`SOCKET_PATH_${id}`], - serviceName: process.env[`SERVICE_NAME_${id}`], - authType: process.env[`AUTH_TYPE_${id}`] || (process.env[`SOCKET_PATH_${id}`] ? 'socket' : undefined), - defaultDatabase: - process.env[`DATABASE_${id}`] || - (process.env[`FILE_${id}`] ? getDatabaseFileLabel(process.env[`FILE_${id}`]) : null), - singleDatabase: !!process.env[`DATABASE_${id}`] || !!process.env[`FILE_${id}`], - displayName: process.env[`LABEL_${id}`], - isReadOnly: process.env[`READONLY_${id}`], - databases: process.env[`DBCONFIG_${id}`] ? safeJsonParse(process.env[`DBCONFIG_${id}`]) : null, - allowedDatabases: process.env[`ALLOWED_DATABASES_${id}`]?.replace(/\|/g, '\n'), - allowedDatabasesRegex: process.env[`ALLOWED_DATABASES_REGEX_${id}`], - parent: process.env[`PARENT_${id}`] || undefined, - useSeparateSchemas: !!process.env[`USE_SEPARATE_SCHEMAS_${id}`], - localDataCenter: process.env[`LOCAL_DATA_CENTER_${id}`], - - // SSH tunnel - useSshTunnel: process.env[`USE_SSH_${id}`], - sshHost: process.env[`SSH_HOST_${id}`], - sshPort: process.env[`SSH_PORT_${id}`], - sshMode: process.env[`SSH_MODE_${id}`], - sshLogin: process.env[`SSH_LOGIN_${id}`], - sshPassword: process.env[`SSH_PASSWORD_${id}`], - sshKeyfile: process.env[`SSH_KEY_FILE_${id}`], - sshKeyfilePassword: process.env[`SSH_KEY_FILE_PASSWORD_${id}`], - - // SSL - useSsl: process.env[`USE_SSL_${id}`], - sslCaFile: process.env[`SSL_CA_FILE_${id}`], - sslCertFile: process.env[`SSL_CERT_FILE_${id}`], - sslCertFilePassword: process.env[`SSL_CERT_FILE_PASSWORD_${id}`], - sslKeyFile: process.env[`SSL_KEY_FILE_${id}`], - sslRejectUnauthorized: process.env[`SSL_REJECT_UNAUTHORIZED_${id}`], - trustServerCertificate: process.env[`SSL_TRUST_CERTIFICATE_${id}`], - })); + const connections = extractConnectionsFromEnv(process.env); for (const conn of connections) { for (const prop in process.env) { @@ -116,7 +73,10 @@ function getPortalCollections() { } } - logger.info({ connections: connections.map(pickSafeConnectionInfo) }, 'DBGM-00005 Using connections from ENV variables'); + logger.info( + { connections: connections.map(pickSafeConnectionInfo) }, + 'DBGM-00005 Using connections from ENV variables' + ); const noengine = connections.filter(x => !x.engine); if (noengine.length > 0) { logger.warn( @@ -222,11 +182,21 @@ module.exports = { ); } await this.checkUnsavedConnectionsLimit(); + + if (process.env.STORAGE_DATABASE && process.env.CONNECTIONS) { + const storage = require('./storage'); + try { + await storage.fillStorageConnectionsFromEnv(); + } catch (err) { + logger.error(extractErrorLogData(err), 'DBGM-00268 Error filling storage connections from env'); + } + } }, list_meta: true, async list(_params, req) { const storage = require('./storage'); + const loadedPermissions = await loadPermissionsFromRequest(req); const storageConnections = await storage.connections(req); if (storageConnections) { @@ -234,9 +204,9 @@ module.exports = { } if (portalConnections) { if (platformInfo.allowShellConnection) return portalConnections; - return portalConnections.map(maskConnection).filter(x => connectionHasPermission(x, req)); + return portalConnections.map(maskConnection).filter(x => connectionHasPermission(x, loadedPermissions)); } - return (await this.datastore.find()).filter(x => connectionHasPermission(x, req)); + return (await this.datastore.find()).filter(x => connectionHasPermission(x, loadedPermissions)); }, async getUsedEngines() { @@ -375,7 +345,7 @@ module.exports = { update_meta: true, async update({ _id, values }, req) { if (portalConnections) return; - testConnectionPermission(_id, req); + await testConnectionPermission(_id, req); const res = await this.datastore.patch(_id, values); socket.emitChanged('connection-list-changed'); return res; @@ -392,7 +362,7 @@ module.exports = { updateDatabase_meta: true, async updateDatabase({ conid, database, values }, req) { if (portalConnections) return; - testConnectionPermission(conid, req); + await testConnectionPermission(conid, req); const conn = await this.datastore.get(conid); let databases = (conn && conn.databases) || []; if (databases.find(x => x.name == database)) { @@ -410,7 +380,7 @@ module.exports = { delete_meta: true, async delete(connection, req) { if (portalConnections) return; - testConnectionPermission(connection, req); + await testConnectionPermission(connection, req); const res = await this.datastore.remove(connection._id); socket.emitChanged('connection-list-changed'); return res; @@ -452,7 +422,7 @@ module.exports = { _id: '__model', }; } - testConnectionPermission(conid, req); + await testConnectionPermission(conid, req); return this.getCore({ conid, mask: true }); }, @@ -501,7 +471,11 @@ module.exports = { state, client: 'web', }); - res.redirect(authResp.url); + if (authResp?.url) { + res.redirect(authResp.url); + return; + } + res.json({ error: 'No URL returned from auth provider' }); }, dbloginApp_meta: true, diff --git a/packages/api/src/controllers/databaseConnections.js b/packages/api/src/controllers/databaseConnections.js index 9a89a87ba..161ab1e32 100644 --- a/packages/api/src/controllers/databaseConnections.js +++ b/packages/api/src/controllers/databaseConnections.js @@ -29,7 +29,17 @@ const generateDeploySql = require('../shell/generateDeploySql'); const { createTwoFilesPatch } = require('diff'); const diff2htmlPage = require('../utility/diff2htmlPage'); const processArgs = require('../utility/processArgs'); -const { testConnectionPermission } = require('../utility/hasPermission'); +const { + testConnectionPermission, + hasPermission, + loadPermissionsFromRequest, + loadTablePermissionsFromRequest, + getTablePermissionRole, + loadDatabasePermissionsFromRequest, + getDatabasePermissionRole, + getTablePermissionRoleLevelIndex, + testDatabaseRolePermission, +} = require('../utility/hasPermission'); const { MissingCredentialsError } = require('../utility/exceptions'); const pipeForkLogs = require('../utility/pipeForkLogs'); const crypto = require('crypto'); @@ -235,7 +245,7 @@ module.exports = { queryData_meta: true, async queryData({ conid, database, sql }, req) { - testConnectionPermission(conid, req); + await testConnectionPermission(conid, req); logger.info({ conid, database, sql }, 'DBGM-00007 Processing query'); const opened = await this.ensureOpened(conid, database); // if (opened && opened.status && opened.status.name == 'error') { @@ -247,7 +257,7 @@ module.exports = { sqlSelect_meta: true, async sqlSelect({ conid, database, select, auditLogSessionGroup }, req) { - testConnectionPermission(conid, req); + await testConnectionPermission(conid, req); const opened = await this.ensureOpened(conid, database); const res = await this.sendRequest( opened, @@ -282,7 +292,9 @@ module.exports = { runScript_meta: true, async runScript({ conid, database, sql, useTransaction, logMessage }, req) { - testConnectionPermission(conid, req); + const loadedPermissions = await loadPermissionsFromRequest(req); + await testConnectionPermission(conid, req, loadedPermissions); + await testDatabaseRolePermission(conid, database, 'run_script', req); logger.info({ conid, database, sql }, 'DBGM-00008 Processing script'); const opened = await this.ensureOpened(conid, database); sendToAuditLog(req, { @@ -303,7 +315,7 @@ module.exports = { runOperation_meta: true, async runOperation({ conid, database, operation, useTransaction }, req) { - testConnectionPermission(conid, req); + await testConnectionPermission(conid, req); logger.info({ conid, database, operation }, 'DBGM-00009 Processing operation'); sendToAuditLog(req, { @@ -325,7 +337,7 @@ module.exports = { collectionData_meta: true, async collectionData({ conid, database, options, auditLogSessionGroup }, req) { - testConnectionPermission(conid, req); + await testConnectionPermission(conid, req); const opened = await this.ensureOpened(conid, database); const res = await this.sendRequest( opened, @@ -356,7 +368,7 @@ module.exports = { }, async loadDataCore(msgtype, { conid, database, ...args }, req) { - testConnectionPermission(conid, req); + await testConnectionPermission(conid, req); const opened = await this.ensureOpened(conid, database); const res = await this.sendRequest(opened, { msgtype, ...args }); if (res.errorMessage) { @@ -371,7 +383,7 @@ module.exports = { schemaList_meta: true, async schemaList({ conid, database }, req) { - testConnectionPermission(conid, req); + await testConnectionPermission(conid, req); return this.loadDataCore('schemaList', { conid, database }); }, @@ -383,43 +395,43 @@ module.exports = { loadKeys_meta: true, async loadKeys({ conid, database, root, filter, limit }, req) { - testConnectionPermission(conid, req); + await testConnectionPermission(conid, req); return this.loadDataCore('loadKeys', { conid, database, root, filter, limit }); }, scanKeys_meta: true, async scanKeys({ conid, database, root, pattern, cursor, count }, req) { - testConnectionPermission(conid, req); + await testConnectionPermission(conid, req); return this.loadDataCore('scanKeys', { conid, database, root, pattern, cursor, count }); }, exportKeys_meta: true, async exportKeys({ conid, database, options }, req) { - testConnectionPermission(conid, req); + await testConnectionPermission(conid, req); return this.loadDataCore('exportKeys', { conid, database, options }); }, loadKeyInfo_meta: true, async loadKeyInfo({ conid, database, key }, req) { - testConnectionPermission(conid, req); + await testConnectionPermission(conid, req); return this.loadDataCore('loadKeyInfo', { conid, database, key }); }, loadKeyTableRange_meta: true, async loadKeyTableRange({ conid, database, key, cursor, count }, req) { - testConnectionPermission(conid, req); + await testConnectionPermission(conid, req); return this.loadDataCore('loadKeyTableRange', { conid, database, key, cursor, count }); }, loadFieldValues_meta: true, async loadFieldValues({ conid, database, schemaName, pureName, field, search, dataType }, req) { - testConnectionPermission(conid, req); + await testConnectionPermission(conid, req); return this.loadDataCore('loadFieldValues', { conid, database, schemaName, pureName, field, search, dataType }); }, callMethod_meta: true, async callMethod({ conid, database, method, args }, req) { - testConnectionPermission(conid, req); + await testConnectionPermission(conid, req); return this.loadDataCore('callMethod', { conid, database, method, args }); // const opened = await this.ensureOpened(conid, database); @@ -432,7 +444,8 @@ module.exports = { updateCollection_meta: true, async updateCollection({ conid, database, changeSet }, req) { - testConnectionPermission(conid, req); + await testConnectionPermission(conid, req); + const opened = await this.ensureOpened(conid, database); const res = await this.sendRequest(opened, { msgtype: 'updateCollection', changeSet }); if (res.errorMessage) { @@ -443,6 +456,44 @@ module.exports = { return res.result || null; }, + saveTableData_meta: true, + async saveTableData({ conid, database, changeSet }, req) { + await testConnectionPermission(conid, req); + + const databasePermissions = await loadDatabasePermissionsFromRequest(req); + const tablePermissions = await loadTablePermissionsFromRequest(req); + const fieldsAndRoles = [ + [changeSet.inserts, 'create_update_delete'], + [changeSet.deletes, 'create_update_delete'], + [changeSet.updates, 'update_only'], + ]; + for (const [operations, requiredRole] of fieldsAndRoles) { + for (const operation of operations) { + const role = getTablePermissionRole( + conid, + database, + 'tables', + operation.schemaName, + operation.pureName, + tablePermissions, + databasePermissions + ); + if (getTablePermissionRoleLevelIndex(role) < getTablePermissionRoleLevelIndex(requiredRole)) { + throw new Error('DBGM-00262 Permission not granted'); + } + } + } + + const opened = await this.ensureOpened(conid, database); + const res = await this.sendRequest(opened, { msgtype: 'saveTableData', changeSet }); + if (res.errorMessage) { + return { + errorMessage: res.errorMessage, + }; + } + return res.result || null; + }, + status_meta: true, async status({ conid, database }, req) { if (!conid) { @@ -451,7 +502,7 @@ module.exports = { message: 'No connection', }; } - testConnectionPermission(conid, req); + await testConnectionPermission(conid, req); const existing = this.opened.find(x => x.conid == conid && x.database == database); if (existing) { return { @@ -474,7 +525,7 @@ module.exports = { ping_meta: true, async ping({ conid, database }, req) { - testConnectionPermission(conid, req); + await testConnectionPermission(conid, req); let existing = this.opened.find(x => x.conid == conid && x.database == database); if (existing) { @@ -502,7 +553,7 @@ module.exports = { refresh_meta: true, async refresh({ conid, database, keepOpen }, req) { - testConnectionPermission(conid, req); + await testConnectionPermission(conid, req); if (!keepOpen) this.close(conid, database); await this.ensureOpened(conid, database); @@ -516,7 +567,7 @@ module.exports = { return { status: 'ok' }; } - testConnectionPermission(conid, req); + await testConnectionPermission(conid, req); const conn = await this.ensureOpened(conid, database); conn.subprocess.send({ msgtype: 'syncModel', isFullRefresh }); return { status: 'ok' }; @@ -553,7 +604,7 @@ module.exports = { disconnect_meta: true, async disconnect({ conid, database }, req) { - testConnectionPermission(conid, req); + await testConnectionPermission(conid, req); await this.close(conid, database, true); return { status: 'ok' }; }, @@ -563,8 +614,9 @@ module.exports = { if (!conid || !database) { return {}; } + const loadedPermissions = await loadPermissionsFromRequest(req); - testConnectionPermission(conid, req); + await testConnectionPermission(conid, req, loadedPermissions); if (conid == '__model') { const model = await importDbModel(database); const trans = await loadModelTransform(modelTransFile); @@ -586,6 +638,46 @@ module.exports = { message: `Loaded database structure for ${database}`, }); + if (process.env.STORAGE_DATABASE && !hasPermission(`all-tables`, loadedPermissions)) { + // filter databases by permissions + const tablePermissions = await loadTablePermissionsFromRequest(req); + const databasePermissions = await loadDatabasePermissionsFromRequest(req); + const databasePermissionRole = getDatabasePermissionRole(conid, database, databasePermissions); + + function applyTablePermissionRole(list, objectTypeField) { + const res = []; + for (const item of list ?? []) { + const tablePermissionRole = getTablePermissionRole( + conid, + database, + objectTypeField, + item.schemaName, + item.pureName, + tablePermissions, + databasePermissionRole + ); + if (tablePermissionRole != 'deny') { + res.push({ + ...item, + tablePermissionRole, + }); + } + } + return res; + } + + const res = { + ...opened.structure, + tables: applyTablePermissionRole(opened.structure.tables, 'tables'), + views: applyTablePermissionRole(opened.structure.views, 'views'), + procedures: applyTablePermissionRole(opened.structure.procedures, 'procedures'), + functions: applyTablePermissionRole(opened.structure.functions, 'functions'), + triggers: applyTablePermissionRole(opened.structure.triggers, 'triggers'), + collections: applyTablePermissionRole(opened.structure.collections, 'collections'), + }; + return res; + } + return opened.structure; // const existing = this.opened.find((x) => x.conid == conid && x.database == database); // if (existing) return existing.status; @@ -600,7 +692,7 @@ module.exports = { if (!conid) { return null; } - testConnectionPermission(conid, req); + await testConnectionPermission(conid, req); if (!conid) return null; const opened = await this.ensureOpened(conid, database); return opened.serverVersion || null; @@ -608,7 +700,7 @@ module.exports = { sqlPreview_meta: true, async sqlPreview({ conid, database, objects, options }, req) { - testConnectionPermission(conid, req); + await testConnectionPermission(conid, req); // wait for structure await this.structure({ conid, database }); @@ -619,7 +711,7 @@ module.exports = { exportModel_meta: true, async exportModel({ conid, database, outputFolder, schema }, req) { - testConnectionPermission(conid, req); + await testConnectionPermission(conid, req); const realFolder = outputFolder.startsWith('archive:') ? resolveArchiveFolder(outputFolder.substring('archive:'.length)) @@ -637,7 +729,7 @@ module.exports = { exportModelSql_meta: true, async exportModelSql({ conid, database, outputFolder, outputFile, schema }, req) { - testConnectionPermission(conid, req); + await testConnectionPermission(conid, req); const connection = await connections.getCore({ conid }); const driver = requireEngineDriver(connection); @@ -651,7 +743,7 @@ module.exports = { generateDeploySql_meta: true, async generateDeploySql({ conid, database, archiveFolder }, req) { - testConnectionPermission(conid, req); + await testConnectionPermission(conid, req); const opened = await this.ensureOpened(conid, database); const res = await this.sendRequest(opened, { msgtype: 'generateDeploySql', @@ -923,9 +1015,12 @@ module.exports = { executeSessionQuery_meta: true, async executeSessionQuery({ sesid, conid, database, sql }, req) { - testConnectionPermission(conid, req); + await testConnectionPermission(conid, req); logger.info({ sesid, sql }, 'DBGM-00010 Processing query'); - sessions.dispatchMessage(sesid, 'Query execution started'); + sessions.dispatchMessage(sesid, { + message: 'Query execution started', + sql, + }); const opened = await this.ensureOpened(conid, database); opened.subprocess.send({ msgtype: 'executeSessionQuery', sql, sesid }); @@ -935,7 +1030,7 @@ module.exports = { evalJsonScript_meta: true, async evalJsonScript({ conid, database, script, runid }, req) { - testConnectionPermission(conid, req); + await testConnectionPermission(conid, req); const opened = await this.ensureOpened(conid, database); opened.subprocess.send({ msgtype: 'evalJsonScript', script, runid }); diff --git a/packages/api/src/controllers/files.js b/packages/api/src/controllers/files.js index 25003e941..57c1b68c2 100644 --- a/packages/api/src/controllers/files.js +++ b/packages/api/src/controllers/files.js @@ -3,7 +3,12 @@ const path = require('path'); const crypto = require('crypto'); const { filesdir, archivedir, resolveArchiveFolder, uploadsdir, appdir, jsldir } = require('../utility/directories'); const getChartExport = require('../utility/getChartExport'); -const { hasPermission } = require('../utility/hasPermission'); +const { + hasPermission, + loadPermissionsFromRequest, + loadFilePermissionsFromRequest, + getFilePermissionRole, +} = require('../utility/hasPermission'); const socket = require('../utility/socket'); const scheduler = require('./scheduler'); const getDiagramExport = require('../utility/getDiagramExport'); @@ -31,7 +36,8 @@ function deserialize(format, text) { module.exports = { list_meta: true, async list({ folder }, req) { - if (!hasPermission(`files/${folder}/read`, req)) return []; + const loadedPermissions = await loadPermissionsFromRequest(req); + if (!hasPermission(`files/${folder}/read`, loadedPermissions)) return []; const dir = path.join(filesdir(), folder); if (!(await fs.exists(dir))) return []; const files = (await fs.readdir(dir)).map(file => ({ folder, file })); @@ -40,10 +46,11 @@ module.exports = { listAll_meta: true, async listAll(_params, req) { + const loadedPermissions = await loadPermissionsFromRequest(req); const folders = await fs.readdir(filesdir()); const res = []; for (const folder of folders) { - if (!hasPermission(`files/${folder}/read`, req)) continue; + if (!hasPermission(`files/${folder}/read`, loadedPermissions)) continue; const dir = path.join(filesdir(), folder); const files = (await fs.readdir(dir)).map(file => ({ folder, file })); res.push(...files); @@ -53,7 +60,8 @@ module.exports = { delete_meta: true, async delete({ folder, file }, req) { - if (!hasPermission(`files/${folder}/write`, req)) return false; + const loadedPermissions = await loadPermissionsFromRequest(req); + if (!hasPermission(`files/${folder}/write`, loadedPermissions)) return false; if (!checkSecureFilePathsWithoutDirectory(folder, file)) { return false; } @@ -65,7 +73,8 @@ module.exports = { rename_meta: true, async rename({ folder, file, newFile }, req) { - if (!hasPermission(`files/${folder}/write`, req)) return false; + const loadedPermissions = await loadPermissionsFromRequest(req); + if (!hasPermission(`files/${folder}/write`, loadedPermissions)) return false; if (!checkSecureFilePathsWithoutDirectory(folder, file, newFile)) { return false; } @@ -86,10 +95,11 @@ module.exports = { copy_meta: true, async copy({ folder, file, newFile }, req) { + const loadedPermissions = await loadPermissionsFromRequest(req); if (!checkSecureFilePathsWithoutDirectory(folder, file, newFile)) { return false; } - if (!hasPermission(`files/${folder}/write`, req)) return false; + if (!hasPermission(`files/${folder}/write`, loadedPermissions)) return false; await fs.copyFile(path.join(filesdir(), folder, file), path.join(filesdir(), folder, newFile)); socket.emitChanged(`files-changed`, { folder }); socket.emitChanged(`all-files-changed`); @@ -113,7 +123,8 @@ module.exports = { }); return deserialize(format, text); } else { - if (!hasPermission(`files/${folder}/read`, req)) return null; + const loadedPermissions = await loadPermissionsFromRequest(req); + if (!hasPermission(`files/${folder}/read`, loadedPermissions)) return null; const text = await fs.readFile(path.join(filesdir(), folder, file), { encoding: 'utf-8' }); return deserialize(format, text); } @@ -131,18 +142,19 @@ module.exports = { save_meta: true, async save({ folder, file, data, format }, req) { + const loadedPermissions = await loadPermissionsFromRequest(req); if (!checkSecureFilePathsWithoutDirectory(folder, file)) { return false; } if (folder.startsWith('archive:')) { - if (!hasPermission(`archive/write`, req)) return false; + if (!hasPermission(`archive/write`, loadedPermissions)) return false; const dir = resolveArchiveFolder(folder.substring('archive:'.length)); await fs.writeFile(path.join(dir, file), serialize(format, data)); socket.emitChanged(`archive-files-changed`, { folder: folder.substring('archive:'.length) }); return true; } else if (folder.startsWith('app:')) { - if (!hasPermission(`apps/write`, req)) return false; + if (!hasPermission(`apps/write`, loadedPermissions)) return false; const app = folder.substring('app:'.length); await fs.writeFile(path.join(appdir(), app, file), serialize(format, data)); socket.emitChanged(`app-files-changed`, { app }); @@ -150,7 +162,7 @@ module.exports = { apps.emitChangedDbApp(folder); return true; } else { - if (!hasPermission(`files/${folder}/write`, req)) return false; + if (!hasPermission(`files/${folder}/write`, loadedPermissions)) return false; const dir = path.join(filesdir(), folder); if (!(await fs.exists(dir))) { await fs.mkdir(dir); @@ -177,7 +189,8 @@ module.exports = { favorites_meta: true, async favorites(_params, req) { - if (!hasPermission(`files/favorites/read`, req)) return []; + const loadedPermissions = await loadPermissionsFromRequest(req); + if (!hasPermission(`files/favorites/read`, loadedPermissions)) return []; const dir = path.join(filesdir(), 'favorites'); if (!(await fs.exists(dir))) return []; const files = await fs.readdir(dir); @@ -234,16 +247,17 @@ module.exports = { getFileRealPath_meta: true, async getFileRealPath({ folder, file }, req) { + const loadedPermissions = await loadPermissionsFromRequest(req); if (folder.startsWith('archive:')) { - if (!hasPermission(`archive/write`, req)) return false; + if (!hasPermission(`archive/write`, loadedPermissions)) return false; const dir = resolveArchiveFolder(folder.substring('archive:'.length)); return path.join(dir, file); } else if (folder.startsWith('app:')) { - if (!hasPermission(`apps/write`, req)) return false; + if (!hasPermission(`apps/write`, loadedPermissions)) return false; const app = folder.substring('app:'.length); return path.join(appdir(), app, file); } else { - if (!hasPermission(`files/${folder}/write`, req)) return false; + if (!hasPermission(`files/${folder}/write`, loadedPermissions)) return false; const dir = path.join(filesdir(), folder); if (!(await fs.exists(dir))) { await fs.mkdir(dir); @@ -297,7 +311,8 @@ module.exports = { exportFile_meta: true, async exportFile({ folder, file, filePath }, req) { - if (!hasPermission(`files/${folder}/read`, req)) return false; + const loadedPermissions = await loadPermissionsFromRequest(req); + if (!hasPermission(`files/${folder}/read`, loadedPermissions)) return false; await fs.copyFile(path.join(filesdir(), folder, file), filePath); return true; }, diff --git a/packages/api/src/controllers/plugins.js b/packages/api/src/controllers/plugins.js index 2b74a4669..adf32ebf5 100644 --- a/packages/api/src/controllers/plugins.js +++ b/packages/api/src/controllers/plugins.js @@ -7,7 +7,7 @@ const socket = require('../utility/socket'); const compareVersions = require('compare-versions'); const requirePlugin = require('../shell/requirePlugin'); const downloadPackage = require('../utility/downloadPackage'); -const { hasPermission } = require('../utility/hasPermission'); +const { hasPermission, loadPermissionsFromRequest } = require('../utility/hasPermission'); const _ = require('lodash'); const packagedPluginsContent = require('../packagedPluginsContent'); @@ -118,7 +118,8 @@ module.exports = { install_meta: true, async install({ packageName }, req) { - if (!hasPermission(`plugins/install`, req)) return; + const loadedPermissions = await loadPermissionsFromRequest(req); + if (!hasPermission(`plugins/install`, loadedPermissions)) return; const dir = path.join(pluginsdir(), packageName); // @ts-ignore if (!(await fs.exists(dir))) { @@ -132,7 +133,8 @@ module.exports = { uninstall_meta: true, async uninstall({ packageName }, req) { - if (!hasPermission(`plugins/install`, req)) return; + const loadedPermissions = await loadPermissionsFromRequest(req); + if (!hasPermission(`plugins/install`, loadedPermissions)) return; const dir = path.join(pluginsdir(), packageName); await fs.rmdir(dir, { recursive: true }); socket.emitChanged(`installed-plugins-changed`); @@ -143,7 +145,8 @@ module.exports = { upgrade_meta: true, async upgrade({ packageName }, req) { - if (!hasPermission(`plugins/install`, req)) return; + const loadedPermissions = await loadPermissionsFromRequest(req); + if (!hasPermission(`plugins/install`, loadedPermissions)) return; const dir = path.join(pluginsdir(), packageName); // @ts-ignore if (await fs.exists(dir)) { diff --git a/packages/api/src/controllers/runners.js b/packages/api/src/controllers/runners.js index 123280194..d11482f47 100644 --- a/packages/api/src/controllers/runners.js +++ b/packages/api/src/controllers/runners.js @@ -21,6 +21,7 @@ const processArgs = require('../utility/processArgs'); const platformInfo = require('../utility/platformInfo'); const { checkSecureDirectories, checkSecureDirectoriesInScript } = require('../utility/security'); const { sendToAuditLog, logJsonRunnerScript } = require('../utility/auditlog'); +const { testStandardPermission } = require('../utility/hasPermission'); const logger = getLogger('runners'); function extractPlugins(script) { @@ -288,6 +289,8 @@ module.exports = { return this.startCore(runid, scriptTemplate(js, false)); } + await testStandardPermission('run-shell-script', req); + if (!platformInfo.allowShellScripting) { sendToAuditLog(req, { category: 'shell', diff --git a/packages/api/src/controllers/scheduler.js b/packages/api/src/controllers/scheduler.js index bd8138c99..a8e1bab3c 100644 --- a/packages/api/src/controllers/scheduler.js +++ b/packages/api/src/controllers/scheduler.js @@ -3,7 +3,7 @@ const fs = require('fs-extra'); const path = require('path'); const cron = require('node-cron'); const runners = require('./runners'); -const { hasPermission } = require('../utility/hasPermission'); +const { hasPermission, loadPermissionsFromRequest } = require('../utility/hasPermission'); const { getLogger } = require('dbgate-tools'); const logger = getLogger('scheduler'); @@ -30,7 +30,8 @@ module.exports = { }, async reload(_params, req) { - if (!hasPermission('files/shell/read', req)) return; + const loadedPermissions = await loadPermissionsFromRequest(req); + if (!hasPermission('files/shell/read', loadedPermissions)) return; const shellDir = path.join(filesdir(), 'shell'); await this.unload(); if (!(await fs.exists(shellDir))) return; diff --git a/packages/api/src/controllers/serverConnections.js b/packages/api/src/controllers/serverConnections.js index 86e461feb..d561f3865 100644 --- a/packages/api/src/controllers/serverConnections.js +++ b/packages/api/src/controllers/serverConnections.js @@ -8,7 +8,13 @@ const { handleProcessCommunication } = require('../utility/processComm'); const lock = new AsyncLock(); const config = require('./config'); const processArgs = require('../utility/processArgs'); -const { testConnectionPermission } = require('../utility/hasPermission'); +const { + testConnectionPermission, + loadPermissionsFromRequest, + hasPermission, + loadDatabasePermissionsFromRequest, + getDatabasePermissionRole, +} = require('../utility/hasPermission'); const { MissingCredentialsError } = require('../utility/exceptions'); const pipeForkLogs = require('../utility/pipeForkLogs'); const { getLogger, extractErrorLogData } = require('dbgate-tools'); @@ -135,7 +141,7 @@ module.exports = { disconnect_meta: true, async disconnect({ conid }, req) { - testConnectionPermission(conid, req); + await testConnectionPermission(conid, req); await this.close(conid, true); return { status: 'ok' }; }, @@ -144,7 +150,9 @@ module.exports = { async listDatabases({ conid }, req) { if (!conid) return []; if (conid == '__model') return []; - testConnectionPermission(conid, req); + const loadedPermissions = await loadPermissionsFromRequest(req); + + await testConnectionPermission(conid, req, loadedPermissions); const opened = await this.ensureOpened(conid); sendToAuditLog(req, { category: 'serverop', @@ -157,12 +165,29 @@ module.exports = { sessionGroup: 'listDatabases', message: `Loaded databases for connection`, }); + + if (process.env.STORAGE_DATABASE && !hasPermission(`all-databases`, loadedPermissions)) { + // filter databases by permissions + const databasePermissions = await loadDatabasePermissionsFromRequest(req); + const res = []; + for (const db of opened?.databases ?? []) { + const databasePermissionRole = getDatabasePermissionRole(db.id, db.name, databasePermissions); + if (databasePermissionRole != 'deny') { + res.push({ + ...db, + databasePermissionRole, + }); + } + } + return res; + } + return opened?.databases ?? []; }, version_meta: true, async version({ conid }, req) { - testConnectionPermission(conid, req); + await testConnectionPermission(conid, req); const opened = await this.ensureOpened(conid); return opened?.version ?? null; }, @@ -184,11 +209,11 @@ module.exports = { return Promise.resolve(); } this.lastPinged[conid] = new Date().getTime(); - const opened = await this.ensureOpened(conid); - if (!opened) { - return Promise.resolve(); - } try { + const opened = await this.ensureOpened(conid); + if (!opened) { + return Promise.resolve(); + } opened.subprocess.send({ msgtype: 'ping' }); } catch (err) { logger.error(extractErrorLogData(err), 'DBGM-00121 Error pinging server connection'); @@ -202,7 +227,7 @@ module.exports = { refresh_meta: true, async refresh({ conid, keepOpen }, req) { - testConnectionPermission(conid, req); + await testConnectionPermission(conid, req); if (!keepOpen) this.close(conid); await this.ensureOpened(conid); @@ -210,7 +235,7 @@ module.exports = { }, async sendDatabaseOp({ conid, msgtype, name }, req) { - testConnectionPermission(conid, req); + await testConnectionPermission(conid, req); const opened = await this.ensureOpened(conid); if (!opened) { return null; @@ -252,7 +277,7 @@ module.exports = { }, async loadDataCore(msgtype, { conid, ...args }, req) { - testConnectionPermission(conid, req); + await testConnectionPermission(conid, req); const opened = await this.ensureOpened(conid); if (!opened) { return null; @@ -270,13 +295,43 @@ module.exports = { serverSummary_meta: true, async serverSummary({ conid }, req) { - testConnectionPermission(conid, req); + await testConnectionPermission(conid, req); + logger.info({ conid }, 'DBGM-00260 Processing server summary'); return this.loadDataCore('serverSummary', { conid }); }, + listDatabaseProcesses_meta: true, + async listDatabaseProcesses(ctx, req) { + const { conid } = ctx; + // logger.info({ conid }, 'DBGM-00261 Listing processes of database server'); + testConnectionPermission(conid, req); + + const opened = await this.ensureOpened(conid); + if (!opened) { + return null; + } + if (opened.connection.isReadOnly) return false; + + return this.sendRequest(opened, { msgtype: 'listDatabaseProcesses' }); + }, + + killDatabaseProcess_meta: true, + async killDatabaseProcess(ctx, req) { + const { conid, pid } = ctx; + testConnectionPermission(conid, req); + + const opened = await this.ensureOpened(conid); + if (!opened) { + return null; + } + if (opened.connection.isReadOnly) return false; + + return this.sendRequest(opened, { msgtype: 'killDatabaseProcess', pid }); + }, + summaryCommand_meta: true, async summaryCommand({ conid, command, row }, req) { - testConnectionPermission(conid, req); + await testConnectionPermission(conid, req); const opened = await this.ensureOpened(conid); if (!opened) { return null; diff --git a/packages/api/src/controllers/sessions.js b/packages/api/src/controllers/sessions.js index 3a3cd9b2b..20eba95c8 100644 --- a/packages/api/src/controllers/sessions.js +++ b/packages/api/src/controllers/sessions.js @@ -8,10 +8,13 @@ const path = require('path'); const { handleProcessCommunication } = require('../utility/processComm'); const processArgs = require('../utility/processArgs'); const { appdir } = require('../utility/directories'); -const { getLogger, extractErrorLogData } = require('dbgate-tools'); +const { getLogger, extractErrorLogData, removeSqlFrontMatter } = require('dbgate-tools'); const pipeForkLogs = require('../utility/pipeForkLogs'); const config = require('./config'); const { sendToAuditLog } = require('../utility/auditlog'); +const { testStandardPermission, testDatabaseRolePermission } = require('../utility/hasPermission'); +const { getStaticTokenSecret } = require('../auth/authCommon'); +const jwt = require('jsonwebtoken'); const logger = getLogger('sessions'); @@ -80,6 +83,16 @@ module.exports = { socket.emit(`session-recordset-${sesid}`, { jslid, resultIndex }); }, + handle_endrecordset(sesid, props) { + const { jslid, rowCount, durationMs } = props; + this.dispatchMessage(sesid, { + message: `Query returned ${rowCount} rows in ${durationMs} ms`, + rowCount, + durationMs, + jslid, + }); + }, + handle_stats(sesid, stats) { jsldata.notifyChangedStats(stats); }, @@ -94,6 +107,12 @@ module.exports = { socket.emit(`session-initialize-file-${jslid}`); }, + handle_changedCurrentDatabase(sesid, props) { + const { database } = props; + this.dispatchMessage(sesid, `Current database changed to ${database}`); + socket.emit(`session-changedb-${sesid}`, { database }); + }, + handle_ping() {}, create_meta: true, @@ -148,10 +167,23 @@ module.exports = { executeQuery_meta: true, async executeQuery({ sesid, sql, autoCommit, autoDetectCharts, limitRows, frontMatter }, req) { + let useTokenIsOk = false; + if (frontMatter?.useToken) { + const decoded = jwt.verify(frontMatter.useToken, getStaticTokenSecret()); + if (decoded?.['contentHash'] == crypto.createHash('md5').update(removeSqlFrontMatter(sql)).digest('hex')) { + useTokenIsOk = true; + } + } + if (!useTokenIsOk) { + await testStandardPermission('dbops/query', req); + } const session = this.opened.find(x => x.sesid == sesid); if (!session) { throw new Error('Invalid session'); } + if (!useTokenIsOk) { + await testDatabaseRolePermission(session.conid, session.database, 'run_script', req); + } sendToAuditLog(req, { category: 'dbop', @@ -166,7 +198,10 @@ module.exports = { }); logger.info({ sesid, sql }, 'DBGM-00019 Processing query'); - this.dispatchMessage(sesid, 'Query execution started'); + this.dispatchMessage(sesid, { + message: 'Query execution started', + sql, + }); session.subprocess.send({ msgtype: 'executeQuery', sql, diff --git a/packages/api/src/controllers/teamFiles.js b/packages/api/src/controllers/teamFiles.js new file mode 100644 index 000000000..dffe3b595 --- /dev/null +++ b/packages/api/src/controllers/teamFiles.js @@ -0,0 +1,6 @@ +module.exports = { + list_meta: true, + async list(req) { + return []; + }, +}; diff --git a/packages/api/src/controllers/uploads.js b/packages/api/src/controllers/uploads.js index b467d7810..fa9d17cfd 100644 --- a/packages/api/src/controllers/uploads.js +++ b/packages/api/src/controllers/uploads.js @@ -1,19 +1,8 @@ const crypto = require('crypto'); const path = require('path'); -const { uploadsdir, getLogsFilePath, filesdir } = require('../utility/directories'); -const { getLogger, extractErrorLogData } = require('dbgate-tools'); +const { uploadsdir } = require('../utility/directories'); +const { getLogger } = require('dbgate-tools'); const logger = getLogger('uploads'); -const axios = require('axios'); -const os = require('os'); -const fs = require('fs/promises'); -const { read } = require('./queryHistory'); -const platformInfo = require('../utility/platformInfo'); -const _ = require('lodash'); -const serverConnections = require('./serverConnections'); -const config = require('./config'); -const gistSecret = require('../gistSecret'); -const currentVersion = require('../currentVersion'); -const socket = require('../utility/socket'); module.exports = { upload_meta: { @@ -51,88 +40,70 @@ module.exports = { res.sendFile(path.join(uploadsdir(), req.query.file)); }, - async getGistToken() { - const settings = await config.getSettings(); +// uploadErrorToGist_meta: true, +// async uploadErrorToGist() { +// const logs = await fs.readFile(getLogsFilePath(), { encoding: 'utf-8' }); +// const connections = await serverConnections.getOpenedConnectionReport(); +// try { +// const response = await axios.default.post( +// 'https://api.github.com/gists', +// { +// description: `DbGate ${currentVersion.version} error report`, +// public: false, +// files: { +// 'logs.jsonl': { +// content: logs, +// }, +// 'os.json': { +// content: JSON.stringify( +// { +// release: os.release(), +// arch: os.arch(), +// machine: os.machine(), +// platform: os.platform(), +// type: os.type(), +// }, +// null, +// 2 +// ), +// }, +// 'platform.json': { +// content: JSON.stringify( +// _.omit( +// { +// ...platformInfo, +// }, +// ['defaultKeyfile', 'sshAuthSock'] +// ), +// null, +// 2 +// ), +// }, +// 'connections.json': { +// content: JSON.stringify(connections, null, 2), +// }, +// 'version.json': { +// content: JSON.stringify(currentVersion, null, 2), +// }, +// }, +// }, +// { +// headers: { +// Authorization: `token ${await this.getGistToken()}`, +// 'Content-Type': 'application/json', +// Accept: 'application/vnd.github.v3+json', +// }, +// } +// ); - return settings['other.gistCreateToken'] || gistSecret; - }, +// return response.data; +// } catch (err) { +// logger.error(extractErrorLogData(err), 'DBGM-00148 Error uploading gist'); - uploadErrorToGist_meta: true, - async uploadErrorToGist() { - const logs = await fs.readFile(getLogsFilePath(), { encoding: 'utf-8' }); - const connections = await serverConnections.getOpenedConnectionReport(); - try { - const response = await axios.default.post( - 'https://api.github.com/gists', - { - description: `DbGate ${currentVersion.version} error report`, - public: false, - files: { - 'logs.jsonl': { - content: logs, - }, - 'os.json': { - content: JSON.stringify( - { - release: os.release(), - arch: os.arch(), - machine: os.machine(), - platform: os.platform(), - type: os.type(), - }, - null, - 2 - ), - }, - 'platform.json': { - content: JSON.stringify( - _.omit( - { - ...platformInfo, - }, - ['defaultKeyfile', 'sshAuthSock'] - ), - null, - 2 - ), - }, - 'connections.json': { - content: JSON.stringify(connections, null, 2), - }, - 'version.json': { - content: JSON.stringify(currentVersion, null, 2), - }, - }, - }, - { - headers: { - Authorization: `token ${await this.getGistToken()}`, - 'Content-Type': 'application/json', - Accept: 'application/vnd.github.v3+json', - }, - } - ); - - return response.data; - } catch (err) { - logger.error(extractErrorLogData(err), 'DBGM-00148 Error uploading gist'); - - return { - apiErrorMessage: err.message, - }; - // console.error('Error creating gist:', error.response ? error.response.data : error.message); - } - }, - - deleteGist_meta: true, - async deleteGist({ url }) { - const response = await axios.default.delete(url, { - headers: { - Authorization: `token ${await this.getGistToken()}`, - 'Content-Type': 'application/json', - Accept: 'application/vnd.github.v3+json', - }, - }); - return true; - }, +// return { +// apiErrorMessage: err.message, +// }; +// // console.error('Error creating gist:', error.response ? error.response.data : error.message); +// } +// }, }; diff --git a/packages/api/src/gistSecret.js b/packages/api/src/gistSecret.js deleted file mode 100644 index 2b1442e4e..000000000 --- a/packages/api/src/gistSecret.js +++ /dev/null @@ -1 +0,0 @@ -module.exports = process.env.GIST_UPLOAD_SECRET; diff --git a/packages/api/src/index.js b/packages/api/src/index.js index 17f55ed7a..61816a866 100644 --- a/packages/api/src/index.js +++ b/packages/api/src/index.js @@ -5,6 +5,7 @@ const moment = require('moment'); const path = require('path'); const { logsdir, setLogsFilePath, getLogsFilePath } = require('./utility/directories'); const currentVersion = require('./currentVersion'); +const _ = require('lodash'); const logger = getLogger('apiIndex'); @@ -68,7 +69,7 @@ function configureLogger() { } const additionals = {}; const finalMsg = - msg.msg && msg.msg.match(/^DBGM-\d\d\d\d\d/) + _.isString(msg.msg) && msg.msg.match(/^DBGM-\d\d\d\d\d/) ? { ...msg, msg: msg.msg.substring(10).trimStart(), diff --git a/packages/api/src/main.js b/packages/api/src/main.js index d59c8b3df..314c5c899 100644 --- a/packages/api/src/main.js +++ b/packages/api/src/main.js @@ -29,6 +29,8 @@ const files = require('./controllers/files'); const scheduler = require('./controllers/scheduler'); const queryHistory = require('./controllers/queryHistory'); const cloud = require('./controllers/cloud'); +const teamFiles = require('./controllers/teamFiles'); + const onFinished = require('on-finished'); const processArgs = require('./utility/processArgs'); @@ -264,6 +266,7 @@ function useAllControllers(app, electron) { useController(app, electron, '/apps', apps); useController(app, electron, '/auth', auth); useController(app, electron, '/cloud', cloud); + useController(app, electron, '/team-files', teamFiles); } function setElectronSender(electronSender) { diff --git a/packages/api/src/proc/databaseConnectionProcess.js b/packages/api/src/proc/databaseConnectionProcess.js index cf3b42639..bed349355 100644 --- a/packages/api/src/proc/databaseConnectionProcess.js +++ b/packages/api/src/proc/databaseConnectionProcess.js @@ -17,13 +17,14 @@ const requireEngineDriver = require('../utility/requireEngineDriver'); const { connectUtility } = require('../utility/connectUtility'); const { handleProcessCommunication } = require('../utility/processComm'); const generateDeploySql = require('../shell/generateDeploySql'); -const { dumpSqlSelect } = require('dbgate-sqltree'); +const { dumpSqlSelect, scriptToSql } = require('dbgate-sqltree'); const { allowExecuteCustomScript, handleQueryStream } = require('../utility/handleQueryStream'); const dbgateApi = require('../shell'); const requirePlugin = require('../shell/requirePlugin'); const path = require('path'); const { rundir } = require('../utility/directories'); const fs = require('fs-extra'); +const { changeSetToSql } = require('dbgate-datalib'); const logger = getLogger('dbconnProcess'); @@ -348,6 +349,25 @@ async function handleUpdateCollection({ msgid, changeSet }) { } } +async function handleSaveTableData({ msgid, changeSet }) { + await waitStructure(); + try { + const driver = requireEngineDriver(storedConnection); + const script = driver.createSaveChangeSetScript(changeSet, analysedStructure, () => + changeSetToSql(changeSet, analysedStructure, driver.dialect) + ); + const sql = scriptToSql(driver, script); + await driver.script(dbhan, sql, { useTransaction: true }); + process.send({ msgtype: 'response', msgid }); + } catch (err) { + process.send({ + msgtype: 'response', + msgid, + errorMessage: extractErrorMessage(err, 'Error executing SQL script'), + }); + } +} + async function handleSqlPreview({ msgid, objects, options }) { await waitStructure(); const driver = requireEngineDriver(storedConnection); @@ -464,6 +484,7 @@ const messageHandlers = { runScript: handleRunScript, runOperation: handleRunOperation, updateCollection: handleUpdateCollection, + saveTableData: handleSaveTableData, collectionData: handleCollectionData, loadKeys: handleLoadKeys, scanKeys: handleScanKeys, diff --git a/packages/api/src/proc/serverConnectionProcess.js b/packages/api/src/proc/serverConnectionProcess.js index d58aa18ed..f9a2325c1 100644 --- a/packages/api/src/proc/serverConnectionProcess.js +++ b/packages/api/src/proc/serverConnectionProcess.js @@ -146,6 +146,30 @@ async function handleServerSummary({ msgid }) { return handleDriverDataCore(msgid, driver => driver.serverSummary(dbhan)); } +async function handleKillDatabaseProcess({ msgid, pid }) { + await waitConnected(); + const driver = requireEngineDriver(storedConnection); + + try { + const result = await driver.killProcess(dbhan, Number(pid)); + process.send({ msgtype: 'response', msgid, result }); + } catch (err) { + process.send({ msgtype: 'response', msgid, errorMessage: err.message }); + } +} + +async function handleListDatabaseProcesses({ msgid }) { + await waitConnected(); + const driver = requireEngineDriver(storedConnection); + + try { + const result = await driver.listProcesses(dbhan); + process.send({ msgtype: 'response', msgid, result }); + } catch (err) { + process.send({ msgtype: 'response', msgid, errorMessage: err.message }); + } +} + async function handleSummaryCommand({ msgid, command, row }) { return handleDriverDataCore(msgid, driver => driver.summaryCommand(dbhan, command, row)); } @@ -154,6 +178,8 @@ const messageHandlers = { connect: handleConnect, ping: handlePing, serverSummary: handleServerSummary, + killDatabaseProcess: handleKillDatabaseProcess, + listDatabaseProcesses: handleListDatabaseProcesses, summaryCommand: handleSummaryCommand, createDatabase: props => handleDatabaseOp('createDatabase', props), dropDatabase: props => handleDatabaseOp('dropDatabase', props), diff --git a/packages/api/src/shell/copyStream.js b/packages/api/src/shell/copyStream.js index 5da2aaad0..978d41261 100644 --- a/packages/api/src/shell/copyStream.js +++ b/packages/api/src/shell/copyStream.js @@ -65,6 +65,8 @@ async function copyStream(input, output, options) { }); } } catch (err) { + logger.error(extractErrorLogData(err, { progressName }), 'DBGM-00157 Import/export job failed'); + process.send({ msgtype: 'copyStreamError', copyStreamError: { @@ -82,8 +84,6 @@ async function copyStream(input, output, options) { errorMessage: extractErrorMessage(err), }); } - - logger.error(extractErrorLogData(err, { progressName }), 'DBGM-00157 Import/export job failed'); // throw err; } } diff --git a/packages/api/src/shell/dataReplicator.js b/packages/api/src/shell/dataReplicator.js index cc97b1a7e..b84829376 100644 --- a/packages/api/src/shell/dataReplicator.js +++ b/packages/api/src/shell/dataReplicator.js @@ -64,6 +64,7 @@ async function dataReplicator({ createNew: compileOperationFunction(item.createNew, item.createCondition), updateExisting: compileOperationFunction(item.updateExisting, item.updateCondition), deleteMissing: !!item.deleteMissing, + skipUpdateColumns: item.skipUpdateColumns, deleteRestrictionColumns: item.deleteRestrictionColumns ?? [], openStream: item.openStream ? item.openStream diff --git a/packages/api/src/storageModel.js b/packages/api/src/storageModel.js index 4086a5b43..1557a224f 100644 --- a/packages/api/src/storageModel.js +++ b/packages/api/src/storageModel.js @@ -360,6 +360,12 @@ module.exports = { "columnName": "value", "dataType": "varchar(1000)", "notNull": false + }, + { + "pureName": "config", + "columnName": "valueType", + "dataType": "varchar(50)", + "notNull": false } ], "foreignKeys": [], @@ -680,9 +686,34 @@ module.exports = { "columnName": "connectionDefinition", "dataType": "text", "notNull": false + }, + { + "pureName": "connections", + "columnName": "import_source_id", + "dataType": "int", + "notNull": false + }, + { + "pureName": "connections", + "columnName": "id_original", + "dataType": "varchar(250)", + "notNull": false + } + ], + "foreignKeys": [ + { + "constraintType": "foreignKey", + "constraintName": "FK_connections_import_source_id", + "pureName": "connections", + "refTableName": "import_sources", + "columns": [ + { + "columnName": "import_source_id", + "refColumnName": "id" + } + ] } ], - "foreignKeys": [], "primaryKey": { "pureName": "connections", "constraintType": "primaryKey", @@ -694,6 +725,131 @@ module.exports = { ] } }, + { + "pureName": "database_permission_roles", + "columns": [ + { + "pureName": "database_permission_roles", + "columnName": "id", + "dataType": "int", + "autoIncrement": true, + "notNull": true + }, + { + "pureName": "database_permission_roles", + "columnName": "name", + "dataType": "varchar(100)", + "notNull": true + } + ], + "foreignKeys": [], + "primaryKey": { + "pureName": "database_permission_roles", + "constraintType": "primaryKey", + "constraintName": "PK_database_permission_roles", + "columns": [ + { + "columnName": "id" + } + ] + }, + "preloadedRows": [ + { + "id": -1, + "name": "view" + }, + { + "id": -2, + "name": "read_content" + }, + { + "id": -3, + "name": "write_data" + }, + { + "id": -4, + "name": "run_script" + }, + { + "id": -5, + "name": "deny" + } + ] + }, + { + "pureName": "file_permission_roles", + "columns": [ + { + "pureName": "file_permission_roles", + "columnName": "id", + "dataType": "int", + "autoIncrement": true, + "notNull": true + }, + { + "pureName": "file_permission_roles", + "columnName": "name", + "dataType": "varchar(100)", + "notNull": true + } + ], + "foreignKeys": [], + "primaryKey": { + "pureName": "file_permission_roles", + "constraintType": "primaryKey", + "constraintName": "PK_file_permission_roles", + "columns": [ + { + "columnName": "id" + } + ] + }, + "preloadedRows": [ + { + "id": -1, + "name": "allow" + }, + { + "id": -2, + "name": "deny" + } + ] + }, + { + "pureName": "import_sources", + "columns": [ + { + "pureName": "import_sources", + "columnName": "id", + "dataType": "int", + "autoIncrement": true, + "notNull": true + }, + { + "pureName": "import_sources", + "columnName": "name", + "dataType": "varchar(250)", + "notNull": true + } + ], + "foreignKeys": [], + "primaryKey": { + "pureName": "import_sources", + "constraintType": "primaryKey", + "constraintName": "PK_import_sources", + "columns": [ + { + "columnName": "id" + } + ] + }, + "preloadedRows": [ + { + "id": -1, + "name": "env" + } + ] + }, { "pureName": "roles", "columns": [ @@ -709,9 +865,34 @@ module.exports = { "columnName": "name", "dataType": "varchar(250)", "notNull": false + }, + { + "pureName": "roles", + "columnName": "import_source_id", + "dataType": "int", + "notNull": false + }, + { + "pureName": "roles", + "columnName": "id_original", + "dataType": "varchar(250)", + "notNull": false + } + ], + "foreignKeys": [ + { + "constraintType": "foreignKey", + "constraintName": "FK_roles_import_source_id", + "pureName": "roles", + "refTableName": "import_sources", + "columns": [ + { + "columnName": "import_source_id", + "refColumnName": "id" + } + ] } ], - "foreignKeys": [], "primaryKey": { "pureName": "roles", "constraintType": "primaryKey", @@ -758,6 +939,12 @@ module.exports = { "columnName": "connection_id", "dataType": "int", "notNull": true + }, + { + "pureName": "role_connections", + "columnName": "import_source_id", + "dataType": "int", + "notNull": false } ], "foreignKeys": [ @@ -786,6 +973,18 @@ module.exports = { "refColumnName": "id" } ] + }, + { + "constraintType": "foreignKey", + "constraintName": "FK_role_connections_import_source_id", + "pureName": "role_connections", + "refTableName": "import_sources", + "columns": [ + { + "columnName": "import_source_id", + "refColumnName": "id" + } + ] } ], "primaryKey": { @@ -799,6 +998,201 @@ module.exports = { ] } }, + { + "pureName": "role_databases", + "columns": [ + { + "pureName": "role_databases", + "columnName": "id", + "dataType": "int", + "autoIncrement": true, + "notNull": true + }, + { + "pureName": "role_databases", + "columnName": "role_id", + "dataType": "int", + "notNull": true + }, + { + "pureName": "role_databases", + "columnName": "connection_id", + "dataType": "int", + "notNull": false + }, + { + "pureName": "role_databases", + "columnName": "database_names_list", + "dataType": "varchar(1000)", + "notNull": false + }, + { + "pureName": "role_databases", + "columnName": "database_names_regex", + "dataType": "varchar(1000)", + "notNull": false + }, + { + "pureName": "role_databases", + "columnName": "database_permission_role_id", + "dataType": "int", + "notNull": true + }, + { + "pureName": "role_databases", + "columnName": "import_source_id", + "dataType": "int", + "notNull": false + }, + { + "pureName": "role_databases", + "columnName": "id_original", + "dataType": "varchar(250)", + "notNull": false + } + ], + "foreignKeys": [ + { + "constraintType": "foreignKey", + "constraintName": "FK_role_databases_role_id", + "pureName": "role_databases", + "refTableName": "roles", + "deleteAction": "CASCADE", + "columns": [ + { + "columnName": "role_id", + "refColumnName": "id" + } + ] + }, + { + "constraintType": "foreignKey", + "constraintName": "FK_role_databases_connection_id", + "pureName": "role_databases", + "refTableName": "connections", + "deleteAction": "CASCADE", + "columns": [ + { + "columnName": "connection_id", + "refColumnName": "id" + } + ] + }, + { + "constraintType": "foreignKey", + "constraintName": "FK_role_databases_database_permission_role_id", + "pureName": "role_databases", + "refTableName": "database_permission_roles", + "columns": [ + { + "columnName": "database_permission_role_id", + "refColumnName": "id" + } + ] + }, + { + "constraintType": "foreignKey", + "constraintName": "FK_role_databases_import_source_id", + "pureName": "role_databases", + "refTableName": "import_sources", + "columns": [ + { + "columnName": "import_source_id", + "refColumnName": "id" + } + ] + } + ], + "primaryKey": { + "pureName": "role_databases", + "constraintType": "primaryKey", + "constraintName": "PK_role_databases", + "columns": [ + { + "columnName": "id" + } + ] + } + }, + { + "pureName": "role_files", + "columns": [ + { + "pureName": "role_files", + "columnName": "id", + "dataType": "int", + "autoIncrement": true, + "notNull": true + }, + { + "pureName": "role_files", + "columnName": "role_id", + "dataType": "int", + "notNull": true + }, + { + "pureName": "role_files", + "columnName": "folder_name", + "dataType": "varchar(100)", + "notNull": false + }, + { + "pureName": "role_files", + "columnName": "file_names_list", + "dataType": "varchar(1000)", + "notNull": false + }, + { + "pureName": "role_files", + "columnName": "file_names_regex", + "dataType": "varchar(1000)", + "notNull": false + }, + { + "pureName": "role_files", + "columnName": "file_permission_role_id", + "dataType": "int", + "notNull": true + } + ], + "foreignKeys": [ + { + "constraintType": "foreignKey", + "constraintName": "FK_role_files_role_id", + "pureName": "role_files", + "refTableName": "roles", + "deleteAction": "CASCADE", + "columns": [ + { + "columnName": "role_id", + "refColumnName": "id" + } + ] + }, + { + "constraintType": "foreignKey", + "constraintName": "FK_role_files_file_permission_role_id", + "pureName": "role_files", + "refTableName": "file_permission_roles", + "columns": [ + { + "columnName": "file_permission_role_id", + "refColumnName": "id" + } + ] + } + ], + "primaryKey": { + "pureName": "role_files", + "constraintType": "primaryKey", + "constraintName": "PK_role_files", + "columns": [ + { + "columnName": "id" + } + ] + } + }, { "pureName": "role_permissions", "columns": [ @@ -820,6 +1214,12 @@ module.exports = { "columnName": "permission", "dataType": "varchar(250)", "notNull": true + }, + { + "pureName": "role_permissions", + "columnName": "import_source_id", + "dataType": "int", + "notNull": false } ], "foreignKeys": [ @@ -835,6 +1235,18 @@ module.exports = { "refColumnName": "id" } ] + }, + { + "constraintType": "foreignKey", + "constraintName": "FK_role_permissions_import_source_id", + "pureName": "role_permissions", + "refTableName": "import_sources", + "columns": [ + { + "columnName": "import_source_id", + "refColumnName": "id" + } + ] } ], "primaryKey": { @@ -848,6 +1260,511 @@ module.exports = { ] } }, + { + "pureName": "role_tables", + "columns": [ + { + "pureName": "role_tables", + "columnName": "id", + "dataType": "int", + "autoIncrement": true, + "notNull": true + }, + { + "pureName": "role_tables", + "columnName": "role_id", + "dataType": "int", + "notNull": true + }, + { + "pureName": "role_tables", + "columnName": "connection_id", + "dataType": "int", + "notNull": false + }, + { + "pureName": "role_tables", + "columnName": "database_names_list", + "dataType": "varchar(1000)", + "notNull": false + }, + { + "pureName": "role_tables", + "columnName": "database_names_regex", + "dataType": "varchar(1000)", + "notNull": false + }, + { + "pureName": "role_tables", + "columnName": "schema_names_list", + "dataType": "varchar(1000)", + "notNull": false + }, + { + "pureName": "role_tables", + "columnName": "schema_names_regex", + "dataType": "varchar(1000)", + "notNull": false + }, + { + "pureName": "role_tables", + "columnName": "table_names_list", + "dataType": "varchar(1000)", + "notNull": false + }, + { + "pureName": "role_tables", + "columnName": "table_names_regex", + "dataType": "varchar(1000)", + "notNull": false + }, + { + "pureName": "role_tables", + "columnName": "table_permission_role_id", + "dataType": "int", + "notNull": true + }, + { + "pureName": "role_tables", + "columnName": "table_permission_scope_id", + "dataType": "int", + "notNull": true + }, + { + "pureName": "role_tables", + "columnName": "import_source_id", + "dataType": "int", + "notNull": false + }, + { + "pureName": "role_tables", + "columnName": "id_original", + "dataType": "varchar(250)", + "notNull": false + } + ], + "foreignKeys": [ + { + "constraintType": "foreignKey", + "constraintName": "FK_role_tables_role_id", + "pureName": "role_tables", + "refTableName": "roles", + "deleteAction": "CASCADE", + "columns": [ + { + "columnName": "role_id", + "refColumnName": "id" + } + ] + }, + { + "constraintType": "foreignKey", + "constraintName": "FK_role_tables_connection_id", + "pureName": "role_tables", + "refTableName": "connections", + "deleteAction": "CASCADE", + "columns": [ + { + "columnName": "connection_id", + "refColumnName": "id" + } + ] + }, + { + "constraintType": "foreignKey", + "constraintName": "FK_role_tables_table_permission_role_id", + "pureName": "role_tables", + "refTableName": "table_permission_roles", + "columns": [ + { + "columnName": "table_permission_role_id", + "refColumnName": "id" + } + ] + }, + { + "constraintType": "foreignKey", + "constraintName": "FK_role_tables_table_permission_scope_id", + "pureName": "role_tables", + "refTableName": "table_permission_scopes", + "columns": [ + { + "columnName": "table_permission_scope_id", + "refColumnName": "id" + } + ] + }, + { + "constraintType": "foreignKey", + "constraintName": "FK_role_tables_import_source_id", + "pureName": "role_tables", + "refTableName": "import_sources", + "columns": [ + { + "columnName": "import_source_id", + "refColumnName": "id" + } + ] + } + ], + "primaryKey": { + "pureName": "role_tables", + "constraintType": "primaryKey", + "constraintName": "PK_role_tables", + "columns": [ + { + "columnName": "id" + } + ] + } + }, + { + "pureName": "role_team_files", + "columns": [ + { + "pureName": "role_team_files", + "columnName": "id", + "dataType": "int", + "autoIncrement": true, + "notNull": true + }, + { + "pureName": "role_team_files", + "columnName": "role_id", + "dataType": "int", + "notNull": true + }, + { + "pureName": "role_team_files", + "columnName": "team_file_id", + "dataType": "int", + "notNull": true + }, + { + "pureName": "role_team_files", + "columnName": "allow_read", + "dataType": "int", + "notNull": false + }, + { + "pureName": "role_team_files", + "columnName": "allow_write", + "dataType": "int", + "notNull": false + }, + { + "pureName": "role_team_files", + "columnName": "allow_use", + "dataType": "int", + "notNull": false + } + ], + "foreignKeys": [ + { + "constraintType": "foreignKey", + "constraintName": "FK_role_team_files_role_id", + "pureName": "role_team_files", + "refTableName": "roles", + "deleteAction": "CASCADE", + "columns": [ + { + "columnName": "role_id", + "refColumnName": "id" + } + ] + }, + { + "constraintType": "foreignKey", + "constraintName": "FK_role_team_files_team_file_id", + "pureName": "role_team_files", + "refTableName": "team_files", + "deleteAction": "CASCADE", + "columns": [ + { + "columnName": "team_file_id", + "refColumnName": "id" + } + ] + } + ], + "primaryKey": { + "pureName": "role_team_files", + "constraintType": "primaryKey", + "constraintName": "PK_role_team_files", + "columns": [ + { + "columnName": "id" + } + ] + } + }, + { + "pureName": "table_permission_roles", + "columns": [ + { + "pureName": "table_permission_roles", + "columnName": "id", + "dataType": "int", + "autoIncrement": true, + "notNull": true + }, + { + "pureName": "table_permission_roles", + "columnName": "name", + "dataType": "varchar(100)", + "notNull": true + } + ], + "foreignKeys": [], + "primaryKey": { + "pureName": "table_permission_roles", + "constraintType": "primaryKey", + "constraintName": "PK_table_permission_roles", + "columns": [ + { + "columnName": "id" + } + ] + }, + "preloadedRows": [ + { + "id": -1, + "name": "read" + }, + { + "id": -2, + "name": "update_only" + }, + { + "id": -3, + "name": "create_update_delete" + }, + { + "id": -4, + "name": "run_script" + }, + { + "id": -5, + "name": "deny" + } + ] + }, + { + "pureName": "table_permission_scopes", + "columns": [ + { + "pureName": "table_permission_scopes", + "columnName": "id", + "dataType": "int", + "autoIncrement": true, + "notNull": true + }, + { + "pureName": "table_permission_scopes", + "columnName": "name", + "dataType": "varchar(100)", + "notNull": true + } + ], + "foreignKeys": [], + "primaryKey": { + "pureName": "table_permission_scopes", + "constraintType": "primaryKey", + "constraintName": "PK_table_permission_scopes", + "columns": [ + { + "columnName": "id" + } + ] + }, + "preloadedRows": [ + { + "id": -1, + "name": "all_objects" + }, + { + "id": -2, + "name": "tables" + }, + { + "id": -3, + "name": "views" + }, + { + "id": -4, + "name": "tables_views_collections" + }, + { + "id": -5, + "name": "procedures" + }, + { + "id": -6, + "name": "functions" + }, + { + "id": -7, + "name": "triggers" + }, + { + "id": -8, + "name": "sql_objects" + }, + { + "id": -9, + "name": "collections" + } + ] + }, + { + "pureName": "team_files", + "columns": [ + { + "pureName": "team_files", + "columnName": "id", + "dataType": "int", + "autoIncrement": true, + "notNull": true + }, + { + "pureName": "team_files", + "columnName": "file_name", + "dataType": "varchar(250)", + "notNull": false + }, + { + "pureName": "team_files", + "columnName": "file_content", + "dataType": "text", + "notNull": false + }, + { + "pureName": "team_files", + "columnName": "file_type_id", + "dataType": "int", + "notNull": true + }, + { + "pureName": "team_files", + "columnName": "owner_user_id", + "dataType": "int", + "notNull": false + }, + { + "pureName": "team_files", + "columnName": "metadata", + "dataType": "varchar(1000)", + "notNull": false + } + ], + "foreignKeys": [ + { + "constraintType": "foreignKey", + "constraintName": "FK_team_files_file_type_id", + "pureName": "team_files", + "refTableName": "team_file_types", + "columns": [ + { + "columnName": "file_type_id", + "refColumnName": "id" + } + ] + }, + { + "constraintType": "foreignKey", + "constraintName": "FK_team_files_owner_user_id", + "pureName": "team_files", + "refTableName": "users", + "columns": [ + { + "columnName": "owner_user_id", + "refColumnName": "id" + } + ] + } + ], + "primaryKey": { + "pureName": "team_files", + "constraintType": "primaryKey", + "constraintName": "PK_team_files", + "columns": [ + { + "columnName": "id" + } + ] + } + }, + { + "pureName": "team_file_types", + "columns": [ + { + "pureName": "team_file_types", + "columnName": "id", + "dataType": "int", + "notNull": true + }, + { + "pureName": "team_file_types", + "columnName": "name", + "dataType": "varchar(250)", + "notNull": true + }, + { + "pureName": "team_file_types", + "columnName": "format", + "dataType": "varchar(50)", + "notNull": false + } + ], + "foreignKeys": [], + "primaryKey": { + "pureName": "team_file_types", + "constraintType": "primaryKey", + "constraintName": "PK_team_file_types", + "columns": [ + { + "columnName": "id" + } + ] + }, + "preloadedRows": [ + { + "id": -1, + "name": "sql", + "format": "text" + }, + { + "id": -2, + "name": "diagrams", + "format": "json" + }, + { + "id": -3, + "name": "query", + "format": "json" + }, + { + "id": -4, + "name": "perspectives", + "format": "json" + }, + { + "id": -5, + "name": "impexp", + "format": "json" + }, + { + "id": -6, + "name": "shell", + "format": "text" + }, + { + "id": -7, + "name": "dbcompare", + "format": "json" + } + ] + }, { "pureName": "users", "columns": [ @@ -951,6 +1868,177 @@ module.exports = { ] } }, + { + "pureName": "user_databases", + "columns": [ + { + "pureName": "user_databases", + "columnName": "id", + "dataType": "int", + "autoIncrement": true, + "notNull": true + }, + { + "pureName": "user_databases", + "columnName": "user_id", + "dataType": "int", + "notNull": true + }, + { + "pureName": "user_databases", + "columnName": "connection_id", + "dataType": "int", + "notNull": false + }, + { + "pureName": "user_databases", + "columnName": "database_names_list", + "dataType": "varchar(1000)", + "notNull": false + }, + { + "pureName": "user_databases", + "columnName": "database_names_regex", + "dataType": "varchar(1000)", + "notNull": false + }, + { + "pureName": "user_databases", + "columnName": "database_permission_role_id", + "dataType": "int", + "notNull": true + } + ], + "foreignKeys": [ + { + "constraintType": "foreignKey", + "constraintName": "FK_user_databases_user_id", + "pureName": "user_databases", + "refTableName": "users", + "deleteAction": "CASCADE", + "columns": [ + { + "columnName": "user_id", + "refColumnName": "id" + } + ] + }, + { + "constraintType": "foreignKey", + "constraintName": "FK_user_databases_connection_id", + "pureName": "user_databases", + "refTableName": "connections", + "deleteAction": "CASCADE", + "columns": [ + { + "columnName": "connection_id", + "refColumnName": "id" + } + ] + }, + { + "constraintType": "foreignKey", + "constraintName": "FK_user_databases_database_permission_role_id", + "pureName": "user_databases", + "refTableName": "database_permission_roles", + "columns": [ + { + "columnName": "database_permission_role_id", + "refColumnName": "id" + } + ] + } + ], + "primaryKey": { + "pureName": "user_databases", + "constraintType": "primaryKey", + "constraintName": "PK_user_databases", + "columns": [ + { + "columnName": "id" + } + ] + } + }, + { + "pureName": "user_files", + "columns": [ + { + "pureName": "user_files", + "columnName": "id", + "dataType": "int", + "autoIncrement": true, + "notNull": true + }, + { + "pureName": "user_files", + "columnName": "user_id", + "dataType": "int", + "notNull": true + }, + { + "pureName": "user_files", + "columnName": "folder_name", + "dataType": "varchar(100)", + "notNull": false + }, + { + "pureName": "user_files", + "columnName": "file_names_list", + "dataType": "varchar(1000)", + "notNull": false + }, + { + "pureName": "user_files", + "columnName": "file_names_regex", + "dataType": "varchar(1000)", + "notNull": false + }, + { + "pureName": "user_files", + "columnName": "file_permission_role_id", + "dataType": "int", + "notNull": true + } + ], + "foreignKeys": [ + { + "constraintType": "foreignKey", + "constraintName": "FK_user_files_user_id", + "pureName": "user_files", + "refTableName": "users", + "deleteAction": "CASCADE", + "columns": [ + { + "columnName": "user_id", + "refColumnName": "id" + } + ] + }, + { + "constraintType": "foreignKey", + "constraintName": "FK_user_files_file_permission_role_id", + "pureName": "user_files", + "refTableName": "file_permission_roles", + "columns": [ + { + "columnName": "file_permission_role_id", + "refColumnName": "id" + } + ] + } + ], + "primaryKey": { + "pureName": "user_files", + "constraintType": "primaryKey", + "constraintName": "PK_user_files", + "columns": [ + { + "columnName": "id" + } + ] + } + }, { "pureName": "user_permissions", "columns": [ @@ -1061,6 +2149,220 @@ module.exports = { } ] } + }, + { + "pureName": "user_tables", + "columns": [ + { + "pureName": "user_tables", + "columnName": "id", + "dataType": "int", + "autoIncrement": true, + "notNull": true + }, + { + "pureName": "user_tables", + "columnName": "user_id", + "dataType": "int", + "notNull": true + }, + { + "pureName": "user_tables", + "columnName": "connection_id", + "dataType": "int", + "notNull": false + }, + { + "pureName": "user_tables", + "columnName": "database_names_list", + "dataType": "varchar(1000)", + "notNull": false + }, + { + "pureName": "user_tables", + "columnName": "database_names_regex", + "dataType": "varchar(1000)", + "notNull": false + }, + { + "pureName": "user_tables", + "columnName": "schema_names_list", + "dataType": "varchar(1000)", + "notNull": false + }, + { + "pureName": "user_tables", + "columnName": "schema_names_regex", + "dataType": "varchar(1000)", + "notNull": false + }, + { + "pureName": "user_tables", + "columnName": "table_names_list", + "dataType": "varchar(1000)", + "notNull": false + }, + { + "pureName": "user_tables", + "columnName": "table_names_regex", + "dataType": "varchar(1000)", + "notNull": false + }, + { + "pureName": "user_tables", + "columnName": "table_permission_role_id", + "dataType": "int", + "notNull": true + }, + { + "pureName": "user_tables", + "columnName": "table_permission_scope_id", + "dataType": "int", + "notNull": true + } + ], + "foreignKeys": [ + { + "constraintType": "foreignKey", + "constraintName": "FK_user_tables_user_id", + "pureName": "user_tables", + "refTableName": "users", + "deleteAction": "CASCADE", + "columns": [ + { + "columnName": "user_id", + "refColumnName": "id" + } + ] + }, + { + "constraintType": "foreignKey", + "constraintName": "FK_user_tables_connection_id", + "pureName": "user_tables", + "refTableName": "connections", + "deleteAction": "CASCADE", + "columns": [ + { + "columnName": "connection_id", + "refColumnName": "id" + } + ] + }, + { + "constraintType": "foreignKey", + "constraintName": "FK_user_tables_table_permission_role_id", + "pureName": "user_tables", + "refTableName": "table_permission_roles", + "columns": [ + { + "columnName": "table_permission_role_id", + "refColumnName": "id" + } + ] + }, + { + "constraintType": "foreignKey", + "constraintName": "FK_user_tables_table_permission_scope_id", + "pureName": "user_tables", + "refTableName": "table_permission_scopes", + "columns": [ + { + "columnName": "table_permission_scope_id", + "refColumnName": "id" + } + ] + } + ], + "primaryKey": { + "pureName": "user_tables", + "constraintType": "primaryKey", + "constraintName": "PK_user_tables", + "columns": [ + { + "columnName": "id" + } + ] + } + }, + { + "pureName": "user_team_files", + "columns": [ + { + "pureName": "user_team_files", + "columnName": "id", + "dataType": "int", + "autoIncrement": true, + "notNull": true + }, + { + "pureName": "user_team_files", + "columnName": "user_id", + "dataType": "int", + "notNull": true + }, + { + "pureName": "user_team_files", + "columnName": "team_file_id", + "dataType": "int", + "notNull": true + }, + { + "pureName": "user_team_files", + "columnName": "allow_read", + "dataType": "int", + "notNull": false + }, + { + "pureName": "user_team_files", + "columnName": "allow_write", + "dataType": "int", + "notNull": false + }, + { + "pureName": "user_team_files", + "columnName": "allow_use", + "dataType": "int", + "notNull": false + } + ], + "foreignKeys": [ + { + "constraintType": "foreignKey", + "constraintName": "FK_user_team_files_user_id", + "pureName": "user_team_files", + "refTableName": "users", + "deleteAction": "CASCADE", + "columns": [ + { + "columnName": "user_id", + "refColumnName": "id" + } + ] + }, + { + "constraintType": "foreignKey", + "constraintName": "FK_user_team_files_team_file_id", + "pureName": "user_team_files", + "refTableName": "team_files", + "deleteAction": "CASCADE", + "columns": [ + { + "columnName": "team_file_id", + "refColumnName": "id" + } + ] + } + ], + "primaryKey": { + "pureName": "user_team_files", + "constraintType": "primaryKey", + "constraintName": "PK_user_team_files", + "columns": [ + { + "columnName": "id" + } + ] + } } ], "collections": [], diff --git a/packages/api/src/utility/cloudIntf.js b/packages/api/src/utility/cloudIntf.js index c96fce922..51e302c15 100644 --- a/packages/api/src/utility/cloudIntf.js +++ b/packages/api/src/utility/cloudIntf.js @@ -13,11 +13,12 @@ const socket = require('./socket'); const config = require('../controllers/config'); const simpleEncryptor = require('simple-encryptor'); const currentVersion = require('../currentVersion'); -const { getPublicIpInfo } = require('./hardwareFingerprint'); const logger = getLogger('cloudIntf'); let cloudFiles = null; +let promoWidgetData = null; +let promoWidgetDataLoaded = false; const DBGATE_IDENTITY_URL = process.env.LOCAL_DBGATE_IDENTITY ? 'http://localhost:3103' @@ -192,7 +193,7 @@ async function getCloudSigninHeaders(holder = null) { return null; } -async function updateCloudFiles(isRefresh) { +async function updateCloudFiles(isRefresh, language) { let lastCloudFilesTags; try { lastCloudFilesTags = await fs.readFile(path.join(datadir(), 'cloud-files-tags.txt'), 'utf-8'); @@ -200,8 +201,6 @@ async function updateCloudFiles(isRefresh) { lastCloudFilesTags = ''; } - const ipInfo = await getPublicIpInfo(); - const tags = (await collectCloudFilesSearchTags()).join(','); let lastCheckedTm = 0; if (tags == lastCloudFilesTags && cloudFiles.length > 0) { @@ -213,12 +212,13 @@ async function updateCloudFiles(isRefresh) { const resp = await axios.default.get( `${DBGATE_CLOUD_URL}/public-cloud-updates?lastCheckedTm=${lastCheckedTm}&tags=${tags}&isRefresh=${ isRefresh ? 1 : 0 - }&country=${ipInfo?.country || ''}`, + }`, { headers: { ...getLicenseHttpHeaders(), ...(await getCloudInstanceHeaders()), 'x-app-version': currentVersion.version, + 'x-app-language': language || 'en', }, } ); @@ -262,15 +262,62 @@ async function getPublicFileData(path) { return resp.data; } -async function refreshPublicFiles(isRefresh) { +async function ensurePromoWidgetDataLoaded() { + if (promoWidgetDataLoaded) { + return; + } + try { + const fileContent = await fs.readFile(path.join(datadir(), 'promo-widget.json'), 'utf-8'); + promoWidgetData = JSON.parse(fileContent); + } catch (err) { + promoWidgetData = null; + } + promoWidgetDataLoaded = true; +} + +async function updatePremiumPromoWidget(language) { + await ensurePromoWidgetDataLoaded(); + + const tags = (await collectCloudFilesSearchTags()).join(','); + + const resp = await axios.default.get( + `${DBGATE_CLOUD_URL}/premium-promo-widget?identifier=${promoWidgetData?.identifier ?? 'empty'}&tags=${tags}`, + { + headers: { + ...getLicenseHttpHeaders(), + ...(await getCloudInstanceHeaders()), + 'x-app-version': currentVersion.version, + 'x-app-language': language || 'en', + }, + } + ); + + if (!resp.data || resp.data?.state == 'unchanged') { + return; + } + + promoWidgetData = resp.data; + await fs.writeFile(path.join(datadir(), 'promo-widget.json'), JSON.stringify(promoWidgetData, null, 2)); + + socket.emitChanged(`promo-widget-changed`); +} + +async function refreshPublicFiles(isRefresh, uiLanguage) { + const language = platformInfo.isElectron + ? (await config.getCachedSettings())?.['localization.language'] || 'en' + : uiLanguage; if (!cloudFiles) { await loadCloudFiles(); } try { - await updateCloudFiles(isRefresh); + await updateCloudFiles(isRefresh, language); } catch (err) { logger.error(extractErrorLogData(err), 'DBGM-00166 Error updating cloud files'); } + const configSettings = await config.get(); + if (!isProApp() || configSettings?.trialDaysLeft != null) { + await updatePremiumPromoWidget(language); + } } async function callCloudApiGet(endpoint, signinHolder = null, additionalHeaders = {}) { @@ -423,6 +470,33 @@ function removeCloudCachedConnection(folid, cntid) { delete cloudConnectionCache[cacheKey]; } +async function getPublicIpInfo() { + try { + const resp = await axios.default.get(`${DBGATE_CLOUD_URL}/ipinfo`); + if (!resp.data?.ip) { + return { ip: 'unknown-ip' }; + } + return resp.data; + } catch (err) { + return { ip: 'unknown-ip' }; + } +} + +async function getPromoWidgetData() { + await ensurePromoWidgetDataLoaded(); + return promoWidgetData; +} + +async function getPromoWidgetPreview(campaign, variant) { + const resp = await axios.default.get(`${DBGATE_CLOUD_URL}/premium-promo-widget-preview/${campaign}/${variant}`); + return resp.data; +} + +async function getPromoWidgetList() { + const resp = await axios.default.get(`${DBGATE_CLOUD_URL}/promo-widget-list`); + return resp.data; +} + module.exports = { createDbGateIdentitySession, startCloudTokenChecking, @@ -439,4 +513,8 @@ module.exports = { removeCloudCachedConnection, readCloudTokenHolder, readCloudTestTokenHolder, + getPublicIpInfo, + getPromoWidgetData, + getPromoWidgetPreview, + getPromoWidgetList, }; diff --git a/packages/api/src/utility/envtools.js b/packages/api/src/utility/envtools.js new file mode 100644 index 000000000..83d09808c --- /dev/null +++ b/packages/api/src/utility/envtools.js @@ -0,0 +1,445 @@ +const path = require('path'); +const _ = require('lodash'); +const { safeJsonParse, getDatabaseFileLabel } = require('dbgate-tools'); +const crypto = require('crypto'); + +function extractConnectionsFromEnv(env) { + if (!env?.CONNECTIONS) { + return null; + } + + const connections = _.compact(env.CONNECTIONS.split(',')).map(id => ({ + _id: id, + engine: env[`ENGINE_${id}`], + server: env[`SERVER_${id}`], + user: env[`USER_${id}`], + password: env[`PASSWORD_${id}`], + passwordMode: env[`PASSWORD_MODE_${id}`], + port: env[`PORT_${id}`], + databaseUrl: env[`URL_${id}`], + useDatabaseUrl: !!env[`URL_${id}`], + databaseFile: env[`FILE_${id}`]?.replace( + '%%E2E_TEST_DATA_DIRECTORY%%', + path.join(path.dirname(path.dirname(__dirname)), 'e2e-tests', 'tmpdata') + ), + socketPath: env[`SOCKET_PATH_${id}`], + serviceName: env[`SERVICE_NAME_${id}`], + authType: env[`AUTH_TYPE_${id}`] || (env[`SOCKET_PATH_${id}`] ? 'socket' : undefined), + defaultDatabase: env[`DATABASE_${id}`] || (env[`FILE_${id}`] ? getDatabaseFileLabel(env[`FILE_${id}`]) : null), + singleDatabase: !!env[`DATABASE_${id}`] || !!env[`FILE_${id}`], + displayName: env[`LABEL_${id}`], + isReadOnly: env[`READONLY_${id}`], + databases: env[`DBCONFIG_${id}`] ? safeJsonParse(env[`DBCONFIG_${id}`]) : null, + allowedDatabases: env[`ALLOWED_DATABASES_${id}`]?.replace(/\|/g, '\n'), + allowedDatabasesRegex: env[`ALLOWED_DATABASES_REGEX_${id}`], + parent: env[`PARENT_${id}`] || undefined, + useSeparateSchemas: !!env[`USE_SEPARATE_SCHEMAS_${id}`], + localDataCenter: env[`LOCAL_DATA_CENTER_${id}`], + + // SSH tunnel + useSshTunnel: env[`USE_SSH_${id}`], + sshHost: env[`SSH_HOST_${id}`], + sshPort: env[`SSH_PORT_${id}`], + sshMode: env[`SSH_MODE_${id}`], + sshLogin: env[`SSH_LOGIN_${id}`], + sshPassword: env[`SSH_PASSWORD_${id}`], + sshKeyfile: env[`SSH_KEY_FILE_${id}`], + sshKeyfilePassword: env[`SSH_KEY_FILE_PASSWORD_${id}`], + + // SSL + useSsl: env[`USE_SSL_${id}`], + sslCaFile: env[`SSL_CA_FILE_${id}`], + sslCertFile: env[`SSL_CERT_FILE_${id}`], + sslCertFilePassword: env[`SSL_CERT_FILE_PASSWORD_${id}`], + sslKeyFile: env[`SSL_KEY_FILE_${id}`], + sslRejectUnauthorized: env[`SSL_REJECT_UNAUTHORIZED_${id}`], + trustServerCertificate: env[`SSL_TRUST_CERTIFICATE_${id}`], + })); + + return connections; +} + +function extractImportEntitiesFromEnv(env) { + const portalConnections = extractConnectionsFromEnv(env) || []; + + const connections = portalConnections.map((conn, index) => ({ + ...conn, + id_original: conn._id, + import_source_id: -1, + conid: crypto.randomUUID(), + _id: undefined, + id: index + 1, // autoincrement id + })); + + const connectionEnvIdToDbId = {}; + for (const conn of connections) { + connectionEnvIdToDbId[conn.id_original] = conn.id; + } + + const connectionsRegex = /^ROLE_(.+)_CONNECTIONS$/; + const permissionsRegex = /^ROLE_(.+)_PERMISSIONS$/; + + const dbConnectionRegex = /^ROLE_(.+)_DATABASES_(.+)_CONNECTION$/; + const dbDatabasesRegex = /^ROLE_(.+)_DATABASES_(.+)_DATABASES$/; + const dbDatabasesRegexRegex = /^ROLE_(.+)_DATABASES_(.+)_DATABASES_REGEX$/; + const dbPermissionRegex = /^ROLE_(.+)_DATABASES_(.+)_PERMISSION$/; + + const tableConnectionRegex = /^ROLE_(.+)_TABLES_(.+)_CONNECTION$/; + const tableDatabasesRegex = /^ROLE_(.+)_TABLES_(.+)_DATABASES$/; + const tableDatabasesRegexRegex = /^ROLE_(.+)_TABLES_(.+)_DATABASES_REGEX$/; + const tableSchemasRegex = /^ROLE_(.+)_TABLES_(.+)_SCHEMAS$/; + const tableSchemasRegexRegex = /^ROLE_(.+)_TABLES_(.+)_SCHEMAS_REGEX$/; + const tableTablesRegex = /^ROLE_(.+)_TABLES_(.+)_TABLES$/; + const tableTablesRegexRegex = /^ROLE_(.+)_TABLES_(.+)_TABLES_REGEX$/; + const tablePermissionRegex = /^ROLE_(.+)_TABLES_(.+)_PERMISSION$/; + const tableScopeRegex = /^ROLE_(.+)_TABLES_(.+)_SCOPE$/; + + const roles = []; + const role_connections = []; + const role_permissions = []; + const role_databases = []; + const role_tables = []; + + // Permission name to ID mappings + const databasePermissionMap = { + view: -1, + read_content: -2, + write_data: -3, + run_script: -4, + deny: -5, + }; + + const tablePermissionMap = { + read: -1, + update_only: -2, + create_update_delete: -3, + run_script: -4, + deny: -5, + }; + + const tableScopeMap = { + all_objects: -1, + tables: -2, + views: -3, + tables_views_collections: -4, + procedures: -5, + functions: -6, + triggers: -7, + sql_objects: -8, + collections: -9, + }; + + // Collect database and table permissions data + const databasePermissions = {}; + const tablePermissions = {}; + + // First pass: collect all database and table permission data + for (const key in env) { + const dbConnMatch = key.match(dbConnectionRegex); + const dbDatabasesMatch = key.match(dbDatabasesRegex); + const dbDatabasesRegexMatch = key.match(dbDatabasesRegexRegex); + const dbPermMatch = key.match(dbPermissionRegex); + + const tableConnMatch = key.match(tableConnectionRegex); + const tableDatabasesMatch = key.match(tableDatabasesRegex); + const tableDatabasesRegexMatch = key.match(tableDatabasesRegexRegex); + const tableSchemasMatch = key.match(tableSchemasRegex); + const tableSchemasRegexMatch = key.match(tableSchemasRegexRegex); + const tableTablesMatch = key.match(tableTablesRegex); + const tableTablesRegexMatch = key.match(tableTablesRegexRegex); + const tablePermMatch = key.match(tablePermissionRegex); + const tableScopeMatch = key.match(tableScopeRegex); + + // Database permissions + if (dbConnMatch) { + const [, roleName, permId] = dbConnMatch; + if (!databasePermissions[roleName]) databasePermissions[roleName] = {}; + if (!databasePermissions[roleName][permId]) databasePermissions[roleName][permId] = {}; + databasePermissions[roleName][permId].connection = env[key]; + } + if (dbDatabasesMatch) { + const [, roleName, permId] = dbDatabasesMatch; + if (!databasePermissions[roleName]) databasePermissions[roleName] = {}; + if (!databasePermissions[roleName][permId]) databasePermissions[roleName][permId] = {}; + databasePermissions[roleName][permId].databases = env[key]?.replace(/\|/g, '\n'); + } + if (dbDatabasesRegexMatch) { + const [, roleName, permId] = dbDatabasesRegexMatch; + if (!databasePermissions[roleName]) databasePermissions[roleName] = {}; + if (!databasePermissions[roleName][permId]) databasePermissions[roleName][permId] = {}; + databasePermissions[roleName][permId].databasesRegex = env[key]; + } + if (dbPermMatch) { + const [, roleName, permId] = dbPermMatch; + if (!databasePermissions[roleName]) databasePermissions[roleName] = {}; + if (!databasePermissions[roleName][permId]) databasePermissions[roleName][permId] = {}; + databasePermissions[roleName][permId].permission = env[key]; + } + + // Table permissions + if (tableConnMatch) { + const [, roleName, permId] = tableConnMatch; + if (!tablePermissions[roleName]) tablePermissions[roleName] = {}; + if (!tablePermissions[roleName][permId]) tablePermissions[roleName][permId] = {}; + tablePermissions[roleName][permId].connection = env[key]; + } + if (tableDatabasesMatch) { + const [, roleName, permId] = tableDatabasesMatch; + if (!tablePermissions[roleName]) tablePermissions[roleName] = {}; + if (!tablePermissions[roleName][permId]) tablePermissions[roleName][permId] = {}; + tablePermissions[roleName][permId].databases = env[key]?.replace(/\|/g, '\n'); + } + if (tableDatabasesRegexMatch) { + const [, roleName, permId] = tableDatabasesRegexMatch; + if (!tablePermissions[roleName]) tablePermissions[roleName] = {}; + if (!tablePermissions[roleName][permId]) tablePermissions[roleName][permId] = {}; + tablePermissions[roleName][permId].databasesRegex = env[key]; + } + if (tableSchemasMatch) { + const [, roleName, permId] = tableSchemasMatch; + if (!tablePermissions[roleName]) tablePermissions[roleName] = {}; + if (!tablePermissions[roleName][permId]) tablePermissions[roleName][permId] = {}; + tablePermissions[roleName][permId].schemas = env[key]; + } + if (tableSchemasRegexMatch) { + const [, roleName, permId] = tableSchemasRegexMatch; + if (!tablePermissions[roleName]) tablePermissions[roleName] = {}; + if (!tablePermissions[roleName][permId]) tablePermissions[roleName][permId] = {}; + tablePermissions[roleName][permId].schemasRegex = env[key]; + } + if (tableTablesMatch) { + const [, roleName, permId] = tableTablesMatch; + if (!tablePermissions[roleName]) tablePermissions[roleName] = {}; + if (!tablePermissions[roleName][permId]) tablePermissions[roleName][permId] = {}; + tablePermissions[roleName][permId].tables = env[key]?.replace(/\|/g, '\n'); + } + if (tableTablesRegexMatch) { + const [, roleName, permId] = tableTablesRegexMatch; + if (!tablePermissions[roleName]) tablePermissions[roleName] = {}; + if (!tablePermissions[roleName][permId]) tablePermissions[roleName][permId] = {}; + tablePermissions[roleName][permId].tablesRegex = env[key]; + } + if (tablePermMatch) { + const [, roleName, permId] = tablePermMatch; + if (!tablePermissions[roleName]) tablePermissions[roleName] = {}; + if (!tablePermissions[roleName][permId]) tablePermissions[roleName][permId] = {}; + tablePermissions[roleName][permId].permission = env[key]; + } + if (tableScopeMatch) { + const [, roleName, permId] = tableScopeMatch; + if (!tablePermissions[roleName]) tablePermissions[roleName] = {}; + if (!tablePermissions[roleName][permId]) tablePermissions[roleName][permId] = {}; + tablePermissions[roleName][permId].scope = env[key]; + } + } + + // Second pass: process roles, connections, and permissions + for (const key in env) { + const connMatch = key.match(connectionsRegex); + const permMatch = key.match(permissionsRegex); + if (connMatch) { + const roleName = connMatch[1]; + let role = roles.find(r => r.name === roleName); + if (!role) { + role = { + id: roles.length + 1, + name: roleName, + import_source_id: -1, + }; + roles.push(role); + } + const connIds = env[key] + .split(',') + .map(id => id.trim()) + .filter(id => id.length > 0); + for (const connId of connIds) { + const dbId = connectionEnvIdToDbId[connId]; + if (dbId) { + role_connections.push({ + role_id: role.id, + connection_id: dbId, + import_source_id: -1, + }); + } + } + } + if (permMatch) { + const roleName = permMatch[1]; + let role = roles.find(r => r.name === roleName); + if (!role) { + role = { + id: roles.length + 1, + name: roleName, + import_source_id: -1, + }; + roles.push(role); + } + const permissions = env[key] + .split(',') + .map(p => p.trim()) + .filter(p => p.length > 0); + for (const permission of permissions) { + role_permissions.push({ + role_id: role.id, + permission, + import_source_id: -1, + }); + } + } + } + + // Process database permissions + for (const roleName in databasePermissions) { + let role = roles.find(r => r.name === roleName); + if (!role) { + role = { + id: roles.length + 1, + name: roleName, + import_source_id: -1, + }; + roles.push(role); + } + + for (const permId in databasePermissions[roleName]) { + const perm = databasePermissions[roleName][permId]; + if (perm.connection && perm.permission) { + const dbId = connectionEnvIdToDbId[perm.connection]; + const permissionId = databasePermissionMap[perm.permission]; + if (dbId && permissionId) { + role_databases.push({ + role_id: role.id, + connection_id: dbId, + database_names_list: perm.databases || null, + database_names_regex: perm.databasesRegex || null, + database_permission_role_id: permissionId, + id_original: permId, + import_source_id: -1, + }); + } + } + } + } + + // Process table permissions + for (const roleName in tablePermissions) { + let role = roles.find(r => r.name === roleName); + if (!role) { + role = { + id: roles.length + 1, + name: roleName, + import_source_id: -1, + }; + roles.push(role); + } + + for (const permId in tablePermissions[roleName]) { + const perm = tablePermissions[roleName][permId]; + if (perm.connection && perm.permission) { + const dbId = connectionEnvIdToDbId[perm.connection]; + const permissionId = tablePermissionMap[perm.permission]; + const scopeId = tableScopeMap[perm.scope || 'all_objects']; + if (dbId && permissionId && scopeId) { + role_tables.push({ + role_id: role.id, + connection_id: dbId, + database_names_list: perm.databases || null, + database_names_regex: perm.databasesRegex || null, + schema_names_list: perm.schemas || null, + schema_names_regex: perm.schemasRegex || null, + table_names_list: perm.tables || null, + table_names_regex: perm.tablesRegex || null, + table_permission_role_id: permissionId, + table_permission_scope_id: scopeId, + id_original: permId, + import_source_id: -1, + }); + } + } + } + } + + if (connections.length == 0 && roles.length == 0) { + return null; + } + + return { + connections, + roles, + role_connections, + role_permissions, + role_databases, + role_tables, + }; +} + +function createStorageFromEnvReplicatorItems(importEntities) { + return [ + { + name: 'connections', + findExisting: true, + createNew: true, + updateExisting: true, + matchColumns: ['id_original', 'import_source_id'], + deleteMissing: true, + deleteRestrictionColumns: ['import_source_id'], + skipUpdateColumns: ['conid'], + jsonArray: importEntities.connections, + }, + { + name: 'roles', + findExisting: true, + createNew: true, + updateExisting: true, + matchColumns: ['name', 'import_source_id'], + deleteMissing: true, + deleteRestrictionColumns: ['import_source_id'], + jsonArray: importEntities.roles, + }, + { + name: 'role_connections', + findExisting: true, + createNew: true, + updateExisting: false, + deleteMissing: true, + matchColumns: ['role_id', 'connection_id', 'import_source_id'], + jsonArray: importEntities.role_connections, + deleteRestrictionColumns: ['import_source_id'], + }, + { + name: 'role_permissions', + findExisting: true, + createNew: true, + updateExisting: false, + deleteMissing: true, + matchColumns: ['role_id', 'permission', 'import_source_id'], + jsonArray: importEntities.role_permissions, + deleteRestrictionColumns: ['import_source_id'], + }, + { + name: 'role_databases', + findExisting: true, + createNew: true, + updateExisting: true, + deleteMissing: true, + matchColumns: ['role_id', 'id_original', 'import_source_id'], + jsonArray: importEntities.role_databases, + deleteRestrictionColumns: ['import_source_id'], + }, + { + name: 'role_tables', + findExisting: true, + createNew: true, + updateExisting: true, + deleteMissing: true, + matchColumns: ['role_id', 'id_original', 'import_source_id'], + jsonArray: importEntities.role_tables, + deleteRestrictionColumns: ['import_source_id'], + }, + ]; +} + +module.exports = { + extractConnectionsFromEnv, + extractImportEntitiesFromEnv, + createStorageFromEnvReplicatorItems, +}; diff --git a/packages/api/src/utility/getChartExport.js b/packages/api/src/utility/getChartExport.js index 53331c5a9..fae05958b 100644 --- a/packages/api/src/utility/getChartExport.js +++ b/packages/api/src/utility/getChartExport.js @@ -53,7 +53,7 @@ const getChartExport = (title, config, imageFile, plugins) => { diff --git a/packages/api/src/utility/getMapExport.js b/packages/api/src/utility/getMapExport.js index a905137e2..99b140a7d 100644 --- a/packages/api/src/utility/getMapExport.js +++ b/packages/api/src/utility/getMapExport.js @@ -18,7 +18,7 @@ const getMapExport = (geoJson) => { leaflet .tileLayer('https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png', { maxZoom: 19, - attribution: 'DbGate | © OpenStreetMap', + attribution: 'DbGate | © OpenStreetMap', }) .addTo(map); diff --git a/packages/api/src/utility/handleQueryStream.js b/packages/api/src/utility/handleQueryStream.js index 617991ea3..eb1d3a0fa 100644 --- a/packages/api/src/utility/handleQueryStream.js +++ b/packages/api/src/utility/handleQueryStream.js @@ -14,9 +14,10 @@ class QueryStreamTableWriter { this.currentChangeIndex = 1; this.initializedFile = false; this.sesid = sesid; + this.started = new Date().getTime(); } - initializeFromQuery(structure, resultIndex, chartDefinition, autoDetectCharts = false) { + initializeFromQuery(structure, resultIndex, chartDefinition, autoDetectCharts = false, options = {}) { this.jslid = crypto.randomUUID(); this.currentFile = path.join(jsldir(), `${this.jslid}.jsonl`); fs.writeFileSync( @@ -24,6 +25,7 @@ class QueryStreamTableWriter { JSON.stringify({ ...structure, __isStreamHeader: true, + ...options }) + '\n' ); this.currentStream = fs.createWriteStream(this.currentFile, { flags: 'a' }); @@ -118,6 +120,13 @@ class QueryStreamTableWriter { this.chartProcessor = null; } } + process.send({ + msgtype: 'endrecordset', + jslid: this.jslid, + rowCount: this.currentRowCount, + sesid: this.sesid, + durationMs: new Date().getTime() - this.started, + }); resolve(); }); } else { @@ -148,6 +157,7 @@ class StreamHandler { // this.error = this.error.bind(this); this.done = this.done.bind(this); this.info = this.info.bind(this); + this.changedCurrentDatabase = this.changedCurrentDatabase.bind(this); // use this for cancelling - not implemented // this.stream = null; @@ -166,7 +176,11 @@ class StreamHandler { } } - recordset(columns) { + changedCurrentDatabase(database) { + process.send({ msgtype: 'changedCurrentDatabase', database, sesid: this.sesid }); + } + + recordset(columns, options) { if (this.rowsLimitOverflow) { return; } @@ -176,7 +190,8 @@ class StreamHandler { Array.isArray(columns) ? { columns } : columns, this.queryStreamInfoHolder.resultIndex, this.frontMatter?.[`chart-${this.queryStreamInfoHolder.resultIndex + 1}`], - this.autoDetectCharts + this.autoDetectCharts, + options ); this.queryStreamInfoHolder.resultIndex += 1; this.rowCounter = 0; diff --git a/packages/api/src/utility/hardwareFingerprint.js b/packages/api/src/utility/hardwareFingerprint.js index c99d86967..b2d7e6646 100644 --- a/packages/api/src/utility/hardwareFingerprint.js +++ b/packages/api/src/utility/hardwareFingerprint.js @@ -3,18 +3,6 @@ const os = require('os'); const crypto = require('crypto'); const platformInfo = require('./platformInfo'); -async function getPublicIpInfo() { - try { - const resp = await axios.default.get('https://ipinfo.io/json'); - if (!resp.data?.ip) { - return { ip: 'unknown-ip' }; - } - return resp.data; - } catch (err) { - return { ip: 'unknown-ip' }; - } -} - function getMacAddress() { try { const interfaces = os.networkInterfaces(); @@ -32,6 +20,7 @@ function getMacAddress() { } async function getHardwareFingerprint() { + const { getPublicIpInfo } = require('./cloudIntf'); const publicIpInfo = await getPublicIpInfo(); const macAddress = getMacAddress(); const platform = os.platform(); @@ -42,8 +31,6 @@ async function getHardwareFingerprint() { return { publicIp: publicIpInfo.ip, country: publicIpInfo.country, - region: publicIpInfo.region, - city: publicIpInfo.city, macAddress, platform, release, @@ -68,9 +55,7 @@ async function getPublicHardwareFingerprint() { hash, payload: { platform: fingerprint.platform, - city: fingerprint.city, country: fingerprint.country, - region: fingerprint.region, isDocker: platformInfo.isDocker, isAwsUbuntuLayout: platformInfo.isAwsUbuntuLayout, isAzureUbuntuLayout: platformInfo.isAzureUbuntuLayout, @@ -87,5 +72,4 @@ module.exports = { getHardwareFingerprint, getHardwareFingerprintHash, getPublicHardwareFingerprint, - getPublicIpInfo, }; diff --git a/packages/api/src/utility/hasPermission.js b/packages/api/src/utility/hasPermission.js index 654ba8ceb..f964593b1 100644 --- a/packages/api/src/utility/hasPermission.js +++ b/packages/api/src/utility/hasPermission.js @@ -1,96 +1,350 @@ -const { compilePermissions, testPermission } = require('dbgate-tools'); +const { compilePermissions, testPermission, getPermissionsCacheKey } = require('dbgate-tools'); const _ = require('lodash'); const { getAuthProviderFromReq } = require('../auth/authProvider'); const cachedPermissions = {}; -function hasPermission(tested, req) { +async function loadPermissionsFromRequest(req) { + const authProvider = getAuthProviderFromReq(req); if (!req) { - // request object not available, allow all + return null; + } + + const loadedPermissions = await authProvider.getCurrentPermissions(req); + return loadedPermissions; +} + +function hasPermission(tested, loadedPermissions) { + if (!loadedPermissions) { + // not available, allow all return true; } - const permissions = getAuthProviderFromReq(req).getCurrentPermissions(req); - - if (!cachedPermissions[permissions]) { - cachedPermissions[permissions] = compilePermissions(permissions); + const permissionsKey = getPermissionsCacheKey(loadedPermissions); + if (!cachedPermissions[permissionsKey]) { + cachedPermissions[permissionsKey] = compilePermissions(loadedPermissions); } - return testPermission(tested, cachedPermissions[permissions]); - - // const { user } = (req && req.auth) || {}; - // const { login } = (process.env.OAUTH_PERMISSIONS && req && req.user) || {}; - // const key = user || login || ''; - // const logins = getLogins(); - - // if (!userPermissions[key]) { - // if (logins) { - // const login = logins.find(x => x.login == user); - // userPermissions[key] = compilePermissions(login ? login.permissions : null); - // } else { - // userPermissions[key] = compilePermissions(process.env.PERMISSIONS); - // } - // } - // return testPermission(tested, userPermissions[key]); + return testPermission(tested, cachedPermissions[permissionsKey]); } -// let loginsCache = null; -// let loginsLoaded = false; - -// function getLogins() { -// if (loginsLoaded) { -// return loginsCache; -// } - -// const res = []; -// if (process.env.LOGIN && process.env.PASSWORD) { -// res.push({ -// login: process.env.LOGIN, -// password: process.env.PASSWORD, -// permissions: process.env.PERMISSIONS, -// }); -// } -// if (process.env.LOGINS) { -// const logins = _.compact(process.env.LOGINS.split(',').map(x => x.trim())); -// for (const login of logins) { -// const password = process.env[`LOGIN_PASSWORD_${login}`]; -// const permissions = process.env[`LOGIN_PERMISSIONS_${login}`]; -// if (password) { -// res.push({ -// login, -// password, -// permissions, -// }); -// } -// } -// } else if (process.env.OAUTH_PERMISSIONS) { -// const login_permission_keys = Object.keys(process.env).filter(key => _.startsWith(key, 'LOGIN_PERMISSIONS_')); -// for (const permissions_key of login_permission_keys) { -// const login = permissions_key.replace('LOGIN_PERMISSIONS_', ''); -// const permissions = process.env[permissions_key]; -// userPermissions[login] = compilePermissions(permissions); -// } -// } - -// loginsCache = res.length > 0 ? res : null; -// loginsLoaded = true; -// return loginsCache; -// } - -function connectionHasPermission(connection, req) { +function connectionHasPermission(connection, loadedPermissions) { if (!connection) { return true; } if (_.isString(connection)) { - return hasPermission(`connections/${connection}`, req); + return hasPermission(`connections/${connection}`, loadedPermissions); } else { - return hasPermission(`connections/${connection._id}`, req); + return hasPermission(`connections/${connection._id}`, loadedPermissions); } } -function testConnectionPermission(connection, req) { - if (!connectionHasPermission(connection, req)) { - throw new Error('Connection permission not granted'); +async function testConnectionPermission(connection, req, loadedPermissions) { + if (!loadedPermissions) { + loadedPermissions = await loadPermissionsFromRequest(req); + } + if (process.env.STORAGE_DATABASE) { + if (hasPermission(`all-connections`, loadedPermissions)) { + return; + } + const conid = _.isString(connection) ? connection : connection?._id; + if (hasPermission('internal-storage', loadedPermissions) && conid == '__storage') { + return; + } + const authProvider = getAuthProviderFromReq(req); + if (!req) { + return; + } + if (!(await authProvider.checkCurrentConnectionPermission(req, conid))) { + throw new Error('DBGM-00263 Connection permission not granted'); + } + } else { + if (!connectionHasPermission(connection, loadedPermissions)) { + throw new Error('DBGM-00264 Connection permission not granted'); + } + } +} + +async function loadDatabasePermissionsFromRequest(req) { + const authProvider = getAuthProviderFromReq(req); + if (!req) { + return null; + } + + const databasePermissions = await authProvider.getCurrentDatabasePermissions(req); + return databasePermissions; +} + +async function loadTablePermissionsFromRequest(req) { + const authProvider = getAuthProviderFromReq(req); + if (!req) { + return null; + } + + const tablePermissions = await authProvider.getCurrentTablePermissions(req); + return tablePermissions; +} + +async function loadFilePermissionsFromRequest(req) { + const authProvider = getAuthProviderFromReq(req); + if (!req) { + return null; + } + + const filePermissions = await authProvider.getCurrentFilePermissions(req); + return filePermissions; +} + +function matchDatabasePermissionRow(conid, database, permissionRow) { + if (permissionRow.connection_id) { + if (conid != permissionRow.connection_id) { + return false; + } + } + if (permissionRow.database_names_list) { + const items = permissionRow.database_names_list.split('\n'); + if (!items.find(item => item.trim()?.toLowerCase() === database?.toLowerCase())) { + return false; + } + } + if (permissionRow.database_names_regex) { + const regex = new RegExp(permissionRow.database_names_regex, 'i'); + if (!regex.test(database)) { + return false; + } + } + return true; +} + +function matchTablePermissionRow(objectTypeField, schemaName, pureName, permissionRow) { + if (permissionRow.table_names_list) { + const items = permissionRow.table_names_list.split('\n'); + if (!items.find(item => item.trim()?.toLowerCase() === pureName?.toLowerCase())) { + return false; + } + } + if (permissionRow.table_names_regex) { + const regex = new RegExp(permissionRow.table_names_regex, 'i'); + if (!regex.test(pureName)) { + return false; + } + } + if (permissionRow.schema_names_list) { + const items = permissionRow.schema_names_list.split('\n'); + if (!items.find(item => item.trim()?.toLowerCase() === schemaName?.toLowerCase())) { + return false; + } + } + if (permissionRow.schema_names_regex) { + const regex = new RegExp(permissionRow.schema_names_regex, 'i'); + if (!regex.test(schemaName)) { + return false; + } + } + + return true; +} + +function matchFilePermissionRow(folder, file, permissionRow) { + if (permissionRow.folder_name) { + if (folder != permissionRow.folder_name) { + return false; + } + } + if (permissionRow.file_names_list) { + const items = permissionRow.file_names_list.split('\n'); + if (!items.find(item => item.trim()?.toLowerCase() === file?.toLowerCase())) { + return false; + } + } + if (permissionRow.file_names_regex) { + const regex = new RegExp(permissionRow.file_names_regex, 'i'); + if (!regex.test(file)) { + return false; + } + } + return true; +} + +const DATABASE_ROLE_ID_NAMES = { + '-1': 'view', + '-2': 'read_content', + '-3': 'write_data', + '-4': 'run_script', + '-5': 'deny', +}; + +const FILE_ROLE_ID_NAMES = { + '-1': 'allow', + '-2': 'deny', +}; + +function getDatabaseRoleLevelIndex(roleName) { + if (!roleName) { + return 6; + } + if (roleName == 'run_script') { + return 5; + } + if (roleName == 'write_data') { + return 4; + } + if (roleName == 'read_content') { + return 3; + } + if (roleName == 'view') { + return 2; + } + if (roleName == 'deny') { + return 1; + } + return 6; +} + +function getTablePermissionRoleLevelIndex(roleName) { + if (!roleName) { + return 6; + } + if (roleName == 'run_script') { + return 5; + } + if (roleName == 'create_update_delete') { + return 4; + } + if (roleName == 'update_only') { + return 3; + } + if (roleName == 'read') { + return 2; + } + if (roleName == 'deny') { + return 1; + } + return 6; +} + +function getDatabasePermissionRole(conid, database, loadedDatabasePermissions) { + let res = 'deny'; + for (const permissionRow of loadedDatabasePermissions) { + if (!matchDatabasePermissionRow(conid, database, permissionRow)) { + continue; + } + res = DATABASE_ROLE_ID_NAMES[permissionRow.database_permission_role_id]; + } + return res; +} + +function getFilePermissionRole(folder, file, loadedFilePermissions) { + let res = 'deny'; + for (const permissionRow of loadedFilePermissions) { + if (!matchFilePermissionRow(folder, file, permissionRow)) { + continue; + } + res = FILE_ROLE_ID_NAMES[permissionRow.file_permission_role_id]; + } + return res; +} + +const TABLE_ROLE_ID_NAMES = { + '-1': 'read', + '-2': 'update_only', + '-3': 'create_update_delete', + '-4': 'run_script', + '-5': 'deny', +}; + +const TABLE_SCOPE_ID_NAMES = { + '-1': 'all_objects', + '-2': 'tables', + '-3': 'views', + '-4': 'tables_views_collections', + '-5': 'procedures', + '-6': 'functions', + '-7': 'triggers', + '-8': 'sql_objects', + '-9': 'collections', +}; + +function getTablePermissionRole( + conid, + database, + objectTypeField, + schemaName, + pureName, + loadedTablePermissions, + databasePermissionRole +) { + let res = + databasePermissionRole == 'read_content' + ? 'read' + : databasePermissionRole == 'write_data' + ? 'create_update_delete' + : databasePermissionRole == 'run_script' + ? 'run_script' + : 'deny'; + for (const permissionRow of loadedTablePermissions) { + if (!matchDatabasePermissionRow(conid, database, permissionRow)) { + continue; + } + if (!matchTablePermissionRow(objectTypeField, schemaName, pureName, permissionRow)) { + continue; + } + const scope = TABLE_SCOPE_ID_NAMES[permissionRow.table_permission_scope_id]; + switch (scope) { + case 'tables': + if (objectTypeField != 'tables') continue; + break; + case 'views': + if (objectTypeField != 'views') continue; + break; + case 'tables_views_collections': + if (objectTypeField != 'tables' && objectTypeField != 'views' && objectTypeField != 'collections') continue; + break; + case 'procedures': + if (objectTypeField != 'procedures') continue; + break; + case 'functions': + if (objectTypeField != 'functions') continue; + break; + case 'triggers': + if (objectTypeField != 'triggers') continue; + break; + case 'sql_objects': + if (objectTypeField != 'procedures' && objectTypeField != 'functions' && objectTypeField != 'triggers') + continue; + break; + case 'collections': + if (objectTypeField != 'collections') continue; + break; + } + res = TABLE_ROLE_ID_NAMES[permissionRow.table_permission_role_id]; + } + return res; +} + +async function testStandardPermission(permission, req, loadedPermissions) { + if (!loadedPermissions) { + loadedPermissions = await loadPermissionsFromRequest(req); + } + if (!hasPermission(permission, loadedPermissions)) { + throw new Error(`DBGM-00265 Permission ${permission} not granted`); + } +} + +async function testDatabaseRolePermission(conid, database, requiredRole, req) { + if (!process.env.STORAGE_DATABASE) { + return; + } + const loadedPermissions = await loadPermissionsFromRequest(req); + if (hasPermission(`all-databases`, loadedPermissions)) { + return; + } + const databasePermissions = await loadDatabasePermissionsFromRequest(req); + const role = getDatabasePermissionRole(conid, database, databasePermissions); + const requiredIndex = getDatabaseRoleLevelIndex(requiredRole); + const roleIndex = getDatabaseRoleLevelIndex(role); + if (roleIndex < requiredIndex) { + throw new Error(`DBGM-00266 Permission ${requiredRole} not granted`); } } @@ -98,4 +352,14 @@ module.exports = { hasPermission, connectionHasPermission, testConnectionPermission, + loadPermissionsFromRequest, + loadDatabasePermissionsFromRequest, + loadTablePermissionsFromRequest, + loadFilePermissionsFromRequest, + getDatabasePermissionRole, + getTablePermissionRole, + getFilePermissionRole, + testStandardPermission, + testDatabaseRolePermission, + getTablePermissionRoleLevelIndex, }; diff --git a/packages/datalib/jest.config.js b/packages/datalib/jest.config.js index 790050941..38f6eab3c 100644 --- a/packages/datalib/jest.config.js +++ b/packages/datalib/jest.config.js @@ -2,4 +2,5 @@ module.exports = { preset: 'ts-jest', testEnvironment: 'node', moduleFileExtensions: ['js'], + reporters: ['default', 'github-actions'], }; diff --git a/packages/datalib/package.json b/packages/datalib/package.json index b82b4585a..12b52917e 100644 --- a/packages/datalib/package.json +++ b/packages/datalib/package.json @@ -3,6 +3,10 @@ "name": "dbgate-datalib", "main": "lib/index.js", "typings": "lib/index.d.ts", + "repository": { + "type": "git", + "url": "https://github.com/dbgate/dbgate.git" + }, "scripts": { "build": "tsc", "test": "jest", diff --git a/packages/datalib/src/DataReplicator.ts b/packages/datalib/src/DataReplicator.ts index 668aee480..8b93e31ec 100644 --- a/packages/datalib/src/DataReplicator.ts +++ b/packages/datalib/src/DataReplicator.ts @@ -23,6 +23,7 @@ export interface DataReplicatorItem { deleteMissing: boolean; deleteRestrictionColumns: string[]; matchColumns: string[]; + skipUpdateColumns?: string[]; } export interface DataReplicatorOptions { @@ -151,7 +152,12 @@ class ReplicatorItemHolder { chunk, this.table.columns.map(x => x.columnName) ), - [this.autoColumn, ...this.backReferences.map(x => x.columnName), ...this.references.map(x => x.columnName)] + [ + this.autoColumn, + ...this.backReferences.map(x => x.columnName), + ...this.references.map(x => x.columnName), + ...(this.item.skipUpdateColumns || []), + ] ); return res; diff --git a/packages/datalib/src/GridConfig.ts b/packages/datalib/src/GridConfig.ts index 650e88217..647b2b307 100644 --- a/packages/datalib/src/GridConfig.ts +++ b/packages/datalib/src/GridConfig.ts @@ -31,6 +31,8 @@ export interface GridConfig extends GridConfigColumns { formFilterColumns: string[]; multiColumnFilter?: string; searchInColumns?: string; + disabledFilterColumns: string[]; + disabledMultiColumnFilter?: boolean; } export interface GridCache { @@ -48,6 +50,7 @@ export function createGridConfig(): GridConfig { focusedColumns: null, grouping: {}, formFilterColumns: [], + disabledFilterColumns: [], }; } diff --git a/packages/datalib/src/GridDisplay.ts b/packages/datalib/src/GridDisplay.ts index 3a2d24d4a..1386efaf5 100644 --- a/packages/datalib/src/GridDisplay.ts +++ b/packages/datalib/src/GridDisplay.ts @@ -13,7 +13,7 @@ import type { FilterBehaviour, } from 'dbgate-types'; import { parseFilter } from 'dbgate-filterparser'; -import { filterName } from 'dbgate-tools'; +import { filterName, shortenIdentifier } from 'dbgate-tools'; import { ChangeSetFieldDefinition, ChangeSetRowDefinition } from './ChangeSet'; import { Expression, Select, treeToSql, dumpSqlSelect, Condition, CompoudCondition } from 'dbgate-sqltree'; import { isTypeLogical, standardFilterBehaviours, detectSqlFilterBehaviour, stringFilterBehaviour } from 'dbgate-tools'; @@ -24,6 +24,7 @@ export interface DisplayColumn { columnName: string; headerText: string; uniqueName: string; + uniqueNameShorten?: string; uniquePath: string[]; notNull?: boolean; autoIncrement?: boolean; @@ -232,6 +233,7 @@ export abstract class GridDisplay { if (!filter) continue; const column = displayedColumnInfo[uniqueName]; if (!column) continue; + if (this.isFilterDisabled(uniqueName)) continue; try { const condition = parseFilter( filter, @@ -258,7 +260,7 @@ export abstract class GridDisplay { } } - if (this.baseTableOrView && this.config.multiColumnFilter) { + if (this.baseTableOrView && this.config.multiColumnFilter && !this.isMultiColumnFilterDisabled()) { const orCondition: CompoudCondition = { conditionType: 'or', conditions: [], @@ -415,6 +417,7 @@ export abstract class GridDisplay { [uniqueName]: value, }, formViewRecordNumber: 0, + disabledFilterColumns: cfg.disabledFilterColumns.filter(x => x != uniqueName), })); this.reload(); } @@ -424,6 +427,7 @@ export abstract class GridDisplay { ...cfg, multiColumnFilter: value, formViewRecordNumber: 0, + disabledMultiColumnFilter: false, })); this.reload(); } @@ -447,6 +451,7 @@ export abstract class GridDisplay { ...cfg, filters: _.omit(cfg.filters, [uniqueName]), formFilterColumns: (cfg.formFilterColumns || []).filter(x => x != uniqueName), + disabledFilterColumns: (cfg.disabledFilterColumns).filter(x => x != uniqueName), })); this.reload(); } @@ -462,6 +467,37 @@ export abstract class GridDisplay { this.reload(); } + toggleFilterEnabled(uniqueName) { + if (this.isFilterDisabled(uniqueName)) { + this.setConfig(cfg => ({ + ...cfg, + disabledFilterColumns: cfg.disabledFilterColumns.filter(x => x != uniqueName), + })); + } else { + this.setConfig(cfg => ({ + ...cfg, + disabledFilterColumns: [...cfg.disabledFilterColumns, uniqueName], + })); + } + this.reload(); + } + + isFilterDisabled(uniqueName: string) { + return this.config.disabledFilterColumns.includes(uniqueName); + } + + toggleMultiColumnFilterEnabled() { + this.setConfig(cfg => ({ + ...cfg, + disabledMultiColumnFilter: !cfg.disabledMultiColumnFilter, + })); + this.reload(); + } + + isMultiColumnFilterDisabled() { + return this.config.disabledMultiColumnFilter; + } + setSort(uniqueName, order) { this.setConfig(cfg => ({ ...cfg, @@ -606,7 +642,9 @@ export abstract class GridDisplay { } return { exprType: 'column', - ...(!this.dialect.omitTableAliases && { alias: alias || col.columnName }), + ...(!this.dialect.omitTableAliases && { + alias: alias ?? col.columnName, + }), source, ...col, }; diff --git a/packages/datalib/src/TableGridDisplay.ts b/packages/datalib/src/TableGridDisplay.ts index dc4f3588d..1453104a3 100644 --- a/packages/datalib/src/TableGridDisplay.ts +++ b/packages/datalib/src/TableGridDisplay.ts @@ -1,5 +1,5 @@ import _ from 'lodash'; -import { filterName, isTableColumnUnique } from 'dbgate-tools'; +import { filterName, isTableColumnUnique, shortenIdentifier } from 'dbgate-tools'; import { GridDisplay, ChangeCacheFunc, DisplayColumn, DisplayedColumnInfo, ChangeConfigFunc } from './GridDisplay'; import type { TableInfo, @@ -39,7 +39,8 @@ export class TableGridDisplay extends GridDisplay { public getDictionaryDescription: DictionaryDescriptionFunc = null, isReadOnly = false, public isRawMode = false, - public currentSettings = null + public currentSettings = null, + public areReferencesAllowed = true ) { super(config, setConfig, cache, setCache, driver, dbinfo, serverVersion, currentSettings); @@ -93,7 +94,7 @@ export class TableGridDisplay extends GridDisplay { ); } - getDisplayColumns(table: TableInfo, parentPath: string[]) { + getDisplayColumns(table: TableInfo, parentPath: string[]): DisplayColumn[] { return ( table?.columns ?.map(col => this.getDisplayColumn(table, col, parentPath)) @@ -101,11 +102,12 @@ export class TableGridDisplay extends GridDisplay { ...col, isChecked: this.isColumnChecked(col), hintColumnNames: - this.getFkDictionaryDescription(col.isForeignKeyUnique ? col.foreignKey : null)?.columns?.map( - columnName => `hint_${col.uniqueName}_${columnName}` + this.getFkDictionaryDescription(col.isForeignKeyUnique ? col.foreignKey : null)?.columns?.map(columnName => + shortenIdentifier(`hint_${col.uniqueName}_${columnName}`, this.driver?.dialect?.maxIdentifierLength) ) || null, hintColumnDelimiter: this.getFkDictionaryDescription(col.isForeignKeyUnique ? col.foreignKey : null) ?.delimiter, + uniqueNameShorten: shortenIdentifier(col.uniqueName, this.driver?.dialect?.maxIdentifierLength), isExpandable: !!col.foreignKey, })) || [] ); @@ -116,7 +118,7 @@ export class TableGridDisplay extends GridDisplay { if (this.isExpandedColumn(column.uniqueName)) { const table = this.getFkTarget(column); if (table) { - const childAlias = `${column.uniqueName}_ref`; + const childAlias = shortenIdentifier(`${column.uniqueName}_ref`, this.driver?.dialect?.maxIdentifierLength); const subcolumns = this.getDisplayColumns(table, column.uniquePath); this.addReferenceToSelect(select, parentAlias, column); @@ -129,7 +131,7 @@ export class TableGridDisplay extends GridDisplay { } addReferenceToSelect(select: Select, parentAlias: string, column: DisplayColumn) { - const childAlias = `${column.uniqueName}_ref`; + const childAlias = shortenIdentifier(`${column.uniqueName}_ref`, this.driver?.dialect?.maxIdentifierLength); if ((select.from.relations || []).find(x => x.alias == childAlias)) return; const table = this.getFkTarget(column); if (table && table.primaryKey) { @@ -191,15 +193,24 @@ export class TableGridDisplay extends GridDisplay { const hintDescription = this.getDictionaryDescription(table); if (hintDescription) { const parentUniqueName = column.uniquePath.slice(0, -1).join('.'); - this.addReferenceToSelect(select, parentUniqueName ? `${parentUniqueName}_ref` : 'basetbl', column); - const childAlias = `${column.uniqueName}_ref`; + this.addReferenceToSelect( + select, + parentUniqueName + ? shortenIdentifier(`${parentUniqueName}_ref`, this.driver?.dialect?.maxIdentifierLength) + : 'basetbl', + column + ); + const childAlias = shortenIdentifier(`${column.uniqueName}_ref`, this.driver?.dialect?.maxIdentifierLength); select.columns.push( ...hintDescription.columns.map( columnName => ({ exprType: 'column', columnName, - alias: `hint_${column.uniqueName}_${columnName}`, + alias: shortenIdentifier( + `hint_${column.uniqueName}_${columnName}`, + this.driver?.dialect?.maxIdentifierLength + ), source: { alias: childAlias }, } as ColumnRefExpression) ) @@ -230,7 +241,7 @@ export class TableGridDisplay extends GridDisplay { } getFkTarget(column: DisplayColumn) { - const { uniqueName, foreignKey, isForeignKeyUnique } = column; + const { foreignKey, isForeignKeyUnique } = column; if (!isForeignKeyUnique) return null; const pureName = foreignKey.refTableName; const schemaName = foreignKey.refSchemaName; @@ -238,6 +249,7 @@ export class TableGridDisplay extends GridDisplay { } processReferences(select: Select, displayedColumnInfo: DisplayedColumnInfo, options) { + if (!this.areReferencesAllowed) return; this.addJoinsFromExpandedColumns(select, this.columns, 'basetbl', displayedColumnInfo); if (!options.isExport && this.displayOptions.showHintColumns) { this.addHintsToSelect(select); @@ -298,7 +310,12 @@ export class TableGridDisplay extends GridDisplay { for (const column of columns) { if (this.addAllExpandedColumnsToSelected || this.config.addedColumns.includes(column.uniqueName)) { select.columns.push( - this.createColumnExpression(column, { name: column, alias: parentAlias }, column.uniqueName, 'view') + this.createColumnExpression( + column, + { name: column, alias: parentAlias }, + column.uniqueNameShorten ?? column.uniqueName, + 'view' + ) ); displayedColumnInfo[column.uniqueName] = { ...column, diff --git a/packages/datalib/src/chartDefinitions.ts b/packages/datalib/src/chartDefinitions.ts index d335c15cd..b0acf608e 100644 --- a/packages/datalib/src/chartDefinitions.ts +++ b/packages/datalib/src/chartDefinitions.ts @@ -4,6 +4,7 @@ export type ChartXTransformFunction = | 'date:minute' | 'date:hour' | 'date:day' + | 'date:week' | 'date:month' | 'date:year'; export type ChartYAggregateFunction = 'sum' | 'first' | 'last' | 'min' | 'max' | 'count' | 'avg'; @@ -70,6 +71,7 @@ export interface ChartDateParsed { minute?: number; second?: number; fraction?: string; + week?: number; } export interface ChartAvailableColumn { diff --git a/packages/datalib/src/chartTools.ts b/packages/datalib/src/chartTools.ts index 15c2cf469..9856d0a12 100644 --- a/packages/datalib/src/chartTools.ts +++ b/packages/datalib/src/chartTools.ts @@ -9,7 +9,7 @@ import { ChartYFieldDefinition, ProcessedChart, } from './chartDefinitions'; -import { addMinutes, addHours, addDays, addMonths, addYears } from 'date-fns'; +import { addMinutes, addHours, addDays, addMonths, addWeeks, addYears, getWeek } from 'date-fns'; export function getChartDebugPrint(chart: ProcessedChart) { let res = ''; @@ -29,6 +29,7 @@ export function tryParseChartDate(dateInput: any): ChartDateParsed | null { return { year: dateInput.getFullYear(), month: dateInput.getMonth() + 1, + week: getWeek(dateInput), day: dateInput.getDate(), hour: dateInput.getHours(), minute: dateInput.getMinutes(), @@ -42,15 +43,21 @@ export function tryParseChartDate(dateInput: any): ChartDateParsed | null { /^(\d{4})-(\d{2})-(\d{2})(?:[ T](\d{2}):(\d{2}):(\d{2})(?:\.(\d+))?(Z|[+-]\d{2}:\d{2})?)?$/ ); const monthMatch = dateInput.match(/^(\d{4})-(\d{2})$/); + const weekMatch = dateInput.match(/^(\d{4})\@(\d{2})$/); // const yearMatch = dateInput.match(/^(\d{4})$/); if (dateMatch) { - const [_notUsed, year, month, day, hour, minute, second, fraction] = dateMatch; + const [_notUsed, yearStr, monthStr, dayStr, hour, minute, second, fraction] = dateMatch; + + const year = parseInt(yearStr, 10); + const month = parseInt(monthStr, 10); + const day = parseInt(dayStr, 10); return { - year: parseInt(year, 10), - month: parseInt(month, 10), - day: parseInt(day, 10), + year, + month, + week: getWeek(new Date(year, month - 1, day)), + day, hour: parseInt(hour, 10) || 0, minute: parseInt(minute, 10) || 0, second: parseInt(second, 10) || 0, @@ -71,6 +78,19 @@ export function tryParseChartDate(dateInput: any): ChartDateParsed | null { }; } + if (weekMatch) { + const [_notUsed, year, week] = weekMatch; + return { + year: parseInt(year, 10), + week: parseInt(week, 10), + day: 1, + hour: 0, + minute: 0, + second: 0, + fraction: undefined, + }; + } + // if (yearMatch) { // const [_notUsed, year] = yearMatch; // return { @@ -97,6 +117,8 @@ export function stringifyChartDate(value: ChartDateParsed, transform: ChartXTran return `${value.year}`; case 'date:month': return `${value.year}-${pad2Digits(value.month)}`; + case 'date:week': + return `${value.year}@${pad2Digits(getWeek(new Date(value.year, (value.month ?? 1) - 1, value.day ?? 1)))}`; case 'date:day': return `${value.year}-${pad2Digits(value.month)}-${pad2Digits(value.day)}`; case 'date:hour': @@ -126,6 +148,9 @@ export function incrementChartDate(value: ChartDateParsed, transform: ChartXTran case 'date:month': newDateRepresentation = addMonths(dateRepresentation, 1); break; + case 'date:week': + newDateRepresentation = addWeeks(dateRepresentation, 1); + break; case 'date:day': newDateRepresentation = addDays(dateRepresentation, 1); break; @@ -144,6 +169,11 @@ export function incrementChartDate(value: ChartDateParsed, transform: ChartXTran year: newDateRepresentation.getFullYear(), month: newDateRepresentation.getMonth() + 1, }; + case 'date:week': + return { + year: newDateRepresentation.getFullYear(), + week: getWeek(newDateRepresentation), + }; case 'date:day': return { year: newDateRepresentation.getFullYear(), @@ -175,6 +205,8 @@ export function runTransformFunction(value: string, transformFunction: ChartXTra return dateParsed ? `${dateParsed.year}` : null; case 'date:month': return dateParsed ? `${dateParsed.year}-${pad2Digits(dateParsed.month)}` : null; + case 'date:week': + return dateParsed ? `${dateParsed.year}@${pad2Digits(dateParsed.week)}` : null; case 'date:day': return dateParsed ? `${dateParsed.year}-${pad2Digits(dateParsed.month)}-${pad2Digits(dateParsed.day)}` : null; case 'date:hour': @@ -211,6 +243,14 @@ export function computeChartBucketKey( month: dateParsed.month, }, ]; + case 'date:week': + return [ + dateParsed ? `${dateParsed.year}@${pad2Digits(dateParsed.week)}` : null, + { + year: dateParsed.year, + week: dateParsed.week, + }, + ]; case 'date:day': return [ dateParsed ? `${dateParsed.year}-${pad2Digits(dateParsed.month)}-${pad2Digits(dateParsed.day)}` : null, @@ -265,6 +305,8 @@ export function computeDateBucketDistance( return end.year - begin.year; case 'date:month': return (end.year - begin.year) * 12 + (end.month - begin.month); + case 'date:week': + return (end.year - begin.year) * 52 + (end.week - begin.week); case 'date:day': return ( (end.year - begin.year) * 365 + @@ -302,6 +344,8 @@ export function compareChartDatesParsed( return a.year - b.year; case 'date:month': return a.year === b.year ? a.month - b.month : a.year - b.year; + case 'date:week': + return a.year === b.year ? a.week - b.week : a.year - b.year; case 'date:day': return a.year === b.year && a.month === b.month ? a.day - b.day @@ -356,6 +400,8 @@ function getParentDateBucketKey( return null; // no parent for year case 'date:month': return bucketKey.slice(0, 4); + case 'date:week': + return bucketKey.slice(0, 4); case 'date:day': return bucketKey.slice(0, 7); case 'date:hour': @@ -371,6 +417,8 @@ function getParentDateBucketTransform(transform: ChartXTransformFunction): Chart return null; // no parent for year case 'date:month': return 'date:year'; + case 'date:week': + return 'date:year'; case 'date:day': return 'date:month'; case 'date:hour': @@ -388,6 +436,8 @@ function getParentKeyParsed(date: ChartDateParsed, transform: ChartXTransformFun return null; // no parent for year case 'date:month': return { year: date.year }; + case 'date:week': + return { year: date.week }; case 'date:day': return { year: date.year, month: date.month }; case 'date:hour': diff --git a/packages/dbmodel/bin/dbmodel.js b/packages/dbmodel/bin/dbmodel.js index 596be53db..54ea49339 100755 --- a/packages/dbmodel/bin/dbmodel.js +++ b/packages/dbmodel/bin/dbmodel.js @@ -35,6 +35,12 @@ program .option('-u, --user ', 'user name') .option('-p, --password ', 'password') .option('-d, --database ', 'database name') + .option('--url ', 'database url') + .option('--file ', 'database file') + .option('--socket-path ', 'socket path') + .option('--service-name ', 'service name (for Oracle)') + .option('--auth-type ', 'authentication type') + .option('--use-ssl', 'use SSL connection') .option('--auto-index-foreign-keys', 'automatically adds indexes to all foreign keys') .option( '--load-data-condition ', @@ -48,7 +54,7 @@ program .command('deploy ') .description('Deploys model to database') .action(modelFolder => { - const { engine, server, user, password, database, transaction } = program.opts(); + const { engine, server, user, password, database, url, file, transaction } = program.opts(); // const hooks = []; // if (program.autoIndexForeignKeys) hooks.push(dbmodel.hooks.autoIndexForeignKeys); @@ -60,6 +66,13 @@ program user, password, database, + databaseUrl: url, + useDatabaseUrl: !!url, + databaseFile: file, + socketPath: program.socketPath, + serviceName: program.serviceName, + authType: program.authType, + useSsl: program.useSsl, }, modelFolder, useTransaction: transaction, diff --git a/packages/filterparser/jest.config.js b/packages/filterparser/jest.config.js index 790050941..38f6eab3c 100644 --- a/packages/filterparser/jest.config.js +++ b/packages/filterparser/jest.config.js @@ -2,4 +2,5 @@ module.exports = { preset: 'ts-jest', testEnvironment: 'node', moduleFileExtensions: ['js'], + reporters: ['default', 'github-actions'], }; diff --git a/packages/filterparser/package.json b/packages/filterparser/package.json index 1f3d32570..94dd86ef2 100644 --- a/packages/filterparser/package.json +++ b/packages/filterparser/package.json @@ -3,6 +3,10 @@ "name": "dbgate-filterparser", "main": "lib/index.js", "typings": "lib/index.d.ts", + "repository": { + "type": "git", + "url": "https://github.com/dbgate/dbgate.git" + }, "scripts": { "build": "tsc", "start": "tsc --watch", diff --git a/packages/filterparser/src/filterTool.ts b/packages/filterparser/src/filterTool.ts index 864e16021..3455262df 100644 --- a/packages/filterparser/src/filterTool.ts +++ b/packages/filterparser/src/filterTool.ts @@ -1,4 +1,4 @@ -import { arrayToHexString, evalFilterBehaviour, isTypeDateTime } from 'dbgate-tools'; +import { arrayToHexString, base64ToHex, evalFilterBehaviour, isTypeDateTime } from 'dbgate-tools'; import { format, toDate } from 'date-fns'; import _isString from 'lodash/isString'; import _cloneDeepWith from 'lodash/cloneDeepWith'; @@ -21,10 +21,13 @@ export function getFilterValueExpression(value, dataType?) { if (value === false) return 'FALSE'; if (value.$oid) return `ObjectId("${value.$oid}")`; if (value.$bigint) return value.$bigint; + if (value.$decimal) return value.$decimal; if (value.type == 'Buffer' && Array.isArray(value.data)) { return '0x' + arrayToHexString(value.data); } - + if (value?.$binary?.base64) { + return base64ToHex(value.$binary.base64); + } return `="${value}"`; } diff --git a/packages/filterparser/src/parseFilter.ts b/packages/filterparser/src/parseFilter.ts index 8b24c6cc9..60fb58071 100644 --- a/packages/filterparser/src/parseFilter.ts +++ b/packages/filterparser/src/parseFilter.ts @@ -2,7 +2,7 @@ import P from 'parsimmon'; import moment from 'moment'; import { Condition } from 'dbgate-sqltree'; import { interpretEscapes, token, word, whitespace } from './common'; -import { hexStringToArray, parseNumberSafe } from 'dbgate-tools'; +import { hexToBase64, parseNumberSafe } from 'dbgate-tools'; import { FilterBehaviour, TransformType } from 'dbgate-types'; const binaryCondition = @@ -385,10 +385,7 @@ const createParser = (filterBehaviour: FilterBehaviour) => { hexstring: () => token(P.regexp(/0x(([0-9a-fA-F][0-9a-fA-F])+)/, 1)) - .map(x => ({ - type: 'Buffer', - data: hexStringToArray(x), - })) + .map(x => ({ $binary: { base64: hexToBase64(x) } })) .desc('hex string'), noQuotedString: () => P.regexp(/[^\s^,^'^"]+/).desc('string unquoted'), diff --git a/packages/sqltree/src/dumpSqlCommand.ts b/packages/sqltree/src/dumpSqlCommand.ts index e8286bafb..b3b9373a4 100644 --- a/packages/sqltree/src/dumpSqlCommand.ts +++ b/packages/sqltree/src/dumpSqlCommand.ts @@ -1,7 +1,7 @@ import type { SqlDumper } from 'dbgate-types'; import { Command, Select, Update, Delete, Insert } from './types'; import { dumpSqlExpression } from './dumpSqlExpression'; -import { dumpSqlFromDefinition, dumpSqlSourceRef } from './dumpSqlSource'; +import { dumpSqlFromDefinition, dumpSqlSourceDef, dumpSqlSourceRef } from './dumpSqlSource'; import { dumpSqlCondition } from './dumpSqlCondition'; export function dumpSqlSelect(dmp: SqlDumper, cmd: Select) { @@ -115,7 +115,10 @@ export function dumpSqlInsert(dmp: SqlDumper, cmd: Insert) { cmd.fields.map(x => x.targetColumn) ); dmp.putCollection(',', cmd.fields, x => dumpSqlExpression(dmp, x)); - if (dmp.dialect.requireFromDual) { + if (cmd.whereNotExistsSource) { + dmp.put(' ^from '); + dumpSqlSourceDef(dmp, cmd.whereNotExistsSource); + } else if (dmp.dialect.requireFromDual) { dmp.put(' ^from ^dual '); } dmp.put(' ^where ^not ^exists (^select * ^from %f ^where ', cmd.targetTable); diff --git a/packages/sqltree/src/dumpSqlExpression.ts b/packages/sqltree/src/dumpSqlExpression.ts index aeaf3b288..f9447f792 100644 --- a/packages/sqltree/src/dumpSqlExpression.ts +++ b/packages/sqltree/src/dumpSqlExpression.ts @@ -2,6 +2,7 @@ import _ from 'lodash'; import type { SqlDumper } from 'dbgate-types'; import { Expression, ColumnRefExpression } from './types'; import { dumpSqlSourceRef } from './dumpSqlSource'; +import { dumpSqlSelect } from './dumpSqlCommand'; export function dumpSqlExpression(dmp: SqlDumper, expr: Expression) { switch (expr.exprType) { @@ -67,5 +68,11 @@ export function dumpSqlExpression(dmp: SqlDumper, expr: Expression) { }); dmp.put(')'); break; + + case 'select': + dmp.put('('); + dumpSqlSelect(dmp, expr.select); + dmp.put(')'); + break; } } diff --git a/packages/sqltree/src/evaluateCondition.ts b/packages/sqltree/src/evaluateCondition.ts index ec5bbd2f7..f15be8053 100644 --- a/packages/sqltree/src/evaluateCondition.ts +++ b/packages/sqltree/src/evaluateCondition.ts @@ -19,6 +19,7 @@ function isLike(value, test) { function extractRawValue(value) { if (value?.$bigint) return value.$bigint; if (value?.$oid) return value.$oid; + if (value?.$decimal) return value.$decimal; return value; } diff --git a/packages/sqltree/src/types.ts b/packages/sqltree/src/types.ts index bce0e61d3..ea7c5c994 100644 --- a/packages/sqltree/src/types.ts +++ b/packages/sqltree/src/types.ts @@ -44,6 +44,7 @@ export interface Insert { fields: UpdateField[]; targetTable: NamedObjectInfo; insertWhereNotExistsCondition?: Condition; + whereNotExistsSource?: Source; } export interface AllowIdentityInsert { @@ -226,6 +227,11 @@ export interface RowNumberExpression { orderBy: OrderByExpression[]; } +export interface SelectExpression { + exprType: 'select'; + select: Select; +} + export type Expression = | ColumnRefExpression | ValueExpression @@ -235,7 +241,8 @@ export type Expression = | CallExpression | MethodCallExpression | TranformExpression - | RowNumberExpression; + | RowNumberExpression + | SelectExpression; export type OrderByExpression = Expression & { direction: 'ASC' | 'DESC' }; export type ResultField = Expression & { alias?: string }; diff --git a/packages/tools/jest.config.js b/packages/tools/jest.config.js index 790050941..38f6eab3c 100644 --- a/packages/tools/jest.config.js +++ b/packages/tools/jest.config.js @@ -2,4 +2,5 @@ module.exports = { preset: 'ts-jest', testEnvironment: 'node', moduleFileExtensions: ['js'], + reporters: ['default', 'github-actions'], }; diff --git a/packages/tools/package.json b/packages/tools/package.json index f8f571d72..5478edfe7 100644 --- a/packages/tools/package.json +++ b/packages/tools/package.json @@ -32,7 +32,8 @@ "typescript": "^4.4.3" }, "dependencies": { - "dbgate-query-splitter": "^4.11.5", + "blueimp-md5": "^2.19.0", + "dbgate-query-splitter": "^4.11.9", "dbgate-sqltree": "^6.0.0-alpha.1", "debug": "^4.3.4", "json-stable-stringify": "^1.0.1", diff --git a/packages/tools/src/DatabaseAnalyser.ts b/packages/tools/src/DatabaseAnalyser.ts index 3d938fe11..f394cc939 100644 --- a/packages/tools/src/DatabaseAnalyser.ts +++ b/packages/tools/src/DatabaseAnalyser.ts @@ -49,6 +49,8 @@ export class DatabaseAnalyser { singleObjectId: string = null; dialect: SqlDialect; logger: Logger; + startedTm = Date.now(); + analyseIdentifier = Math.random().toString().substring(2); constructor(public dbhan: DatabaseHandle, public driver: EngineDriver, version) { this.dialect = (driver?.dialectByVersion && driver?.dialectByVersion(version)) || driver?.dialect; @@ -78,14 +80,24 @@ export class DatabaseAnalyser { } getLogDbInfo() { - return this.driver.getLogDbInfo(this.dbhan); + return { + ...this.driver.getLogDbInfo(this.dbhan), + analyserTime: Date.now() - this.startedTm, + analyseIdentifier: this.analyseIdentifier, + }; } async fullAnalysis() { logger.debug(this.getLogDbInfo(), 'DBGM-00126 Performing full analysis'); - const res = this.addEngineField(await this._runAnalysis()); + try { + const res = this.addEngineField(await this._runAnalysis()); + logger.debug(this.getLogDbInfo(), 'DBGM-00271 Full analysis finished successfully'); + return res; + } catch (err) { + logger.error(extractErrorLogData(err, this.getLogDbInfo()), 'DBGM-00272 Error during full analysis'); + throw err; + } // console.log('FULL ANALYSIS', res); - return res; } async singleObjectAnalysis(name, typeField) { @@ -106,31 +118,40 @@ export class DatabaseAnalyser { logger.info(this.getLogDbInfo(), 'DBGM-00127 Performing incremental analysis'); this.structure = structure; - const modifications = await this.getModifications(); - if (modifications == null) { - // modifications not implemented, perform full analysis - this.structure = null; - return this.addEngineField(await this._runAnalysis()); - } - const structureModifications = modifications.filter(x => x.action != 'setTableRowCounts'); - const setTableRowCounts = modifications.find(x => x.action == 'setTableRowCounts'); - - let structureWithRowCounts = null; - if (setTableRowCounts) { - const newStructure = mergeTableRowCounts(structure, setTableRowCounts.rowCounts); - if (areDifferentRowCounts(structure, newStructure)) { - structureWithRowCounts = newStructure; + try { + const modifications = await this.getModifications(); + if (modifications == null) { + // modifications not implemented, perform full analysis + this.structure = null; + return this.addEngineField(await this._runAnalysis()); } - } + const structureModifications = modifications.filter(x => x.action != 'setTableRowCounts'); + const setTableRowCounts = modifications.find(x => x.action == 'setTableRowCounts'); - if (structureModifications.length == 0) { - return structureWithRowCounts ? this.addEngineField(structureWithRowCounts) : null; - } + let structureWithRowCounts = null; + if (setTableRowCounts) { + const newStructure = mergeTableRowCounts(structure, setTableRowCounts.rowCounts); + if (areDifferentRowCounts(structure, newStructure)) { + structureWithRowCounts = newStructure; + } + } - this.modifications = structureModifications; - if (structureWithRowCounts) this.structure = structureWithRowCounts; - logger.info({ ...this.getLogDbInfo(), modifications: this.modifications }, 'DBGM-00128 DB modifications detected'); - return this.addEngineField(this.mergeAnalyseResult(await this._runAnalysis())); + if (structureModifications.length == 0) { + logger.debug(this.getLogDbInfo(), 'DBGM-00267 No changes in database structure detected'); + return structureWithRowCounts ? this.addEngineField(structureWithRowCounts) : null; + } + + this.modifications = structureModifications; + if (structureWithRowCounts) this.structure = structureWithRowCounts; + logger.info( + { ...this.getLogDbInfo(), modifications: this.modifications }, + 'DBGM-00128 DB modifications detected' + ); + return this.addEngineField(this.mergeAnalyseResult(await this._runAnalysis())); + } catch (err) { + logger.error(extractErrorLogData(err, this.getLogDbInfo()), 'DBGM-00273 Error during incremental analysis'); + throw err; + } } mergeAnalyseResult(newlyAnalysed) { @@ -143,6 +164,11 @@ export class DatabaseAnalyser { const res = {}; for (const field of STRUCTURE_FIELDS) { + const isAll = this.modifications.some(x => x.action == 'all' && x.objectTypeField == field); + if (isAll) { + res[field] = newlyAnalysed[field] || []; + continue; + } const removedIds = this.modifications .filter(x => x.action == 'remove' && x.objectTypeField == field) .map(x => x.objectId); diff --git a/packages/tools/src/SqlDumper.ts b/packages/tools/src/SqlDumper.ts index ac347857b..b71d97830 100644 --- a/packages/tools/src/SqlDumper.ts +++ b/packages/tools/src/SqlDumper.ts @@ -26,6 +26,7 @@ import _isDate from 'lodash/isDate'; import _isArray from 'lodash/isArray'; import _isPlainObject from 'lodash/isPlainObject'; import _keys from 'lodash/keys'; +import _cloneDeep from 'lodash/cloneDeep'; import uuidv1 from 'uuid/v1'; export class SqlDumper implements AlterProcessor { @@ -78,7 +79,16 @@ export class SqlDumper implements AlterProcessor { else if (_isNumber(value)) this.putRaw(value.toString()); else if (_isDate(value)) this.putStringValue(new Date(value).toISOString()); else if (value?.type == 'Buffer' && _isArray(value?.data)) this.putByteArrayValue(value?.data); + else if (value?.$binary?.base64) { + const binary = atob(value.$binary.base64); + const bytes = new Array(binary.length); + for (let i = 0; i < binary.length; i++) { + bytes[i] = binary.charCodeAt(i); + } + this.putByteArrayValue(bytes); + } else if (value?.$bigint) this.putRaw(value?.$bigint); + else if (value?.$decimal) this.putRaw(value?.$decimal); else if (_isPlainObject(value) || _isArray(value)) this.putStringValue(JSON.stringify(value)); else this.put('^null'); } @@ -658,6 +668,68 @@ export class SqlDumper implements AlterProcessor { } } + sanitizeTableConstraints(table: TableInfo): TableInfo { + // Create a deep copy of the table + const sanitized = _cloneDeep(table); + + // Get the set of existing column names + const existingColumns = new Set(sanitized.columns.map(col => col.columnName)); + + // Filter primary key columns to only include existing columns + if (sanitized.primaryKey) { + const validPkColumns = sanitized.primaryKey.columns.filter(col => existingColumns.has(col.columnName)); + if (validPkColumns.length === 0) { + // If no valid columns remain, remove the primary key entirely + sanitized.primaryKey = null; + } else if (validPkColumns.length < sanitized.primaryKey.columns.length) { + // Update primary key with only valid columns + sanitized.primaryKey = { + ...sanitized.primaryKey, + columns: validPkColumns + }; + } + } + + // Filter sorting key columns to only include existing columns + if (sanitized.sortingKey) { + const validSkColumns = sanitized.sortingKey.columns.filter(col => existingColumns.has(col.columnName)); + if (validSkColumns.length === 0) { + sanitized.sortingKey = null; + } else if (validSkColumns.length < sanitized.sortingKey.columns.length) { + sanitized.sortingKey = { + ...sanitized.sortingKey, + columns: validSkColumns + }; + } + } + + // Filter foreign keys to only include those with all columns present + if (sanitized.foreignKeys) { + sanitized.foreignKeys = sanitized.foreignKeys.filter(fk => + fk.columns.every(col => existingColumns.has(col.columnName)) + ); + } + + // Filter indexes to only include those with all columns present + if (sanitized.indexes) { + sanitized.indexes = sanitized.indexes.filter(idx => + idx.columns.every(col => existingColumns.has(col.columnName)) + ); + } + + // Filter unique constraints to only include those with all columns present + if (sanitized.uniques) { + sanitized.uniques = sanitized.uniques.filter(uq => + uq.columns.every(col => existingColumns.has(col.columnName)) + ); + } + + // Filter dependencies (references from other tables) - these should remain as-is + // since they don't affect the CREATE TABLE statement for this table + + return sanitized; + } + recreateTable(oldTable: TableInfo, newTable: TableInfo) { if (!oldTable.pairingId || !newTable.pairingId || oldTable.pairingId != newTable.pairingId) { throw new Error('Recreate is not possible: oldTable.paringId != newTable.paringId'); @@ -672,48 +744,51 @@ export class SqlDumper implements AlterProcessor { })) .filter(x => x.newcol); + // Create a sanitized version of newTable with constraints that only reference existing columns + const sanitizedNewTable = this.sanitizeTableConstraints(newTable); + if (this.driver.supportsTransactions) { this.dropConstraints(oldTable, true); this.renameTable(oldTable, tmpTable); - this.createTable(newTable); + this.createTable(sanitizedNewTable); - const autoinc = newTable.columns.find(x => x.autoIncrement); + const autoinc = sanitizedNewTable.columns.find(x => x.autoIncrement); if (autoinc) { - this.allowIdentityInsert(newTable, true); + this.allowIdentityInsert(sanitizedNewTable, true); } this.putCmd( '^insert ^into %f (%,i) select %,i ^from %f', - newTable, + sanitizedNewTable, columnPairs.map(x => x.newcol.columnName), columnPairs.map(x => x.oldcol.columnName), { ...oldTable, pureName: tmpTable } ); if (autoinc) { - this.allowIdentityInsert(newTable, false); + this.allowIdentityInsert(sanitizedNewTable, false); } if (this.dialect.dropForeignKey) { - newTable.dependencies.forEach(cnt => this.createConstraint(cnt)); + sanitizedNewTable.dependencies.forEach(cnt => this.createConstraint(cnt)); } this.dropTable({ ...oldTable, pureName: tmpTable }); } else { // we have to preserve old table as long as possible - this.createTable({ ...newTable, pureName: tmpTable }); + this.createTable({ ...sanitizedNewTable, pureName: tmpTable }); this.putCmd( '^insert ^into %f (%,i) select %,s ^from %f', - { ...newTable, pureName: tmpTable }, + { ...sanitizedNewTable, pureName: tmpTable }, columnPairs.map(x => x.newcol.columnName), columnPairs.map(x => x.oldcol.columnName), oldTable ); this.dropTable(oldTable); - this.renameTable({ ...newTable, pureName: tmpTable }, newTable.pureName); + this.renameTable({ ...sanitizedNewTable, pureName: tmpTable }, newTable.pureName); } } diff --git a/packages/tools/src/alterPlan.ts b/packages/tools/src/alterPlan.ts index 5462efab8..a1a8dfb78 100644 --- a/packages/tools/src/alterPlan.ts +++ b/packages/tools/src/alterPlan.ts @@ -91,8 +91,8 @@ interface AlterOperation_RenameConstraint { } interface AlterOperation_RecreateTable { operationType: 'recreateTable'; - table: TableInfo; - operations: AlterOperation[]; + oldTable: TableInfo; + newTable: TableInfo; } interface AlterOperation_FillPreloadedRows { operationType: 'fillPreloadedRows'; @@ -249,11 +249,11 @@ export class AlterPlan { }); } - recreateTable(table: TableInfo, operations: AlterOperation[]) { + recreateTable(oldTable: TableInfo, newTable: TableInfo) { this.operations.push({ operationType: 'recreateTable', - table, - operations, + oldTable, + newTable, }); this.recreates.tables += 1; } @@ -337,7 +337,13 @@ export class AlterPlan { return opRes; }), op, - ]; + ].filter(op => { + // filter duplicated drops + const existingDrop = this.operations.find( + o => o.operationType == 'dropConstraint' && o.oldObject === op['oldObject'] + ); + return existingDrop == null; + }); return res; } @@ -498,53 +504,121 @@ export class AlterPlan { return []; } - const table = this.wholeNewDb.tables.find( + const oldTable = this.wholeOldDb.tables.find( + x => x.pureName == op[objectField].pureName && x.schemaName == op[objectField].schemaName + ); + const newTable = this.wholeNewDb.tables.find( x => x.pureName == op[objectField].pureName && x.schemaName == op[objectField].schemaName ); this.recreates.tables += 1; return [ { operationType: 'recreateTable', - table, - operations: [op], + oldTable, + newTable, + // operations: [op], }, ]; } return null; } - _groupTableRecreations(): AlterOperation[] { - const res = []; - const recreates = {}; + _removeRecreatedTableAlters(): AlterOperation[] { + const res: AlterOperation[] = []; + const recreates = new Set(); for (const op of this.operations) { - if (op.operationType == 'recreateTable' && op.table) { - const existingRecreate = recreates[`${op.table.schemaName}||${op.table.pureName}`]; - if (existingRecreate) { - existingRecreate.operations.push(...op.operations); - } else { - const recreate = { - ...op, - operations: [...op.operations], - }; - res.push(recreate); - recreates[`${op.table.schemaName}||${op.table.pureName}`] = recreate; - } - } else { - // @ts-ignore - const oldObject: TableInfo = op.oldObject || op.object; - if (oldObject) { - const recreated = recreates[`${oldObject.schemaName}||${oldObject.pureName}`]; - if (recreated) { - recreated.operations.push(op); - continue; - } - } - res.push(op); + if (op.operationType == 'recreateTable' && op.oldTable && op.newTable) { + const key = `${op.oldTable.schemaName}||${op.oldTable.pureName}`; + recreates.add(key); } } + + for (const op of this.operations) { + switch (op.operationType) { + case 'createColumn': + case 'createConstraint': + { + const key = `${op.newObject.schemaName}||${op.newObject.pureName}`; + if (recreates.has(key)) { + // skip create inside recreated table + continue; + } + } + break; + case 'dropColumn': + case 'dropConstraint': + case 'changeColumn': + { + const key = `${op.oldObject.schemaName}||${op.oldObject.pureName}`; + if (recreates.has(key)) { + // skip drop/change inside recreated table + continue; + } + } + break; + case 'renameColumn': + { + const key = `${op.object.schemaName}||${op.object.pureName}`; + if (recreates.has(key)) { + // skip rename inside recreated table + continue; + } + } + break; + } + res.push(op); + } return res; } + _groupTableRecreations(): AlterOperation[] { + const res = []; + const recreates = new Set(); + for (const op of this.operations) { + if (op.operationType == 'recreateTable' && op.oldTable && op.newTable) { + const key = `${op.oldTable.schemaName}||${op.oldTable.pureName}`; + if (recreates.has(key)) { + // prevent duplicate recreates + continue; + } + recreates.add(key); + } + + res.push(op); + } + return res; + + // const res = []; + // const recreates = {}; + // for (const op of this.operations) { + // if (op.operationType == 'recreateTable' && op.table) { + // const existingRecreate = recreates[`${op.table.schemaName}||${op.table.pureName}`]; + // if (existingRecreate) { + // existingRecreate.operations.push(...op.operations); + // } else { + // const recreate = { + // ...op, + // operations: [...op.operations], + // }; + // res.push(recreate); + // recreates[`${op.table.schemaName}||${op.table.pureName}`] = recreate; + // } + // } else { + // // @ts-ignore + // const oldObject: TableInfo = op.oldObject || op.object; + // if (oldObject) { + // const recreated = recreates[`${oldObject.schemaName}||${oldObject.pureName}`]; + // if (recreated) { + // recreated.operations.push(op); + // continue; + // } + // } + // res.push(op); + // } + // } + // return res; + } + _moveForeignKeysToLast(): AlterOperation[] { if (!this.dialect.createForeignKey) { return this.operations; @@ -611,6 +685,8 @@ export class AlterPlan { // console.log('*****************OPERATIONS3', this.operations); + this.operations = this._removeRecreatedTableAlters(); + this.operations = this._moveForeignKeysToLast(); // console.log('*****************OPERATIONS4', this.operations); @@ -673,16 +749,16 @@ export function runAlterOperation(op: AlterOperation, processor: AlterProcessor) break; case 'recreateTable': { - const oldTable = generateTablePairingId(op.table); - const newTable = _.cloneDeep(oldTable); - const newDb = DatabaseAnalyser.createEmptyStructure(); - newDb.tables.push(newTable); - // console.log('////////////////////////////newTable1', newTable); - op.operations.forEach(child => runAlterOperation(child, new DatabaseInfoAlterProcessor(newDb))); - // console.log('////////////////////////////op.operations', op.operations); - // console.log('////////////////////////////op.table', op.table); - // console.log('////////////////////////////newTable2', newTable); - processor.recreateTable(oldTable, newTable); + // const oldTable = generateTablePairingId(op.table); + // const newTable = _.cloneDeep(oldTable); + // const newDb = DatabaseAnalyser.createEmptyStructure(); + // newDb.tables.push(newTable); + // // console.log('////////////////////////////newTable1', newTable); + // op.operations.forEach(child => runAlterOperation(child, new DatabaseInfoAlterProcessor(newDb))); + // // console.log('////////////////////////////op.operations', op.operations); + // // console.log('////////////////////////////op.table', op.table); + // // console.log('////////////////////////////newTable2', newTable); + processor.recreateTable(op.oldTable, op.newTable); } break; } diff --git a/packages/tools/src/diagramTools.ts b/packages/tools/src/diagramTools.ts index 8fded94c2..ccad4fef4 100644 --- a/packages/tools/src/diagramTools.ts +++ b/packages/tools/src/diagramTools.ts @@ -60,4 +60,4 @@ export function chooseTopTables(tables: TableInfo[], count: number, tableFilter: export const DIAGRAM_ZOOMS = [0.1, 0.15, 0.2, 0.3, 0.4, 0.5, 0.6, 0.8, 1, 1.25, 1.5, 1.75, 2]; -export const DIAGRAM_DEFAULT_WATERMARK = 'Powered by [dbgate.io](https://dbgate.io)'; +export const DIAGRAM_DEFAULT_WATERMARK = 'Powered by [dbgate.io](https://www.dbgate.io)'; diff --git a/packages/tools/src/filterBehaviours.ts b/packages/tools/src/filterBehaviours.ts index 61ebdf5a6..0eeb2f073 100644 --- a/packages/tools/src/filterBehaviours.ts +++ b/packages/tools/src/filterBehaviours.ts @@ -47,6 +47,7 @@ export const mongoFilterBehaviour: FilterBehaviour = { allowStringToken: true, allowNumberDualTesting: true, allowObjectIdTesting: true, + allowHexString: true, }; export const evalFilterBehaviour: FilterBehaviour = { diff --git a/packages/tools/src/nameTools.ts b/packages/tools/src/nameTools.ts index 617481a3e..e9bf3c26d 100644 --- a/packages/tools/src/nameTools.ts +++ b/packages/tools/src/nameTools.ts @@ -1,4 +1,5 @@ import _cloneDeep from 'lodash/cloneDeep'; +import _uniq from 'lodash/uniq'; import _isString from 'lodash/isString'; import type { ColumnInfo, @@ -75,9 +76,27 @@ export function findForeignKeyForColumn(table: TableInfo, column: ColumnInfo | s return (table.foreignKeys || []).find(fk => fk.columns.find(col => col.columnName == column.columnName)); } +export function getConflictingColumnNames(columns: ColumnInfo[]): Set { + const conflictingNames = new Set( + _uniq(columns.map(x => x.columnName).filter((item, index, arr) => arr.indexOf(item) !== index)) + ); + return conflictingNames; +} + export function makeUniqueColumnNames(res: ColumnInfo[]) { const usedNames = new Set(); + const conflictingNames = getConflictingColumnNames(res); for (let i = 0; i < res.length; i++) { + if ( + conflictingNames.has(res[i].columnName) && + res[i].pureName && + !usedNames.has(`${res[i].pureName}_${res[i].columnName}`) + ) { + res[i].columnName = `${res[i].pureName}_${res[i].columnName}`; + usedNames.add(res[i].columnName); + continue; + } + if (usedNames.has(res[i].columnName)) { let suffix = 2; while (usedNames.has(`${res[i].columnName}${suffix}`)) suffix++; @@ -111,3 +130,20 @@ export function fillConstraintNames(table: TableInfo, dialect: SqlDialect) { } return res; } + +export const DATA_FOLDER_NAMES = [ + { name: 'sql', label: 'SQL scripts' }, + { name: 'shell', label: 'Shell scripts' }, + { name: 'markdown', label: 'Markdown files' }, + { name: 'charts', label: 'Charts' }, + { name: 'query', label: 'Query designs' }, + { name: 'sqlite', label: 'SQLite files' }, + { name: 'duckdb', label: 'DuckDB files' }, + { name: 'diagrams', label: 'Diagrams' }, + { name: 'perspectives', label: 'Perspectives' }, + { name: 'impexp', label: 'Import/Export jobs' }, + { name: 'modtrans', label: 'Model transforms' }, + { name: 'datadeploy', label: 'Data deploy jobs' }, + { name: 'dbcompare', label: 'Database compare jobs' }, + { name: 'apps', label: 'Applications' }, +]; diff --git a/packages/tools/src/stringTools.ts b/packages/tools/src/stringTools.ts index 3f8675a8a..c5f24f690 100644 --- a/packages/tools/src/stringTools.ts +++ b/packages/tools/src/stringTools.ts @@ -9,6 +9,7 @@ import _isEmpty from 'lodash/isEmpty'; import _omitBy from 'lodash/omitBy'; import { DataEditorTypesBehaviour } from 'dbgate-types'; import isPlainObject from 'lodash/isPlainObject'; +import md5 from 'blueimp-md5'; export const MAX_GRID_TEXT_LENGTH = 1000; // maximum length of text in grid cell, longer text is truncated @@ -42,6 +43,20 @@ export function hexStringToArray(inputString) { return res; } +export function base64ToHex(base64String) { + const binaryString = atob(base64String); + const hexString = Array.from(binaryString, c => c.charCodeAt(0).toString(16).padStart(2, '0')).join(''); + return '0x' + hexString.toUpperCase(); +} + +export function hexToBase64(hexString) { + const binaryString = hexString + .match(/.{1,2}/g) + .map(byte => String.fromCharCode(parseInt(byte, 16))) + .join(''); + return btoa(binaryString); +} + export function parseCellValue(value, editorTypes?: DataEditorTypesBehaviour) { if (!_isString(value)) return value; @@ -53,8 +68,9 @@ export function parseCellValue(value, editorTypes?: DataEditorTypesBehaviour) { const mHex = value.match(/^0x([0-9a-fA-F][0-9a-fA-F])+$/); if (mHex) { return { - type: 'Buffer', - data: hexStringToArray(value.substring(2)), + $binary: { + base64: hexToBase64(value.substring(2)), + }, }; } } @@ -185,6 +201,26 @@ function stringifyJsonToGrid(value): ReturnType { return { value: '(JSON)', gridStyle: 'nullCellStyle' }; } +function formatNumberCustomSeparator(value, thousandsSeparator) { + const [intPart, decPart] = value.split('.'); + const intPartWithSeparator = intPart.replace(/\B(?=(\d{3})+(?!\d))/g, thousandsSeparator); + return decPart ? `${intPartWithSeparator}.${decPart}` : intPartWithSeparator; +} + +function formatCellNumber(value, gridFormattingOptions?: { thousandsSeparator?: string }) { + const separator = gridFormattingOptions?.thousandsSeparator; + if (_isNumber(value)) { + if (separator === 'none' || (value < 1000 && value > -1000)) return value.toString(); + if (separator === 'system') return value.toLocaleString(); + } + // fallback for system locale + if (separator === 'space' || separator === 'system') return formatNumberCustomSeparator(value.toString(), ' '); + if (separator === 'narrowspace') return formatNumberCustomSeparator(value.toString(), '\u202F'); + if (separator === 'comma') return formatNumberCustomSeparator(value.toString(), ','); + if (separator === 'dot') return formatNumberCustomSeparator(value.toString(), '.'); + return value.toString(); +} + export function stringifyCellValue( value, intent: @@ -195,7 +231,7 @@ export function stringifyCellValue( | 'exportIntent' | 'clipboardIntent', editorTypes?: DataEditorTypesBehaviour, - gridFormattingOptions?: { useThousandsSeparator?: boolean }, + gridFormattingOptions?: { thousandsSeparator?: string }, jsonParsedValue?: any ): { value: string; @@ -229,11 +265,26 @@ export function stringifyCellValue( if (value === true) return { value: 'true', gridStyle: 'valueCellStyle' }; if (value === false) return { value: 'false', gridStyle: 'valueCellStyle' }; - if (editorTypes?.parseHexAsBuffer) { - if (value?.type == 'Buffer' && _isArray(value.data)) { - return { value: '0x' + arrayToHexString(value.data), gridStyle: 'valueCellStyle' }; - } + if (value?.$binary?.base64) { + return { + value: base64ToHex(value.$binary.base64), + gridStyle: 'valueCellStyle', + }; } + + if (value?.$decimal) { + return { + value: formatCellNumber(value.$decimal, gridFormattingOptions), + gridStyle: 'valueCellStyle', + }; + } + + if (editorTypes?.parseHexAsBuffer) { + // if (value?.type == 'Buffer' && _isArray(value.data)) { + // return { value: '0x' + arrayToHexString(value.data), gridStyle: 'valueCellStyle' }; + // } + } + if (editorTypes?.parseObjectIdAsDollar) { if (value?.$oid) { switch (intent) { @@ -247,13 +298,13 @@ export function stringifyCellValue( } if (value?.$bigint) { return { - value: value.$bigint, + value: formatCellNumber(value.$bigint, gridFormattingOptions), gridStyle: 'valueCellStyle', }; } if (typeof value === 'bigint') { return { - value: value.toString(), + value: formatCellNumber(value.toString(), gridFormattingOptions), gridStyle: 'valueCellStyle', }; } @@ -328,13 +379,8 @@ export function stringifyCellValue( if (_isNumber(value)) { switch (intent) { case 'gridCellIntent': - return { - value: - gridFormattingOptions?.useThousandsSeparator && (value >= 10000 || value <= -10000) - ? value.toLocaleString() - : value.toString(), - gridStyle: 'valueCellStyle', - }; + const separator = gridFormattingOptions?.thousandsSeparator; + return { value: formatCellNumber(value, gridFormattingOptions), gridStyle: 'valueCellStyle' }; default: return { value: value.toString() }; } @@ -386,6 +432,9 @@ export function safeJsonParse(json, defaultValue?, logError = false) { if (_isArray(json) || _isPlainObject(json)) { return json; } + if (!json) { + return defaultValue; + } try { return JSON.parse(json); } catch (err) { @@ -423,6 +472,9 @@ export function shouldOpenMultilineDialog(value) { if (value?.$bigint) { return false; } + if (value?.$decimal) { + return false; + } if (_isPlainObject(value) || _isArray(value)) { return true; } @@ -478,6 +530,9 @@ export function getAsImageSrc(obj) { if (obj?.type == 'Buffer' && _isArray(obj?.data)) { return `data:image/png;base64, ${arrayBufferToBase64(obj?.data)}`; } + if (obj?.$binary?.base64) { + return `data:image/png;base64, ${obj.$binary.base64}`; + } if (_isString(obj) && (obj.startsWith('http://') || obj.startsWith('https://'))) { return obj; @@ -670,6 +725,9 @@ export function deserializeJsTypesFromJsonParse(obj) { if (value?.$bigint) { return BigInt(value.$bigint); } + if (value?.$decimal) { + return value.$decimal; + } }); } @@ -684,6 +742,9 @@ export function deserializeJsTypesReviver(key, value) { if (value?.$bigint) { return BigInt(value.$bigint); } + if (value?.$decimal) { + return value.$decimal; + } return value; } @@ -734,3 +795,12 @@ export function setSqlFrontMatter(text: string, data: { [key: string]: any }, ya const frontMatterContent = `-- >>>\n${yamlContentMapped}\n-- <<<\n`; return frontMatterContent + (textClean || ''); } + +export function shortenIdentifier(s: string, maxLength?: number) { + if (!maxLength || maxLength < 10) return s; + if (s.length <= maxLength) return s; + const hash = md5(s).substring(0, 8); + const partLength = Math.floor((maxLength - 9) / 2); + const restLength = maxLength - 10 - partLength; + return s.substring(0, partLength) + '_' + hash + '_' + s.substring(s.length - restLength); +} diff --git a/packages/tools/src/testPermission.ts b/packages/tools/src/testPermission.ts index 5727d726f..390146fbb 100644 --- a/packages/tools/src/testPermission.ts +++ b/packages/tools/src/testPermission.ts @@ -57,6 +57,12 @@ export function compilePermissions(permissions: string[] | string): CompiledPerm return res; } +export function getPermissionsCacheKey(permissions: string[] | string) { + if (!permissions) return null; + if (_isString(permissions)) return permissions; + return permissions.join('|'); +} + export function testPermission(tested: string, permissions: CompiledPermissions) { let allow = true; @@ -103,9 +109,25 @@ export function getPredefinedPermissions(predefinedRoleName: string) { case 'superadmin': return ['*', '~widgets/*', 'widgets/admin', 'widgets/database', '~all-connections']; case 'logged-user': - return ['*', '~widgets/admin', '~admin/*', '~internal-storage', '~all-connections']; + return [ + '*', + '~widgets/admin', + '~admin/*', + '~internal-storage', + '~all-connections', + '~run-shell-script', + '~all-team-files/*', + ]; case 'anonymous-user': - return ['*', '~widgets/admin', '~admin/*', '~internal-storage', '~all-connections']; + return [ + '*', + '~widgets/admin', + '~admin/*', + '~internal-storage', + '~all-connections', + '~run-shell-script', + '~all-team-files/*', + ]; default: return null; } diff --git a/packages/types/appdefs.d.ts b/packages/types/appdefs.d.ts index b9826bebf..549b9cd5c 100644 --- a/packages/types/appdefs.d.ts +++ b/packages/types/appdefs.d.ts @@ -1,12 +1,12 @@ -interface ApplicationCommand { - name: string; - sql: string; -} +// interface ApplicationCommand { +// name: string; +// sql: string; +// } -interface ApplicationQuery { - name: string; - sql: string; -} +// interface ApplicationQuery { +// name: string; +// sql: string; +// } interface VirtualReferenceDefinition { pureName: string; @@ -27,11 +27,31 @@ interface DictionaryDescriptionDefinition { delimiter: string; } -export interface ApplicationDefinition { - name: string; - - queries: ApplicationQuery[]; - commands: ApplicationCommand[]; - virtualReferences: VirtualReferenceDefinition[]; - dictionaryDescriptions: DictionaryDescriptionDefinition[]; +interface ApplicationUsageRule { + conditionGroup?: string; + serverHostsRegex?: string; + serverHostsList?: string[]; + databaseNamesRegex?: string; + databaseNamesList?: string[]; + tableNamesRegex?: string; + tableNamesList?: string[]; + columnNamesRegex?: string; + columnNamesList?: string[]; +} + +export interface ApplicationDefinition { + appid: string; + applicationName: string; + applicationIcon?: string; + applicationColor?: string; + usageRules?: ApplicationUsageRule[]; + files?: { + [key: string]: { + label: string; + sql: string; + type: 'query' | 'command'; + }; + }; + virtualReferences?: VirtualReferenceDefinition[]; + dictionaryDescriptions?: DictionaryDescriptionDefinition[]; } diff --git a/packages/types/dbinfo.d.ts b/packages/types/dbinfo.d.ts index ba4d33a4c..787de5c32 100644 --- a/packages/types/dbinfo.d.ts +++ b/packages/types/dbinfo.d.ts @@ -22,7 +22,7 @@ export interface ColumnsConstraintInfo extends ConstraintInfo { columns: ColumnReference[]; } -export interface PrimaryKeyInfo extends ColumnsConstraintInfo {} +export interface PrimaryKeyInfo extends ColumnsConstraintInfo { } export interface ForeignKeyInfo extends ColumnsConstraintInfo { refSchemaName?: string; @@ -39,7 +39,7 @@ export interface IndexInfo extends ColumnsConstraintInfo { filterDefinition?: string; } -export interface UniqueInfo extends ColumnsConstraintInfo {} +export interface UniqueInfo extends ColumnsConstraintInfo { } export interface CheckInfo extends ConstraintInfo { definition: string; @@ -77,6 +77,7 @@ export interface DatabaseObjectInfo extends NamedObjectInfo { hashCode?: string; objectTypeField?: string; objectComment?: string; + tablePermissionRole?: 'read' | 'update_only' | 'create_update_delete' | 'deny'; } export interface SqlObjectInfo extends DatabaseObjectInfo { @@ -134,7 +135,7 @@ export interface CallableObjectInfo extends SqlObjectInfo { parameters?: ParameterInfo[]; } -export interface ProcedureInfo extends CallableObjectInfo {} +export interface ProcedureInfo extends CallableObjectInfo { } export interface FunctionInfo extends CallableObjectInfo { returnType?: string; @@ -145,17 +146,17 @@ export interface TriggerInfo extends SqlObjectInfo { functionName?: string; tableName?: string; triggerTiming?: - | 'BEFORE' - | 'AFTER' - | 'INSTEAD OF' - | 'BEFORE EACH ROW' - | 'INSTEAD OF' - | 'AFTER EACH ROW' - | 'AFTER STATEMENT' - | 'BEFORE STATEMENT' - | 'AFTER EVENT' - | 'BEFORE EVENT' - | null; + | 'BEFORE' + | 'AFTER' + | 'INSTEAD OF' + | 'BEFORE EACH ROW' + | 'INSTEAD OF' + | 'AFTER EACH ROW' + | 'AFTER STATEMENT' + | 'BEFORE STATEMENT' + | 'AFTER EVENT' + | 'BEFORE EVENT' + | null; triggerLevel?: 'ROW' | 'STATEMENT'; eventType?: 'INSERT' | 'UPDATE' | 'DELETE' | 'TRUNCATE'; } diff --git a/packages/types/dialect.d.ts b/packages/types/dialect.d.ts index b06a30071..9f6099290 100644 --- a/packages/types/dialect.d.ts +++ b/packages/types/dialect.d.ts @@ -22,6 +22,7 @@ export interface SqlDialect { requireStandaloneSelectForScopeIdentity?: boolean; allowMultipleValuesInsert?: boolean; useServerDatabaseFile?: boolean; + maxIdentifierLength?: number; dropColumnDependencies?: string[]; changeColumnDependencies?: string[]; diff --git a/packages/types/dumper.d.ts b/packages/types/dumper.d.ts index 746cbbfc2..c5f380a22 100644 --- a/packages/types/dumper.d.ts +++ b/packages/types/dumper.d.ts @@ -16,6 +16,7 @@ export interface SqlDumper extends AlterProcessor { transform(type: TransformType, dumpExpr: () => void); createDatabase(name: string); dropDatabase(name: string); + comment(value: string); callableTemplate(func: CallableObjectInfo); diff --git a/packages/types/engines.d.ts b/packages/types/engines.d.ts index 4d8fd53b5..5556b5b10 100644 --- a/packages/types/engines.d.ts +++ b/packages/types/engines.d.ts @@ -21,6 +21,7 @@ export interface StreamOptions { error?: (error) => void; done?: (result) => void; info?: (info) => void; + changedCurrentDatabase?: (database: string) => void; } export type CollectionOperationInfo = @@ -99,19 +100,46 @@ export interface SupportedDbKeyType { showItemList?: boolean; } +export type DatabaseProcess = { + processId: number; + connectionId: number; + client: string; + operation?: string; + namespace?: string; + command?: any; + runningTime: number; + state?: any; + waitingFor?: boolean; + locks?: any; + progress?: any; +}; + +export type DatabaseVariable = { + variable: string; + value: any; +}; + export interface SqlBackupDumper { run(); } -export interface SummaryColumn { - fieldName: string; - header: string; - dataType: 'string' | 'number' | 'bytes'; +export interface ServerSummaryDatabases { + rows: any[]; + columns: SummaryDatabaseColumn[]; } -export interface ServerSummaryDatabase {} + +export type SummaryDatabaseColumn = { + header: string; + fieldName: string; + type: 'data' | 'fileSize'; + filterable?: boolean; + sortable?: boolean; +}; + export interface ServerSummary { - columns: SummaryColumn[]; - databases: ServerSummaryDatabase[]; + processes: DatabaseProcess[]; + variables: DatabaseVariable[]; + databases: ServerSummaryDatabases; } export type CollectionAggregateFunction = 'count' | 'sum' | 'avg' | 'min' | 'max'; @@ -161,12 +189,12 @@ export interface FilterBehaviourProvider { getFilterBehaviour(dataType: string, standardFilterBehaviours: { [id: string]: FilterBehaviour }): FilterBehaviour; } -export interface DatabaseHandle { +export interface DatabaseHandle { client: TClient; database?: string; conid?: string; feedback?: (message: any) => void; - getDatabase?: () => any; + getDatabase?: () => TDataBase; connectionType?: string; treeKeySeparator?: string; } @@ -196,7 +224,7 @@ export interface RestoreDatabaseSettings extends BackupRestoreSettingsBase { inputFile: string; } -export interface EngineDriver extends FilterBehaviourProvider { +export interface EngineDriver extends FilterBehaviourProvider { engine: string; title: string; defaultPort?: number; @@ -210,6 +238,7 @@ export interface EngineDriver extends FilterBehaviourProvider { supportsDatabaseRestore?: boolean; supportsServerSummary?: boolean; supportsDatabaseProfiler?: boolean; + supportsIncrementalAnalysis?: boolean; requiresDefaultSortCriteria?: boolean; profilerFormatterFunction?: string; profilerTimestampFunction?: string; @@ -242,61 +271,88 @@ export interface EngineDriver extends FilterBehaviourProvider { defaultSocketPath?: string; authTypeLabel?: string; importExportArgs?: any[]; - connect({ server, port, user, password, database, connectionDefinition }): Promise>; - close(dbhan: DatabaseHandle): Promise; - query(dbhan: DatabaseHandle, sql: string, options?: QueryOptions): Promise; - stream(dbhan: DatabaseHandle, sql: string, options: StreamOptions); - readQuery(dbhan: DatabaseHandle, sql: string, structure?: TableInfo): Promise; - readJsonQuery(dbhan: DatabaseHandle, query: any, structure?: TableInfo): Promise; + connect({ + server, + port, + user, + password, + database, + connectionDefinition, + }): Promise>; + close(dbhan: DatabaseHandle): Promise; + query(dbhan: DatabaseHandle, sql: string, options?: QueryOptions): Promise; + stream(dbhan: DatabaseHandle, sql: string, options: StreamOptions); + readQuery(dbhan: DatabaseHandle, sql: string, structure?: TableInfo): Promise; + readJsonQuery(dbhan: DatabaseHandle, query: any, structure?: TableInfo): Promise; // eg. PostgreSQL COPY FROM stdin - writeQueryFromStream(dbhan: DatabaseHandle, sql: string): Promise; - writeTable(dbhan: DatabaseHandle, name: NamedObjectInfo, options: WriteTableOptions): Promise; + writeQueryFromStream(dbhan: DatabaseHandle, sql: string): Promise; + writeTable( + dbhan: DatabaseHandle, + name: NamedObjectInfo, + options: WriteTableOptions + ): Promise; analyseSingleObject( - dbhan: DatabaseHandle, + dbhan: DatabaseHandle, name: NamedObjectInfo, objectTypeField: keyof DatabaseInfo ): Promise; - analyseSingleTable(dbhan: DatabaseHandle, name: NamedObjectInfo): Promise; - getVersion(dbhan: DatabaseHandle): Promise<{ version: string; versionText?: string }>; - listDatabases(dbhan: DatabaseHandle): Promise< + analyseSingleTable(dbhan: DatabaseHandle, name: NamedObjectInfo): Promise; + getVersion(dbhan: DatabaseHandle): Promise<{ version: string; versionText?: string }>; + listDatabases(dbhan: DatabaseHandle): Promise< { name: string; + sizeOnDisk?: number; + empty?: boolean; }[] >; - loadKeys(dbhan: DatabaseHandle, root: string, filter?: string): Promise; - scanKeys(dbhan: DatabaseHandle, root: string, pattern: string, cursor: string, count: number): Promise; - exportKeys(dbhan: DatabaseHandle, options: {}): Promise; - loadKeyInfo(dbhan: DatabaseHandle, key): Promise; - loadKeyTableRange(dbhan: DatabaseHandle, key, cursor, count): Promise; + loadKeys(dbhan: DatabaseHandle, root: string, filter?: string): Promise; + scanKeys( + dbhan: DatabaseHandle, + root: string, + pattern: string, + cursor: string, + count: number + ): Promise; + exportKeys(dbhan: DatabaseHandle, options: {}): Promise; + loadKeyInfo(dbhan: DatabaseHandle, key): Promise; + loadKeyTableRange(dbhan: DatabaseHandle, key, cursor, count): Promise; loadFieldValues( - dbhan: DatabaseHandle, + dbhan: DatabaseHandle, name: NamedObjectInfo, field: string, search: string, dataType: string ): Promise; - analyseFull(dbhan: DatabaseHandle, serverVersion): Promise; - analyseIncremental(dbhan: DatabaseHandle, structure: DatabaseInfo, serverVersion): Promise; + analyseFull(dbhan: DatabaseHandle, serverVersion): Promise; + analyseIncremental( + dbhan: DatabaseHandle, + structure: DatabaseInfo, + serverVersion + ): Promise; dialect: SqlDialect; dialectByVersion(version): SqlDialect; createDumper(options = null): SqlDumper; - createBackupDumper(dbhan: DatabaseHandle, options): Promise; + createBackupDumper(dbhan: DatabaseHandle, options): Promise; getAuthTypes(): EngineAuthType[]; - readCollection(dbhan: DatabaseHandle, options: ReadCollectionOptions): Promise; - updateCollection(dbhan: DatabaseHandle, changeSet: any): Promise; + readCollection(dbhan: DatabaseHandle, options: ReadCollectionOptions): Promise; + updateCollection(dbhan: DatabaseHandle, changeSet: any): Promise; getCollectionUpdateScript(changeSet: any, collectionInfo: CollectionInfo): string; - createDatabase(dbhan: DatabaseHandle, name: string): Promise; - dropDatabase(dbhan: DatabaseHandle, name: string): Promise; + createDatabase(dbhan: DatabaseHandle, name: string): Promise; + dropDatabase(dbhan: DatabaseHandle, name: string): Promise; getQuerySplitterOptions(usage: 'stream' | 'script' | 'editor' | 'import'): any; - script(dbhan: DatabaseHandle, sql: string, options?: RunScriptOptions): Promise; - operation(dbhan: DatabaseHandle, operation: CollectionOperationInfo, options?: RunScriptOptions): Promise; + script(dbhan: DatabaseHandle, sql: string, options?: RunScriptOptions): Promise; + operation( + dbhan: DatabaseHandle, + operation: CollectionOperationInfo, + options?: RunScriptOptions + ): Promise; getNewObjectTemplates(): NewObjectTemplate[]; // direct call of dbhan.client method, only some methods could be supported, on only some drivers - callMethod(dbhan: DatabaseHandle, method, args); - serverSummary(dbhan: DatabaseHandle): Promise; - summaryCommand(dbhan: DatabaseHandle, command, row): Promise; - startProfiler(dbhan: DatabaseHandle, options): Promise; - stopProfiler(dbhan: DatabaseHandle, profiler): Promise; + callMethod(dbhan: DatabaseHandle, method, args); + serverSummary(dbhan: DatabaseHandle): Promise; + summaryCommand(dbhan: DatabaseHandle, command, row): Promise; + startProfiler(dbhan: DatabaseHandle, options): Promise; + stopProfiler(dbhan: DatabaseHandle, profiler): Promise; getRedirectAuthUrl(connection, options): Promise<{ url: string; sid: string }>; getAuthTokenFromCode(connection, options): Promise; getAccessTokenFromAuth(connection, req): Promise; @@ -313,7 +369,10 @@ export interface EngineDriver extends FilterBehaviourProvider { adaptTableInfo(table: TableInfo): TableInfo; // simple data type adapter adaptDataType(dataType: string): string; - listSchemas(dbhan: DatabaseHandle): Promise; + listSchemas(dbhan: DatabaseHandle): Promise; + listProcesses(dbhan: DatabaseHandle): Promise; + listVariables(dbhan: DatabaseHandle): Promise; + killProcess(dbhan: DatabaseHandle, pid: number): Promise; backupDatabaseCommand( connection: any, settings: BackupDatabaseSettings, @@ -337,7 +396,7 @@ export interface EngineDriver extends FilterBehaviourProvider { analyserClass?: any; dumperClass?: any; singleConnectionOnly?: boolean; - getLogDbInfo(dbhan: DatabaseHandle): { + getLogDbInfo(dbhan: DatabaseHandle): { database?: string; engine: string; conid?: string; diff --git a/packages/types/test-engines.d.ts b/packages/types/test-engines.d.ts index 9f4e2e3e9..74e58de59 100644 --- a/packages/types/test-engines.d.ts +++ b/packages/types/test-engines.d.ts @@ -96,4 +96,6 @@ export type TestEngineInfo = { }>; objects?: Array; + + binaryDataType?: string; }; diff --git a/packages/web/index.html.tpl b/packages/web/index.html.tpl index 040ba47de..d061ca7d2 100644 --- a/packages/web/index.html.tpl +++ b/packages/web/index.html.tpl @@ -26,12 +26,23 @@ + if (localStorage.getItem('currentThemeType') == 'dark') { + document.documentElement.style.setProperty('--theme-background', '#111'); + document.documentElement.style.setProperty('--theme-foreground', '#e3e3e3'); + } else { + document.documentElement.style.setProperty('--theme-background', '#fff'); + document.documentElement.style.setProperty('--theme-foreground', '#262626'); + } + diff --git a/packages/web/src/buttons/FormStyledButton.svelte b/packages/web/src/buttons/FormStyledButton.svelte index 484987e1b..c117f46c7 100644 --- a/packages/web/src/buttons/FormStyledButton.svelte +++ b/packages/web/src/buttons/FormStyledButton.svelte @@ -9,6 +9,7 @@ export let title = null; export let skipWidth = false; export let outline = false; + export let colorClass = ''; function handleClick() { if (!disabled) dispatch('click'); @@ -31,6 +32,8 @@ bind:this={domButton} class:skipWidth class:outline + class={colorClass} + class:setBackgroundColor={!colorClass} /> diff --git a/packages/web/src/buttons/InlineUploadButton.svelte b/packages/web/src/buttons/InlineUploadButton.svelte index 9c6d8bd92..7d1c6aa32 100644 --- a/packages/web/src/buttons/InlineUploadButton.svelte +++ b/packages/web/src/buttons/InlineUploadButton.svelte @@ -5,6 +5,7 @@ import getElectron from '../utility/getElectron'; import InlineButtonLabel from '../buttons/InlineButtonLabel.svelte'; import resolveApi, { resolveApiHeaders } from '../utility/resolveApi'; + import { _t } from '../translations'; import uuidv1 from 'uuid/v1'; @@ -49,11 +50,11 @@ {#if electron} - + {:else} - {}} title="Upload file" data-testid={$$props['data-testid']} htmlFor={inputId}> + {}} title={_t('files.uploadFile', { defaultMessage: "Upload file" })} data-testid={$$props['data-testid']} htmlFor={inputId}> {/if} diff --git a/packages/web/src/buttons/NewObjectButton.svelte b/packages/web/src/buttons/NewObjectButton.svelte index 61d6bb5cd..17fd1944e 100644 --- a/packages/web/src/buttons/NewObjectButton.svelte +++ b/packages/web/src/buttons/NewObjectButton.svelte @@ -1,6 +1,7 @@
diff --git a/packages/web/src/buttons/ToolStripDropDownButton.svelte b/packages/web/src/buttons/ToolStripDropDownButton.svelte index b84536163..e6bea59b6 100644 --- a/packages/web/src/buttons/ToolStripDropDownButton.svelte +++ b/packages/web/src/buttons/ToolStripDropDownButton.svelte @@ -20,7 +20,7 @@ } - + {label} diff --git a/packages/web/src/buttons/ToolStripExportButton.svelte b/packages/web/src/buttons/ToolStripExportButton.svelte index 6d800f757..07e7e9c26 100644 --- a/packages/web/src/buttons/ToolStripExportButton.svelte +++ b/packages/web/src/buttons/ToolStripExportButton.svelte @@ -23,7 +23,8 @@ import hasPermission from '../utility/hasPermission'; import ToolStripCommandButton from './ToolStripCommandButton.svelte'; import ToolStripDropDownButton from './ToolStripDropDownButton.svelte'; - + import _ from 'lodash'; + import { _tval } from '../translations'; export let quickExportHandlerRef = null; export let command = 'sqlDataGrid.export'; export let label = 'Export'; @@ -39,7 +40,7 @@ {#if hasPermission('dbops/export')} {#if quickExportHandlerRef} - + {:else} {/if} diff --git a/packages/web/src/buttons/UploadButton.svelte b/packages/web/src/buttons/UploadButton.svelte index 0d9a18a22..33b9f6484 100644 --- a/packages/web/src/buttons/UploadButton.svelte +++ b/packages/web/src/buttons/UploadButton.svelte @@ -1,6 +1,7 @@
- Upload file + {_t('files.uploadFile', { defaultMessage: "Upload file" })}
diff --git a/packages/web/src/celldata/FormCellView.svelte b/packages/web/src/celldata/FormCellView.svelte new file mode 100644 index 000000000..ca4a2254b --- /dev/null +++ b/packages/web/src/celldata/FormCellView.svelte @@ -0,0 +1,398 @@ + + +
+
+ {#if rowData} +
+ + + + + { + // @ts-ignore + notNull = e.target.checked; + setLocalStorage('dataGridCellDataFormNotNull', notNull ? 'true' : 'false'); + }} + /> + {_t('tableCell.hideNullValues', { defaultMessage: 'Hide NULL values' })} +
+ {/if} +
+ {#if !rowData} +
{_t('tableCell.noDataSelected', { defaultMessage: 'No data selected' })}
+ {:else} + {#each filteredFields as field (field.uniqueName)} +
+
+ handleEdit(field)} + >{_t('tableCell.edit', { defaultMessage: 'Edit' })} + +
+
handleClick(field)}> + {#if editingColumn === field.uniqueName} +
+ isChangedRef.set(true)} + on:keydown={e => handleKeyDown(e, field)} + on:blur={() => handleBlur(field)} + class="inline-editor" + /> +
+ {:else if field.hasMultipleValues} + ({_t('tableCell.multipleValues', { defaultMessage: 'Multiple values' })}) + {:else if isJsonValue(field.value)} + + {:else} + + {/if} +
+
+ {/each} + {/if} +
+
+
+ + diff --git a/packages/web/src/celldata/PictureCellView.svelte b/packages/web/src/celldata/PictureCellView.svelte index 1d6eff082..fe5b4efbc 100644 --- a/packages/web/src/celldata/PictureCellView.svelte +++ b/packages/web/src/celldata/PictureCellView.svelte @@ -10,6 +10,9 @@ if (value?.type == 'Buffer' && _.isArray(value?.data)) { return 'data:image/png;base64, ' + btoa(String.fromCharCode.apply(null, value?.data)); } + if (value?.$binary?.base64) { + return 'data:image/png;base64, ' + value.$binary.base64; + } return null; } catch (err) { console.log('Error showing picture', err); diff --git a/packages/web/src/celldata/TextCellView.svelte b/packages/web/src/celldata/TextCellView.svelte index 41f91391c..f775a2f0a 100644 --- a/packages/web/src/celldata/TextCellView.svelte +++ b/packages/web/src/celldata/TextCellView.svelte @@ -3,12 +3,21 @@ export let selection; export let wrap; + + $: singleSelection = selection?.length == 1 && selection?.[0]; + $: grider = singleSelection?.grider; + $: editable = grider?.editable ?? false; + + function setCellValue(value) { + if (!editable) return; + grider.setCellValue(singleSelection.row, singleSelection.column, value); + }