diff --git a/.gitignore b/.gitignore index 4ca65c5..f43d2ac 100644 --- a/.gitignore +++ b/.gitignore @@ -204,3 +204,8 @@ coverage/ *password* # Notes to self: NEVER commit files with these patterns! + +# =========================================== +# IGNORED DATA FOLDERS +# =========================================== +SAMPLE/ diff --git a/deploy-sync-upgrade-b64.ps1 b/deploy-sync-upgrade-b64.ps1 new file mode 100644 index 0000000..e79cfa3 --- /dev/null +++ b/deploy-sync-upgrade-b64.ps1 @@ -0,0 +1,48 @@ +$Server = "172.16.69.219" +$User = "root" +$Pass = "Q4dv!Z`$nCe#`$OT&h" +$RemotePath = "/var/www/web-page-performance-test" + +function Send-File-B64 { + param($LocalFile, $RemotePath) + echo "📄 Sending $LocalFile (Base64)..." + + # Read file bytes and convert to Base64 string (single line) + $ContentBytes = [System.IO.File]::ReadAllBytes($LocalFile) + $B64 = [Convert]::ToBase64String($ContentBytes) + + # Send command: echo "B64" | base64 -d > path + $Command = "echo '$B64' | base64 -d > $RemotePath" + + # Execute via plink (non-interactive) + plink -batch -pw "$Pass" "$User@$Server" $Command + + if ($LASTEXITCODE -ne 0) { throw "Failed to send $LocalFile" } +} + +try { + # Send files using Base64 method + Send-File-B64 ".\auto-sync-robust.sh" "$RemotePath/auto-sync.sh" + Send-File-B64 ".\web-page-performance-test-sync.service" "/etc/systemd/system/web-page-performance-test-sync.service" + Send-File-B64 ".\web-page-performance-test-sync.timer" "/etc/systemd/system/web-page-performance-test-sync.timer" + + # Configure server + echo "⚙️ Configuring Systemd Timer on server..." + $Commands = @( + "chmod +x $RemotePath/auto-sync.sh", + "crontab -l | grep -v 'auto-sync.sh' | crontab -", # Remove old cron job + "systemctl daemon-reload", + "systemctl enable web-page-performance-test-sync.timer", + "systemctl start web-page-performance-test-sync.timer", + "systemctl status web-page-performance-test-sync.timer --no-pager", + "echo '✅ Systemd Timer Upgrade Complete!'" + ) + $CommandStr = $Commands -join " && " + + plink -batch -pw "$Pass" "$User@$Server" $CommandStr + +} +catch { + echo "❌ Error: $_" + exit 1 +} diff --git a/index.html b/index.html index 44828be..37dd8d4 100644 --- a/index.html +++ b/index.html @@ -91,6 +91,20 @@
TBT (ms)
+ + + + + + diff --git a/lib/har-parser.js b/lib/har-parser.js new file mode 100644 index 0000000..b7a92dd --- /dev/null +++ b/lib/har-parser.js @@ -0,0 +1,201 @@ +/** + * HAR Parser - Extract network request data from Lighthouse results + * Provides timing, size, and metadata for waterfall visualization + */ + +/** + * Parse Lighthouse result and generate HAR-compatible data + * @param {object} lighthouseResult - Full Lighthouse result object + * @returns {object} HAR data structure + */ +function parseHAR(lighthouseResult) { + const audits = lighthouseResult.audits; + const networkRequests = audits['network-requests']?.details?.items || []; + + const entries = networkRequests.map((request, index) => { + const timing = calculateTiming(request); + + return { + requestId: index + 1, + url: request.url, + method: request.requestMethod || 'GET', + status: request.statusCode, + mimeType: request.mimeType || request.resourceType, + resourceType: request.resourceType, + protocol: request.protocol, + priority: request.priority, + + // Timing breakdown (in milliseconds) + timing: { + dns: timing.dns, + connect: timing.connect, + ssl: timing.ssl, + send: timing.send, + wait: timing.wait, + receive: timing.receive, + total: timing.total, + startTime: request.startTime, + endTime: request.endTime + }, + + // Size information (in bytes) + size: { + transferSize: request.transferSize || 0, + resourceSize: request.resourceSize || 0, + compressionRatio: request.resourceSize > 0 + ? (request.transferSize / request.resourceSize).toFixed(2) + : 1 + }, + + // Additional metadata + isSecure: request.url?.startsWith('https://'), + domain: extractDomain(request.url), + isThirdParty: checkIfThirdParty(request.url, lighthouseResult.finalUrl), + renderBlocking: request.renderBlocking === 'blocking', + + // Extracted from response headers if available + cacheControl: request.responseHeaders?.['cache-control'], + contentEncoding: request.responseHeaders?.['content-encoding'] + }; + }); + + return { + entries, + summary: calculateSummary(entries), + pageMetrics: extractPageMetrics(lighthouseResult) + }; +} + +/** + * Calculate timing breakdown for a single request + */ +function calculateTiming(request) { + const start = request.startTime || 0; + const end = request.endTime || start; + const total = (end - start) * 1000; // Convert to ms + + // Parse timing object if available + const timing = request.timing || {}; + + return { + dns: timing.dnsEnd >= 0 ? (timing.dnsEnd - timing.dnsStart) : -1, + connect: timing.connectEnd >= 0 ? (timing.connectEnd - timing.connectStart) : -1, + ssl: timing.sslEnd >= 0 ? (timing.sslEnd - timing.sslStart) : -1, + send: timing.sendEnd >= 0 ? (timing.sendEnd - timing.sendStart) : -1, + wait: timing.receiveHeadersEnd >= 0 ? (timing.receiveHeadersEnd - timing.sendEnd) : -1, + receive: -1, // Calculated from total - (dns + connect + ssl + send + wait) + total: total > 0 ? total : (request.networkEndTime - request.networkRequestTime) * 1000 || 0, + startTime: start, + endTime: end + }; +} + +/** + * Calculate summary statistics + */ +function calculateSummary(entries) { + const byType = {}; + let totalSize = 0; + let totalTransfer = 0; + + entries.forEach(entry => { + const type = entry.resourceType || 'Other'; + + if (!byType[type]) { + byType[type] = { count: 0, size: 0, transfer: 0 }; + } + + byType[type].count++; + byType[type].size += entry.size.resourceSize; + byType[type].transfer += entry.size.transferSize; + + totalSize += entry.size.resourceSize; + totalTransfer += entry.size.transferSize; + }); + + return { + totalRequests: entries.length, + totalSize, + totalTransfer, + byType, + compressionSavings: totalSize > 0 ? ((totalSize - totalTransfer) / totalSize * 100).toFixed(1) : 0 + }; +} + +/** + * Extract key page metrics from Lighthouse + */ +function extractPageMetrics(lighthouseResult) { + const audits = lighthouseResult.audits; + + return { + firstContentfulPaint: audits['first-contentful-paint']?.numericValue, + largestContentfulPaint: audits['largest-contentful-paint']?.numericValue, + cumulativeLayoutShift: audits['cumulative-layout-shift']?.numericValue, + totalBlockingTime: audits['total-blocking-time']?.numericValue, + speedIndex: audits['speed-index']?.numericValue, + timeToInteractive: audits['interactive']?.numericValue, + domContentLoaded: lighthouseResult.audits['metrics']?.details?.items?.[0]?.observedDomContentLoaded, + loadEventEnd: lighthouseResult.audits['metrics']?.details?.items?.[0]?.observedLoad + }; +} + +/** + * Extract domain from URL + */ +function extractDomain(url) { + try { + const urlObj = new URL(url); + return urlObj.hostname; + } catch { + return 'unknown'; + } +} + +/** + * Check if request is third-party + */ +function checkIfThirdParty(requestUrl, pageUrl) { + const requestDomain = extractDomain(requestUrl); + const pageDomain = extractDomain(pageUrl); + return requestDomain !== pageDomain; +} + +/** + * Group entries by domain + */ +function groupByDomain(entries) { + const domains = {}; + + entries.forEach(entry => { + const domain = entry.domain; + if (!domains[domain]) { + domains[domain] = { + count: 0, + size: 0, + transfer: 0, + requests: [] + }; + } + + domains[domain].count++; + domains[domain].size += entry.size.resourceSize; + domains[domain].transfer += entry.size.transferSize; + domains[domain].requests.push(entry.requestId); + }); + + // Sort by transfer size (descending) + return Object.entries(domains) + .sort((a, b) => b[1].transfer - a[1].transfer) + .reduce((acc, [domain, data]) => { + acc[domain] = data; + return acc; + }, {}); +} + +module.exports = { + parseHAR, + calculateTiming, + calculateSummary, + groupByDomain +}; diff --git a/lib/runner.js b/lib/runner.js index 4009818..6abfa8c 100644 --- a/lib/runner.js +++ b/lib/runner.js @@ -141,9 +141,15 @@ async function _executeTest(url, options) { isMobile: isMobile }; - // Save JSON Summary + // Save JSON Summary const jsonPath = path.join(reportDir, `${testId}.json`); fs.writeFileSync(jsonPath, JSON.stringify(summary, null, 2)); + + // Parse and save HAR data for waterfall + const harParser = require('./har-parser'); + const harData = harParser.parseHAR(lhr); + const harPath = path.join(reportDir, `${testId}.har.json`); + fs.writeFileSync(harPath, JSON.stringify(harData, null, 2)); await chrome.kill(); diff --git a/main.js b/main.js index 643ab58..5dcb62f 100644 --- a/main.js +++ b/main.js @@ -107,6 +107,16 @@ function displayResults(data) { `; resultsArea.appendChild(actionsDiv); + + // Show waterfall link + const waterfallContainer = document.getElementById('waterfall-link-container'); + if (waterfallContainer) { + waterfallContainer.style.display = 'block'; + document.getElementById('view-waterfall').onclick = (e) => { + e.preventDefault(); + window.open(`/waterfall.html?id=${data.id}`, '_blank'); + }; + } resultsArea.classList.add('visible'); diff --git a/package-lock.json b/package-lock.json index cee1c94..d9c8ebb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,11 +7,13 @@ "": { "name": "web-page-performance-test", "version": "1.0.0", + "hasInstallScript": true, "license": "MIT", "dependencies": { "chrome-launcher": "^1.2.1", "express": "^4.18.2", "lighthouse": "^13.0.1", + "pg": "^8.16.3", "uuid": "^13.0.0" }, "devDependencies": { @@ -2729,6 +2731,47 @@ "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==", "license": "MIT" }, + "node_modules/pg": { + "version": "8.16.3", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.16.3.tgz", + "integrity": "sha512-enxc1h0jA/aq5oSDMvqyW3q89ra6XIIDZgCX9vkMrnz5DFTw/Ny3Li2lFQ+pt3L6MCgm/5o2o8HW9hiJji+xvw==", + "license": "MIT", + "peer": true, + "dependencies": { + "pg-connection-string": "^2.9.1", + "pg-pool": "^3.10.1", + "pg-protocol": "^1.10.3", + "pg-types": "2.2.0", + "pgpass": "1.0.5" + }, + "engines": { + "node": ">= 16.0.0" + }, + "optionalDependencies": { + "pg-cloudflare": "^1.2.7" + }, + "peerDependencies": { + "pg-native": ">=3.0.1" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } + } + }, + "node_modules/pg-cloudflare": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.2.7.tgz", + "integrity": "sha512-YgCtzMH0ptvZJslLM1ffsY4EuGaU0cx4XSdXLRFae8bPP4dS5xL1tNB3k2o/N64cHJpwU7dxKli/nZ2lUa5fLg==", + "license": "MIT", + "optional": true + }, + "node_modules/pg-connection-string": { + "version": "2.9.1", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.9.1.tgz", + "integrity": "sha512-nkc6NpDcvPVpZXxrreI/FOtX3XemeLl8E0qFr6F2Lrm/I8WOnaWNhIPK2Z7OHpw7gh5XJThi6j6ppgNoaT1w4w==", + "license": "MIT" + }, "node_modules/pg-int8": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", @@ -2738,6 +2781,15 @@ "node": ">=4.0.0" } }, + "node_modules/pg-pool": { + "version": "3.10.1", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.10.1.tgz", + "integrity": "sha512-Tu8jMlcX+9d8+QVzKIvM/uJtp07PKr82IUOYEphaWcoBhIYkoHpLXN3qO59nAI11ripznDsEzEv8nUxBVWajGg==", + "license": "MIT", + "peerDependencies": { + "pg": ">=8.0" + } + }, "node_modules/pg-protocol": { "version": "1.10.3", "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.10.3.tgz", @@ -2760,6 +2812,15 @@ "node": ">=4" } }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "license": "MIT", + "dependencies": { + "split2": "^4.1.0" + } + }, "node_modules/picomatch": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", @@ -3359,6 +3420,15 @@ "node": ">=8.0" } }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "license": "ISC", + "engines": { + "node": ">= 10.x" + } + }, "node_modules/statuses": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", diff --git a/remote-debug.ps1 b/remote-debug.ps1 new file mode 100644 index 0000000..5f4d080 --- /dev/null +++ b/remote-debug.ps1 @@ -0,0 +1,25 @@ +$Server = "172.16.69.219" +$User = "root" +$Pass = "Q4dv!Z`$nCe#`$OT&h" + +function Remote-Exec { + param($Cmd) + echo "Running: $Cmd" + plink -batch -pw "$Pass" "$User@$Server" $Cmd +} + +echo "--- 1. Checking Disk Space ---" +Remote-Exec "df -h" + +echo "--- 2. Checking Node Version ---" +Remote-Exec "node -v && npm -v" + +echo "--- 3. Re-installing Dependencies (Verbose) ---" +Remote-Exec "cd /var/www/web-page-performance-test && npm install --verbose" + +echo "--- 4. Manual Server Start (Crash Test) ---" +# Run for 5 seconds then kill, or catch crash output +Remote-Exec "cd /var/www/web-page-performance-test && timeout 5s node server.js || echo 'Crash Detected'" + +echo "--- 5. Service Status ---" +Remote-Exec "systemctl status web-page-performance-test --no-pager" diff --git a/repro_concurrency.js b/repro_concurrency.js new file mode 100644 index 0000000..92a421f --- /dev/null +++ b/repro_concurrency.js @@ -0,0 +1,60 @@ +const https = require('https'); + +const URL = 'https://web-page-performance-test.beyondcloud.technology/api/run-test'; +const TEST_TARGET = 'https://example.com'; + +function startTest(id) { + return new Promise((resolve, reject) => { + console.log(`[${id}] Starting test...`); + const data = JSON.stringify({ + url: TEST_TARGET, + isMobile: true + }); + + const req = https.request(URL, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Content-Length': data.length, + 'x-user-uuid': `stress-test-${id}` + } + }, (res) => { + let body = ''; + res.on('data', chunk => body += chunk); + res.on('end', () => { + if (res.statusCode === 200) { + console.log(`[${id}] Success! Status: ${res.statusCode}`); + resolve(true); + } else { + console.error(`[${id}] Failed! Status: ${res.statusCode}`); + // console.error(`[${id}] Body: ${body}`); + resolve(false); + } + }); + }); + + req.on('error', (e) => { + console.error(`[${id}] Error: ${e.message}`); + resolve(false); + }); + + req.write(data); + req.end(); + }); +} + +async function run() { + console.log('Sending 2 concurrent requests...'); + const results = await Promise.all([ + startTest('A'), + startTest('B') + ]); + + if (results.every(r => r)) { + console.log('PASS: Both tests succeeded.'); + } else { + console.log('FAIL: At least one test failed.'); + } +} + +run(); diff --git a/test-db.js b/test-db.js new file mode 100644 index 0000000..2f3a136 --- /dev/null +++ b/test-db.js @@ -0,0 +1,18 @@ +const db = require('./lib/db'); + +(async () => { + console.log('Testing DB connectivity...'); + try { + const client = await db.pool.connect(); + console.log('Successfully connected to PostgreSQL!'); + client.release(); + + console.log('Initializing Schema...'); + await db.initSchema(); + console.log('Schema Check Complete.'); + } catch (err) { + console.error('DB Connection Failed:', err); + } finally { + process.exit(); + } +})(); diff --git a/waterfall.html b/waterfall.html new file mode 100644 index 0000000..ed32774 --- /dev/null +++ b/waterfall.html @@ -0,0 +1,203 @@ + + + + + + Waterfall View - Web Page Performance Test + + + + + +
+

Request Waterfall

+ +
+ + + + + + +
+ +
+
+ +
+
+
+

Request Details

+ +
+
+
+ + + + diff --git a/waterfall.js b/waterfall.js new file mode 100644 index 0000000..c9f2040 --- /dev/null +++ b/waterfall.js @@ -0,0 +1,230 @@ +/** + * Waterfall Chart Renderer + * Visualizes network requests timeline with interactive details + */ + +let currentHarData = null; +let currentFilter = 'all'; + +// Load HAR data from URL parameter +async function init() { + const params = new URLSearchParams(window.location.search); + const testId = params.get('id'); + + if (!testId) { + document.getElementById('waterfallCanvas').innerHTML = + '

Error: No test ID provided

'; + return; + } + + try { + const response = await fetch(`/reports/${testId}.har.json`); + if (!response.ok) throw new Error('HAR data not found'); + + currentHarData = await response.json(); + renderWaterfall(); + setupEventListeners(); + } catch (error) { + document.getElementById('waterfallCanvas').innerHTML = + `

Error loading waterfall data: ${error.message}

`; + } +} + +function renderWaterfall() { + const canvas = document.getElementById('waterfallCanvas'); + const entries = currentHarData.entries; + + if (!entries || entries.length === 0) { + canvas.innerHTML = '

No requests found

'; + return; + } + + // Calculate timeline scale + const maxTime = Math.max(...entries.map(e => e.timing.endTime)); + const scale = 1000 / maxTime; // pixels per second + + // Filter entries based on active filter + const filteredEntries = currentFilter === 'all' + ? entries + : entries.filter(e => e.resourceType === currentFilter); + + let html = ''; + + filteredEntries.forEach((entry, index) => { + const label = truncateUrl(entry.url, 50); + const timingBars = renderTimingBars(entry.timing, scale); + + html += ` +
+
${label}
+
${timingBars}
+
+ `; + }); + + canvas.innerHTML = html; + + // Attach click handlers + document.querySelectorAll('.waterfall-row').forEach(row => { + row.addEventListener('click', () => { + const requestId = parseInt(row.dataset.requestId); + showRequestDetails(requestId); + }); + }); +} + +function renderTimingBars(timing, scale) { + let html = ''; + let currentOffset = timing.startTime * scale; + + // DNS + if (timing.dns > 0) { + const width = timing.dns * scale / 1000; + html += `
`; + currentOffset += width; + } + + // Connect + if (timing.connect > 0) { + const width = timing.connect * scale / 1000; + html += `
`; + currentOffset += width; + } + + // SSL + if (timing.ssl > 0) { + const width = timing.ssl * scale / 1000; + html += `
`; + currentOffset += width; + } + + // Wait (TTFB) + if (timing.wait > 0) { + const width = timing.wait * scale / 1000; + html += `
`; + currentOffset += width; + } + + // Receive + const totalWidth = (timing.endTime - timing.startTime) * scale; + const receiveWidth = Math.max(totalWidth - (currentOffset - timing.startTime * scale), 0); + if (receiveWidth > 0) { + html += `
`; + } + + return html; +} + +function showRequestDetails(requestId) { + const entry = currentHarData.entries.find(e => e.requestId === requestId); + if (!entry) return; + + document.getElementById('dialogTitle').textContent = `Request #${requestId}`; + + const content = ` +
+

General

+
+ URL: + ${entry.url} +
+
+ Domain: + ${entry.domain} +
+
+ Method: + ${entry.method} +
+
+ Status: + ${entry.status} +
+
+ Type: + ${entry.resourceType} +
+
+ Protocol: + ${entry.protocol || 'N/A'} +
+
+ Priority: + ${entry.priority || 'N/A'} +
+ ${entry.isThirdParty ? '
⚠️ Third-Party Resource
' : ''} + ${entry.renderBlocking ? '
🚫 Render Blocking
' : ''} +
+ +
+

Timing

+ ${entry.timing.dns > 0 ? `
DNS Lookup:${entry.timing.dns.toFixed(2)} ms
` : ''} + ${entry.timing.connect > 0 ? `
Connection:${entry.timing.connect.toFixed(2)} ms
` : ''} + ${entry.timing.ssl > 0 ? `
SSL:${entry.timing.ssl.toFixed(2)} ms
` : ''} + ${entry.timing.wait > 0 ? `
Time to First Byte:${entry.timing.wait.toFixed(2)} ms
` : ''} +
+ Total Time: + ${entry.timing.total.toFixed(2)} ms +
+
+ +
+

Size

+
+ Transfer Size: + ${formatBytes(entry.size.transferSize)} +
+
+ Resource Size: + ${formatBytes(entry.size.resourceSize)} +
+ ${entry.size.compressionRatio < 1 ? ` +
+ Compression: + ${(entry.size.compressionRatio * 100).toFixed(1)}% (${formatBytes(entry.size.resourceSize - entry.size.transferSize)} saved) +
` : ''} +
+ `; + + document.getElementById('dialogContent').innerHTML = content; + document.getElementById('requestDialog').style.display = 'block'; + document.getElementById('dialogOverlay').style.display = 'block'; +} + +function setupEventListeners() { + // Filter buttons + document.querySelectorAll('.filter-btn').forEach(btn => { + btn.addEventListener('click', () => { + document.querySelectorAll('.filter-btn').forEach(b => b.classList.remove('active')); + btn.classList.add('active'); + currentFilter = btn.dataset.type; + renderWaterfall(); + }); + }); + + // Close dialog + document.getElementById('closeDialog').addEventListener('click', closeDialog); + document.getElementById('dialogOverlay').addEventListener('click', closeDialog); +} + +function closeDialog() { + document.getElementById('requestDialog').style.display = 'none'; + document.getElementById('dialogOverlay').style.display = 'none'; +} + +function truncateUrl(url, maxLength) { + if (url.length <= maxLength) return url; + const parts = url.split('/'); + return parts[parts.length - 1].substring(0, maxLength) + '...'; +} + +function formatBytes(bytes) { + if (bytes === 0) return '0 B'; + const k = 1024; + const sizes = ['B', 'KB', 'MB', 'GB']; + const i = Math.floor(Math.log(bytes) / Math.log(k)); + return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i]; +} + +// Initialize on page load +document.addEventListener('DOMContentLoaded', init);