Implement Full Page Simulation: Realistic asset fetching and Page Load Time metrics

index.html

@@ -160,6 +160,21 @@
               >
             </div>
 
+            <!-- Full Page Simulation -->
+            <div class="form-group crawler-section">
+              <label class="checkbox-label">
+                <input
+                  type="checkbox"
+                  id="simulateAssets"
+                  class="form-checkbox"
+                />
+                <span>🖼️ Full Page Simulation (Load Images/CSS/JS)</span>
+              </label>
+              <small class="help-text"
+                >Calculates realistic "Total Page Load Time" by fetching assets</small
+              >
+            </div>
+
             <!-- Crawler Settings (shown when enabled) -->
             <div
               class="crawler-settings"
@@ -399,6 +414,21 @@
           </div>
         </div>
 
+        <!-- Real World Performance -->
+        <div class="panel-section mt-4" id="pageLoadSection" style="display: none">
+          <h3 class="section-title">Real World Performance</h3>
+          <div class="stats-grid">
+            <div class="stat-card">
+              <div class="stat-label">Avg. Page Load Time</div>
+              <div class="stat-value info" id="avgPageLoadTime">0ms</div>
+            </div>
+            <div class="stat-card">
+              <div class="stat-label">Asset Requests</div>
+              <div class="stat-value" id="totalAssetRequests">0</div>
+            </div>
+          </div>
+        </div>
+
         <!-- Bandwidth -->
         <div class="panel-section mt-4">
           <h3 class="section-title">Bandwidth Usage</h3>

script.js

@@ -62,6 +62,8 @@ class StressTestingTool {
       requestsPerSecond: [],
       workers: [], // Web Worker instances
       workerStats: new Map(), // Stats per worker
+      pageLoadTimes: [], // All page load times for percentiles
+      totalAssetRequests: 0,
       updateInterval: null,
       chartUpdateInterval: null,
       userErrorData: [],
@@ -164,6 +166,12 @@ class StressTestingTool {
       errorsNetwork: document.getElementById("errorsNetwork"),
       totalBandwidth: document.getElementById("totalBandwidth"),
 
+      // Page Load Simulation
+      simulateAssets: document.getElementById("simulateAssets"),
+      pageLoadSection: document.getElementById("pageLoadSection"),
+      avgPageLoadTime: document.getElementById("avgPageLoadTime"),
+      totalAssetRequests: document.getElementById("totalAssetRequests"),
+
       // Request history
       requestHistoryBody: document.getElementById("requestHistoryBody"),
 
@@ -747,6 +755,8 @@ class StressTestingTool {
         this.elements.linksPerPage?.value || 10
       );
     }
+
+    this.config.simulateAssets = this.elements.simulateAssets?.checked || false;
   }
 
   resetState() {
@@ -768,6 +778,8 @@ class StressTestingTool {
     };
     this.state.totalBytesSent = 0;
     this.state.totalBytesReceived = 0;
+    this.state.pageLoadTimes = [];
+    this.state.totalAssetRequests = 0;
     this.state.requestHistory = [];
     this.state.percentiles = { p50: 0, p95: 0, p99: 0 };
 
@@ -876,6 +888,20 @@ class StressTestingTool {
     this.state.totalBytesReceived = bytesReceived;
     this.state.errorsByCategory = errors;
     this.state.responseTimes = allResponseTimes.slice(-1000); // Sample for percentiles
+
+    // Aggregate page load times and assets
+    let totalAssets = 0;
+    let newPageLoadTimes = [];
+    for (const stats of this.state.workerStats.values()) {
+      totalAssets += stats.totalAssetRequests || 0;
+      if (stats.pageLoadTimes && stats.pageLoadTimes.length > 0) {
+        newPageLoadTimes = newPageLoadTimes.concat(stats.pageLoadTimes);
+      }
+    }
+    this.state.totalAssetRequests = totalAssets;
+    if (newPageLoadTimes.length > 0) {
+      this.state.pageLoadTimes = this.state.pageLoadTimes.concat(newPageLoadTimes).slice(-1000);
+    }
   }
 
   addToRequestHistory(request) {
@@ -1010,6 +1036,19 @@ class StressTestingTool {
         this.state.totalBytesSent + this.state.totalBytesReceived;
       this.elements.totalBandwidth.textContent = formatBytes(totalBytes);
     }
+
+    // Update simulation metrics
+    if (this.config.simulateAssets) {
+      this.elements.pageLoadSection.style.display = 'block';
+      this.elements.totalAssetRequests.textContent = this.state.totalAssetRequests.toLocaleString();
+
+      const avgPageLoad = this.state.pageLoadTimes.length > 0
+        ? Math.round(this.state.pageLoadTimes.reduce((a, b) => a + b, 0) / this.state.pageLoadTimes.length)
+        : 0;
+      this.elements.avgPageLoadTime.textContent = `${avgPageLoad}ms`;
+    } else {
+      this.elements.pageLoadSection.style.display = 'none';
+    }
   }
 
   updateCharts() {
@@ -1211,8 +1250,17 @@ class StressTestingTool {
       "Error Threshold": this.state.errorThreshold
         ? `${this.state.errorThreshold.users} users at ${this.state.errorThreshold.time}s (${this.state.errorThreshold.errorRate}% error rate)`
         : "No errors detected",
+      "Full Page Simulation": this.config.simulateAssets ? "Enabled" : "Disabled",
     };
 
+    if (this.config.simulateAssets) {
+      const avgPageLoad = this.state.pageLoadTimes.length > 0
+        ? Math.round(this.state.pageLoadTimes.reduce((a, b) => a + b, 0) / this.state.pageLoadTimes.length)
+        : 0;
+      results["Average Page Load Time"] = `${avgPageLoad}ms`;
+      results["Total Asset Requests"] = this.state.totalAssetRequests.toLocaleString();
+    }
+
     if (this.config.crawlerEnabled) {
       results["Unique URLs Visited"] = this.state.visitedUrls.size;
     }

worker.js

@@ -14,6 +14,8 @@ let state = {
     responseTimes: [],
     bytesSent: 0,
     bytesReceived: 0,
+    pageLoadTimes: [],
+    totalAssetRequests: 0,
     errorsByCategory: {
         "4xx": 0,
         "5xx": 0,
@@ -89,7 +91,21 @@ async function runUser(id) {
     let crawlDepth = 0;
 
     while (state.active && Date.now() < endTime) {
+        const pageLoadStart = performance.now();
         const result = await makeRequest(currentUrl);
+        let totalPageTime = result.responseTime;
+
+        // asset simulation
+        if (config.simulateAssets && result.success && result.body) {
+            const assets = extractAssets(result.body, currentUrl);
+            if (assets.length > 0) {
+                const assetResults = await fetchAssetsThrottled(assets);
+                const pageLoadEnd = performance.now();
+                totalPageTime = pageLoadEnd - pageLoadStart;
+                state.pageLoadTimes.push(totalPageTime);
+                state.totalAssetRequests += assets.length;
+            }
+        }
 
         // Report individual request for history log (sampled)
         if (Math.random() < 0.1 || config.userCount < 50) {
@@ -235,3 +251,68 @@ function extractRandomLink(html, baseUrl) {
     } catch (e) { }
     return null;
 }
+
+function extractAssets(html, baseUrl) {
+    const assets = [];
+    try {
+        // Regex for scripts, links (css), and images
+        const scriptRegex = /<script\b[^>]*src=["']([^"']+)["'][^>]*>/gi;
+        const linkRegex = /<link\b[^>]*href=["']([^"']+)["'][^>]*>/gi;
+        const imgRegex = /<img\b[^>]*src=["']([^"']+)["'][^>]*>/gi;
+
+        const extract = (regex) => {
+            let match;
+            while ((match = regex.exec(html)) !== null) {
+                try {
+                    const url = new URL(match[1], baseUrl).href;
+                    assets.push(url);
+                } catch (e) { }
+                if (assets.length > 20) break; // Limit per page for performance
+            }
+        };
+
+        extract(scriptRegex);
+        extract(linkRegex);
+        extract(imgRegex);
+    } catch (e) { }
+    return assets;
+}
+
+async function fetchAssetsThrottled(assets) {
+    const limit = 6; // Max concurrent connections like a browser
+    const results = [];
+
+    for (let i = 0; i < assets.length; i += limit) {
+        const batch = assets.slice(i, i + limit);
+        const promises = batch.map(url => fetchAsset(url));
+        results.push(...(await Promise.all(promises)));
+        if (!state.active) break;
+    }
+    return results;
+}
+
+async function fetchAsset(url) {
+    try {
+        const payload = JSON.stringify({
+            targetUrl: url,
+            method: 'GET',
+            headers: config.customHeaders
+        });
+
+        state.bytesSent += payload.length;
+
+        const response = await fetch(config.proxyUrl, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: payload
+        });
+
+        const data = await response.json();
+        if (data.body) {
+            state.bytesReceived += data.body.length;
+        }
+        return data.success;
+    } catch (e) {
+        return false;
+    }
+}