Add integration tests for StackOverflow page download with markdown and image support

jest.config.mjs

@@ -13,7 +13,8 @@ export default {
   testMatch: [
     '**/tests/unit/**/*.test.js',
     '**/tests/mock/**/*.test.js',
-    '**/tests/e2e/**/*.test.js'
+    '**/tests/e2e/**/*.test.js',
+    '**/tests/integration/**/*.test.js'
   ],
   setupFiles: ['./tests/jest.setup.mjs']
 };

src/browser.js

@@ -130,12 +130,25 @@ function createPuppeteerPageAdapter(page) {
  * @returns {PageAdapter}
  */
 function createPlaywrightPageAdapter(page) {
+  // Store user agent to apply before navigation
+  let userAgentToSet = null;
+
   return {
     async setExtraHTTPHeaders(headers) {
       await page.setExtraHTTPHeaders(headers);
     },
     async setUserAgent(userAgent) {
-      await page.setUserAgent(userAgent);
+      // Playwright doesn't have page.setUserAgent(),
+      // so we store it and use context.addInitScript or route
+      userAgentToSet = userAgent;
+      // Set user agent using route to intercept and modify
+      await page.route('**/*', (route) => {
+        const headers = route.request().headers();
+        if (userAgentToSet) {
+          headers['user-agent'] = userAgentToSet;
+        }
+        route.continue({ headers });
+      });
     },
     async setViewport(viewport) {
       // Playwright uses setViewportSize instead of setViewport

tests/integration/browser-engines.test.js

@@ -1,4 +1,5 @@
 import { createBrowser } from '../../src/browser.js';
+import { convertHtmlToMarkdown } from '../../src/lib.js';
 
 describe('Browser Engine Integration Tests', () => {
   describe('Puppeteer Engine', () => {
@@ -101,4 +102,102 @@ describe('Browser Engine Integration Tests', () => {
       await playwrightBrowser.close();
     }, 60000);
   });
+
+  describe('StackOverflow Page Download', () => {
+    const stackOverflowUrl = 'https://stackoverflow.com/questions/927358/how-do-i-undo-the-most-recent-local-commits-in-git';
+
+    describe('Puppeteer Engine', () => {
+      let browser;
+
+      beforeEach(async () => {
+        browser = await createBrowser('puppeteer');
+      });
+
+      afterEach(async () => {
+        if (browser) {
+          await browser.close();
+        }
+      });
+
+      it('can download StackOverflow page and convert to markdown', async () => {
+        const page = await browser.newPage();
+        await page.setViewport({ width: 1280, height: 800 });
+        await page.goto(stackOverflowUrl, { waitUntil: 'networkidle0', timeout: 60000 });
+
+        const html = await page.content();
+        expect(html).toBeTruthy();
+        expect(html.length).toBeGreaterThan(1000);
+
+        // Convert HTML to markdown
+        const markdown = convertHtmlToMarkdown(html, stackOverflowUrl);
+        expect(markdown).toBeTruthy();
+        expect(markdown.length).toBeGreaterThan(100);
+
+        // Verify markdown contains expected content
+        expect(markdown).toContain('How do I undo the most recent local commits in Git');
+        expect(markdown).toContain('git');
+      }, 90000);
+
+      it('can take a screenshot of StackOverflow page', async () => {
+        const page = await browser.newPage();
+        await page.setViewport({ width: 1280, height: 800 });
+        await page.goto(stackOverflowUrl, { waitUntil: 'networkidle0', timeout: 60000 });
+
+        const screenshot = await page.screenshot({ type: 'png' });
+        expect(screenshot).toBeInstanceOf(Buffer);
+        expect(screenshot.length).toBeGreaterThan(1000);
+
+        // Verify it's a valid PNG
+        const pngSignature = Buffer.from([0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A]);
+        expect(screenshot.slice(0, 8).equals(pngSignature)).toBe(true);
+      }, 90000);
+    });
+
+    describe('Playwright Engine', () => {
+      let browser;
+
+      beforeEach(async () => {
+        browser = await createBrowser('playwright');
+      });
+
+      afterEach(async () => {
+        if (browser) {
+          await browser.close();
+        }
+      });
+
+      it('can download StackOverflow page and convert to markdown', async () => {
+        const page = await browser.newPage();
+        await page.setViewport({ width: 1280, height: 800 });
+        await page.goto(stackOverflowUrl, { waitUntil: 'networkidle0', timeout: 60000 });
+
+        const html = await page.content();
+        expect(html).toBeTruthy();
+        expect(html.length).toBeGreaterThan(1000);
+
+        // Convert HTML to markdown
+        const markdown = convertHtmlToMarkdown(html, stackOverflowUrl);
+        expect(markdown).toBeTruthy();
+        expect(markdown.length).toBeGreaterThan(100);
+
+        // Verify markdown contains expected content
+        expect(markdown).toContain('How do I undo the most recent local commits in Git');
+        expect(markdown).toContain('git');
+      }, 90000);
+
+      it('can take a screenshot of StackOverflow page', async () => {
+        const page = await browser.newPage();
+        await page.setViewport({ width: 1280, height: 800 });
+        await page.goto(stackOverflowUrl, { waitUntil: 'networkidle0', timeout: 60000 });
+
+        const screenshot = await page.screenshot({ type: 'png' });
+        expect(screenshot).toBeInstanceOf(Buffer);
+        expect(screenshot.length).toBeGreaterThan(1000);
+
+        // Verify it's a valid PNG
+        const pngSignature = Buffer.from([0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A]);
+        expect(screenshot.slice(0, 8).equals(pngSignature)).toBe(true);
+      }, 90000);
+    });
+  });
 });