Browser API: Headless scraping browser in cloud

const puppeteer = require('puppeteer-core');

async function main() {
  const connectionUri = process.env.MEGAINDEX_BROWSER_CDP_URL;

  if (!connectionUri) {
    throw new Error('MEGAINDEX_BROWSER_CDP_URL is not set');
  }

  const browser = await puppeteer.connect({
    browserWSEndpoint: connectionUri,
  });

  const page = await browser.newPage();
  await page.goto('https://example.com', {
    waitUntil: 'networkidle2',
    timeout: 60_000,
  });

  console.log(await page.title());
  await browser.disconnect();
}

main().catch((error) => {
  console.error(error);
  process.exit(1);
});

const { chromium } = require('playwright');

async function main() {
  const connectionUri = process.env.MEGAINDEX_BROWSER_CDP_URL;

  if (!connectionUri) {
    throw new Error('MEGAINDEX_BROWSER_CDP_URL is not set');
  }

  const browser = await chromium.connectOverCDP(connectionUri);
  const context = browser.contexts()[0];
  const page = await context.newPage();

  await page.goto('https://example.com', {
    waitUntil: 'networkidle',
    timeout: 60_000,
  });

  console.log(await page.title());
  await page.screenshot({ path: 'browser-api-test.png', fullPage: true });
  await browser.close();
}

main().catch((error) => {
  console.error(error);
  process.exit(1);
});

import asyncio
import os

from playwright.async_api import async_playwright

async def main() -> None:
    connection_uri = os.environ["MEGAINDEX_BROWSER_CDP_URL"]

    async with async_playwright() as playwright:
        browser = await playwright.chromium.connect_over_cdp(connection_uri)
        context = browser.contexts[0]
        page = await context.new_page()

        await page.goto(
            "https://example.com",
            wait_until="networkidle",
            timeout=60_000,
        )

        print(await page.title())
        await page.screenshot(path="browser-api-test.png", full_page=True)
        await browser.close()


asyncio.run(main())

import java.nio.file.Paths;

import com.microsoft.playwright.Browser;
import com.microsoft.playwright.BrowserContext;
import com.microsoft.playwright.Page;
import com.microsoft.playwright.Playwright;

public class BrowserApiExample {
    public static void main(String[] args) {
        String connectionUri = System.getenv("MEGAINDEX_BROWSER_CDP_URL");

        if (connectionUri == null || connectionUri.isBlank()) {
            throw new IllegalStateException("MEGAINDEX_BROWSER_CDP_URL is not set");
        }

        try (Playwright playwright = Playwright.create()) {
            Browser browser = playwright.chromium().connectOverCDP(connectionUri);
            BrowserContext context = browser.contexts().get(0);
            Page page = context.newPage();

            page.navigate("https://example.com");
            System.out.println(page.title());
            page.screenshot(new Page.ScreenshotOptions()
                .setPath(Paths.get("browser-api-test.png"))
                .setFullPage(true));

            browser.close();
        }
    }
}

using Microsoft.Playwright;

var connectionUri = Environment.GetEnvironmentVariable(
    "MEGAINDEX_BROWSER_CDP_URL"
) ?? throw new InvalidOperationException(
    "MEGAINDEX_BROWSER_CDP_URL is not set"
);

using var playwright = await Playwright.CreateAsync();
await using var browser = await playwright.Chromium.ConnectOverCDPAsync(
    connectionUri
);

var context = browser.Contexts[0];
var page = await context.NewPageAsync();

await page.GotoAsync("https://example.com", new PageGotoOptions
{
    WaitUntil = WaitUntilState.NetworkIdle,
    Timeout = 60_000,
});

Console.WriteLine(await page.TitleAsync());
await page.ScreenshotAsync(new PageScreenshotOptions
{
    Path = "browser-api-test.png",
    FullPage = true,
});

package main

import (
	"context"
	"fmt"
	"log"
	"os"
	"time"

	"github.com/chromedp/chromedp"
)

func main() {
	connectionURI := os.Getenv("MEGAINDEX_BROWSER_CDP_URL")
	if connectionURI == "" {
		log.Fatal("MEGAINDEX_BROWSER_CDP_URL is not set")
	}

	allocatorCtx, cancelAllocator := chromedp.NewRemoteAllocator(
		context.Background(),
		connectionURI,
		chromedp.NoModifyURL,
	)
	defer cancelAllocator()

	ctx, cancelBrowser := chromedp.NewContext(allocatorCtx)
	defer cancelBrowser()

	ctx, cancelTimeout := context.WithTimeout(ctx, 60*time.Second)
	defer cancelTimeout()

	var title string
	if err := chromedp.Run(
		ctx,
		chromedp.Navigate("https://example.com"),
		chromedp.Title(&title),
	); err != nil {
		log.Fatal(err)
	}

	fmt.Println(title)
}