Handle pagination

import { goToUrl, withNetworkSettledWait } from "@intuned/browser";
import { BrowserContext, Page } from "playwright-core";

interface Params {
  maxPages: number;
}

interface Product {
  name: string;
  price: string;
}

// Extract products from the current page
async function extractDataFromCurrentPage(page: Page): Promise<Product[]> {
  const results: Product[] = [];
  const productCards = page.locator(".product-item");
  const count = await productCards.count();

  for (let i = 0; i < count; i++) {
    const card = productCards.nth(i);
    const name = await card.locator(".product-name").textContent();
    const price = await card.locator(".product-price").textContent();

    if (name && price) {
      results.push({
        name: name.trim(),
        price: price.trim(),
      });
    }
  }

  return results;
}

// Check if next page exists
async function hasNextPage(page: Page): Promise<boolean> {
  const nextButton = page.locator('a:has-text("Next page")');
  return (await nextButton.count()) > 0;
}

// Navigate to the next page

async function goToNextPage(page: Page): Promise<void> {
  const nextButton = page.locator('a:has-text("Next page")');
  await withNetworkSettledWait(
    async (page) => {
      await nextButton.click();
    },
    {
      page,
      timeoutInMs: 10000,
      maxInflightRequests: 0,
    }
  );
}

export default async function handler(
  params: Params | null,
  page: Page,
  context: BrowserContext
) {
  const maxPages = params?.maxPages ?? 5;
  await goToUrl({ page, url: "https://www.scrapingcourse.com/pagination" });

  const allProducts: Product[] = [];
  let currentPage = 0;

  while (currentPage < maxPages) {
    // Extract data from current page
    const results = await extractDataFromCurrentPage(page);
    console.log(
      `Extracted ${results.length} results from page ${currentPage + 1}`
    );
    allProducts.push(...results);

    // Check if there's a next page
    const canContinue = await hasNextPage(page);
    if (!canContinue) {
      console.log("No more pages available.");
      break;
    }

    currentPage++;
    if (currentPage >= maxPages) {
      break;
    }

    // Navigate to next page
    await goToNextPage(page);
  }

  return allProducts;
}

from playwright.async_api import Page
from typing import List, TypedDict
from intuned_browser import go_to_url, wait_for_network_settled


class Params(TypedDict):
    max_pages: int


class Product(TypedDict):
    name: str
    price: str


async def extract_data_from_current_page(page: Page) -> List[Product]:
    """Extract products from the current page."""
    results = []
    product_cards = page.locator(".product-item")
    count = await product_cards.count()

    for i in range(count):
        card = product_cards.nth(i)
        name = await card.locator(".product-name").text_content()
        price = await card.locator(".product-price").text_content()

        if name and price:
            results.append(
                {
                    "name": name.strip(),
                    "price": price.strip(),
                }
            )

    return results


async def has_next_page(page: Page) -> bool:
    """Check if next page exists."""
    next_button = page.locator('a:has-text("Next page")')
    return await next_button.count() > 0


@wait_for_network_settled()
async def go_to_next_page(page: Page) -> None:
    """Navigate to the next page."""
    next_button = page.locator('a:has-text("Next page")')
    await next_button.click()


async def automation(page: Page, params: Params | None = None, **_kwargs):
    max_pages = params.get("max_pages", 5) if params else 5
    await go_to_url(page=page, url="https://www.scrapingcourse.com/pagination")

    all_products = []
    current_page = 0

    while current_page < max_pages:
        # Extract data from current page
        results = await extract_data_from_current_page(page)
        print(f"Extracted {len(results)} results from page {current_page + 1}")
        all_products.extend(results)

        # Check if there's a next page
        can_continue = await has_next_page(page)
        if not can_continue:
            print("No more pages available.")
            break

        current_page += 1
        if current_page >= max_pages:
            break

        # Navigate to next page
        await go_to_next_page(page)

    return all_products