Recipe
This recipe shows how to scrape data from pages with “Load More” buttons by clicking until the button disappears or reaches a maximum number of clicks.TypeScript
import { BrowserContext, Page, Locator } from "playwright";
interface BoxItem {
text: string;
}
// Click button until it disappears or max clicks reached
async function clickUntilExhausted(
buttonLocator: Locator,
maxClicks: number,
page:Page
): Promise<void> {
let clicks = 0;
while (clicks < maxClicks) {
await page.waitForTimeout(3000)
const isVisible = await buttonLocator.isVisible();
if (!isVisible) {
break;
}
await buttonLocator.click();
clicks++;
}
}
// Extract box items from the page
async function extractItems(page: Page): Promise<BoxItem[]> {
const results: BoxItem[] = [];
const boxes = page.locator("div.grid div.h-64");
const count = await boxes.count();
for (let i = 0; i < count; i++) {
const box = boxes.nth(i);
const text = await box.textContent();
if (text) {
results.push({
text: text.trim(),
});
}
}
return results;
}
export default async function handler(
params: { maxClicks?: number },
page: Page,
context: BrowserContext
) {
await page.goto("https://sandbox.intuned.dev/load-more");
// Locate the "Load More" button in main content area (not sidebar)
const loadMoreButton = page.locator("main.flex-1 button:has-text('Load More')");
// Click until button disappears or max clicks reached
await clickUntilExhausted(loadMoreButton, params.maxClicks ?? 50,page);
// Extract all items after content is loaded
const items = await extractItems(page);
console.log(`Extracted ${items.length} items`);
return items;
}
from playwright.async_api import Page, Locator
from typing import Any, Dict, List, Optional
async def click_until_exhausted(button_locator: Locator, max_clicks: int,page:Page) -> None:
"""Click button until it disappears or max clicks reached."""
clicks = 0
while clicks < max_clicks:
await page.wait_for_timeout(3000)
is_visible = await button_locator.is_visible()
if not is_visible:
break
await button_locator.click()
clicks += 1
async def extract_items(page: Page) -> List[str]:
"""Extract box items from the page."""
items = []
boxes = page.locator("div.grid div.h-64")
count = await boxes.count()
for i in range(count):
box = boxes.nth(i)
text = await box.text_content()
if text:
items.append(text.strip())
return items
async def automation(page: Page, params: Optional[Dict] = None, **_kwargs):
await page.goto("https://sandbox.intuned.dev/load-more")
# Wait for page to load
await page.wait_for_load_state("networkidle")
max_clicks = params.get("max_clicks", 50) if params else 50
# Locate the "Load More" button
load_more_button = page.locator("main.flex-1 button:has-text('Load More')")
# Click until button disappears or max clicks reached
await click_until_exhausted(load_more_button, max_clicks,page)
# Extract all items after content is loaded
items = await extract_items(page)
print(f"Extracted {len(items)} items")
return items
Related links
Cookbook (Python)
Python quick recipes in the Intuned Cookbook
Cookbook (TypeScript)
TypeScript quick recipes in the Intuned Cookbook