Загрузка и индексирование отдельных веб-страниц

// The instance that indexes the fetched page.
const INSTANCE_ID = "my-instance";

// Build a stable item key that ends in .html, so AI Search converts the HTML
// to Markdown before indexing it.
function itemKey(pageUrl) {
	const slug = `${pageUrl.hostname}${pageUrl.pathname}`
		.replace(/[^a-zA-Z0-9]+/g, "-")
		.replace(/^-+|-+$/g, "");
	return `${slug || "index"}.html`;
}

export default {
	async fetch(request, env) {
		const url = new URL(request.url);

		// Search route: query the indexed content and return the matching chunks.
		if (url.pathname === "/search") {
			const query = url.searchParams.get("q");
			if (!query) {
				return new Response("Add a ?q= query parameter", { status: 400 });
			}
			const results = await env.AI_SEARCH.get(INSTANCE_ID).search({ query });
			return Response.json({
				query: results.search_query,
				results: results.chunks.map((chunk) => ({
					key: chunk.item.key,
					score: chunk.score,
					text: chunk.text,
				})),
			});
		}

		// Index route: fetch a URL's rendered HTML and index it.
		const target = url.searchParams.get("url");
		if (!target) {
			return new Response(
				"Add a ?url= parameter to index a page, or use /search?q= to search",
				{ status: 400 },
			);
		}

		const pageUrl = new URL(target);

		// Fetch the fully rendered HTML with the Browser Run /content endpoint.
		// networkidle2 waits until the page has no more than two network
		// connections for at least 500 ms, giving client-side JavaScript time
		// to render the content.
		const response = await env.BROWSER.quickAction("content", {
			url: pageUrl.toString(),
			gotoOptions: {
				waitUntil: "networkidle2",
				timeout: 30000,
			},
		});

		if (!response.ok) {
			const detail = (await response.text()).slice(0, 500);
			return new Response(
				`Browser Run failed with ${response.status}: ${detail}`,
				{ status: 502 },
			);
		}

		// The /content endpoint returns a JSON envelope with the rendered HTML
		// in the result field.
		const data = await response.json();

		if (!data.success || typeof data.result !== "string") {
			return new Response("Browser Run returned an unsuccessful response", {
				status: 502,
			});
		}

		const html = data.result;

		// Upload the rendered HTML to built-in storage. uploadAndPoll waits until
		// the page is indexed and searchable.
		const item = await env.AI_SEARCH.get(INSTANCE_ID).items.uploadAndPoll(
			itemKey(pageUrl),
			html,
			{ timeoutMs: 60_000 },
		);

		return Response.json({ key: item.key, status: item.status });
	},
};

export interface Env {
	BROWSER: BrowserRun;
	AI_SEARCH: AiSearchNamespace;
}

// The instance that indexes the fetched page.
const INSTANCE_ID = "my-instance";

// Build a stable item key that ends in .html, so AI Search converts the HTML
// to Markdown before indexing it.
function itemKey(pageUrl: URL): string {
	const slug = `${pageUrl.hostname}${pageUrl.pathname}`
		.replace(/[^a-zA-Z0-9]+/g, "-")
		.replace(/^-+|-+$/g, "");
	return `${slug || "index"}.html`;
}

export default {
	async fetch(request, env): Promise<Response> {
		const url = new URL(request.url);

		// Search route: query the indexed content and return the matching chunks.
		if (url.pathname === "/search") {
			const query = url.searchParams.get("q");
			if (!query) {
				return new Response("Add a ?q= query parameter", { status: 400 });
			}
			const results = await env.AI_SEARCH.get(INSTANCE_ID).search({ query });
			return Response.json({
				query: results.search_query,
				results: results.chunks.map((chunk) => ({
					key: chunk.item.key,
					score: chunk.score,
					text: chunk.text,
				})),
			});
		}

		// Index route: fetch a URL's rendered HTML and index it.
		const target = url.searchParams.get("url");
		if (!target) {
			return new Response(
				"Add a ?url= parameter to index a page, or use /search?q= to search",
				{ status: 400 },
			);
		}

		const pageUrl = new URL(target);

		// Fetch the fully rendered HTML with the Browser Run /content endpoint.
		// networkidle2 waits until the page has no more than two network
		// connections for at least 500 ms, giving client-side JavaScript time
		// to render the content.
		const response = await env.BROWSER.quickAction("content", {
			url: pageUrl.toString(),
			gotoOptions: {
				waitUntil: "networkidle2",
				timeout: 30000,
			},
		});

		if (!response.ok) {
			const detail = (await response.text()).slice(0, 500);
			return new Response(
				`Browser Run failed with ${response.status}: ${detail}`,
				{ status: 502 },
			);
		}

		// The /content endpoint returns a JSON envelope with the rendered HTML
		// in the result field.
		const data = (await response.json()) as {
			success: boolean;
			result?: string;
		};

		if (!data.success || typeof data.result !== "string") {
			return new Response("Browser Run returned an unsuccessful response", {
				status: 502,
			});
		}

		const html = data.result;

		// Upload the rendered HTML to built-in storage. uploadAndPoll waits until
		// the page is indexed and searchable.
		const item = await env.AI_SEARCH.get(INSTANCE_ID).items.uploadAndPoll(
			itemKey(pageUrl),
			html,
			{ timeoutMs: 60_000 },
		);

		return Response.json({ key: item.key, status: item.status });
	},
} satisfies ExportedHandler<Env>;