11 e.preventDefault();
12 setLoading(true);
13 console.log(`Submitting URL for scraping: ${url}`);
14
15 const maxRetries = 3;
28 if (!response.ok) {
29 if (response.status === 400) {
30 throw new APIError("Bad request. Check your parameters.", response.status);
31 } else if (response.status === 401) {
32 throw new APIError("No more credit available. Please upgrade your plan.", response.status);
33 } else if (response.status === 404) {
34 throw new APIError("Requested URL not found.", response.status);
35 } else if (response.status === 429) {
36 throw new APIError("Too many concurrent requests. Please upgrade your plan.", response.status);
37 } else {
38 throw new APIError(`HTTP error! status: ${response.status}`, response.status);
39 }
40 }
44 break; // Exit the loop if successful
45 } catch (error) {
46 console.log(`Error occurred while scraping URL: ${url}. Error details: ${error.message}`);
47
48 if (error.name === "TypeError" && error.message === "Failed to fetch") {
76 />
77 <button type="submit" disabled={loading}>
78 {loading ? "Scraping..." : "Scrape"}
79 </button>
80 </form>
130async function scrapePage(url) {
131 console.log(`Starting to scrape page: ${url}`);
132 const apiKey = await getApiKey();
133
134 url = normalizeUrl(url);
135 console.log(`Normalized URL for scraping: ${url}`);
136
137 const extractRules = {
161 };
162
163 const result = await scrapePageWithRules(url, apiKey, extractRules);
164 return processResults(result);
165}
186}
187
188class APIError extends Error {
189 constructor(message, status) {
190 super(message);
191 this.name = "APIError";
192 this.status = status;
193 }
201}
202
203async function scrapePageWithRules(url, apiKey, extractRules) {
204 console.log(`Sending request to ScrapingBee API for URL: ${url}`);
205 try {
206 const response = await fetch(
207 `https://app.scrapingbee.com/api/v1/?api_key=${apiKey}&url=${
208 encodeURIComponent(url)
209 }&render_js=true&json_response=true&extract_rules=${encodeURIComponent(JSON.stringify(extractRules))}`,
211
212 if (!response.ok) {
213 throw new APIError(`HTTP error! status: ${response.status}`, response.status);
214 }
215
216 console.log("Response received from ScrapingBee API");
217 console.log(`Cost: ${response.headers.get("Spb-cost")}`);
218 console.log(`Initial status code: ${response.headers.get("Spb-initial-status-code")}`);
226 }
227
228 console.log("ScrapingBee API response parsed successfully");
229 return result;
230 } catch (error) {
231 if (error instanceof TypeError) {
232 console.error("Network error:", error.message);
233 throw new NetworkError("Failed to connect to the API");
234 } else if (error instanceof APIError) {
235 console.error("API error:", error.message, "Status:", error.status);
236 throw error;
237 } else if (error instanceof ParsingError) {
286}
287
288async function getApiKey() {
289 const apiKey = Deno.env.get("ScrapingBeeAPIkey");
290 if (!apiKey) {
291 console.log("ScrapingBee API key not found in environment variables");
292 throw new Error("ScrapingBee API key not found in environment variables");
293 }
294 return apiKey;
295}
296
302 const result = await scrapePage(url);
303 console.log(`Successfully scraped URL: ${url}`);
304 console.log(`Scraping completed for URL: ${url}
305 Result summary:
306 Title: ${result.title}
312 console.log(`Error occurred while processing request. Error details: ${error.message}`);
313 console.log(`Error stack trace: ${error.stack}`);
314 console.error(`Error occurred during scraping operation:
315 Error message: ${error.message}
316 Stack trace:
