6});
7
8const STEEL_API_KEY = Deno.env.get("STEEL_API_KEY");
9// Initialize Steel client with the API key from environment variables
10const client = new Steel({
11 steelAPIKey: STEEL_API_KEY,
12});
13
31 // Connect Puppeteer to the Steel session
32 browser = await puppeteer.connect({
33 browserWSEndpoint: `wss://connect.steel.dev?apiKey=${STEEL_API_KEY}&sessionId=${session.id}`,
34 });
35
21To run it:
22
231. Get your free Steel API key at https://app.steel.dev/settings/api-keys
242. Add it to your [Val Town Environment Variables](https://www.val.town/settings/environment-variables) as `STEEL_API_KEY`
253. Click `Fork` on this val
264. Click `Run` on this val
72
73- [Steel Documentation](https://docs.steel.dev)
74- [API Reference](https://docs.steel.dev/api-reference)
75- [Discord Community](https://discord.gg/gPpvhNvc5R)
1import * as linkify from "https://esm.sh/linkifyjs";
2
3const TWEETS_URL = "https://milei.nulo.lol/api/datasets/all-tweets.jsonl?limit=500";
4const ARCHIVO_API_URL = "https://archivo.nulo.lol/api/crawls";
5
6const ARCHIVO_API_TOKEN = Deno.env.get("ARCHIVO_API_TOKEN");
7
8async function fetchLatestTweetUrls(): Promise<string[]> {
23
24async function checkIfCrawled(url: string): Promise<boolean> {
25 const response = await fetch(ARCHIVO_API_URL, {
26 headers: {
27 "Authorization": ARCHIVO_API_TOKEN,
28 },
29 });
35async function crawlUrls(urls: string[]): Promise<void> {
36 if (urls.length === 0) return;
37 const res = await fetch(ARCHIVO_API_URL, {
38 method: "POST",
39 headers: {
40 "Content-Type": "application/json",
41 "Authorization": ARCHIVO_API_TOKEN,
42 },
43 body: JSON.stringify({ urls }),
34 const useMiddleware = Array.isArray(opts); // for backwards compat
35 const options: ReactStreamOptions = !Array.isArray(opts) ? opts : {};
36 const { api, getInitialProps } = options;
37
38 if (typeof document !== "undefined" && module) {
48 // DEPRECATED (for backwards compat)
49 options.robots && robots(options.robots),
50 options.api && deprecatedCustomAPI(options.api),
51 options.getInitialProps && deprecatedGetInitiaProps(options.getInitialProps),
52 // New custom middleware
127// DEPRECATED
128// DEPRECATE (for backwards compat)
129const deprecatedCustomAPI = (api?: RequestHandler): Middleware => async (req, res, next) => {
130 if (!api) return next();
131 if (req.method === "GET") return next();
132 return api(req);
133};
134const deprecatedGetInitiaProps = (getProps: DataFetcher<any>): Middleware => async (req, res, next) => {
142 /** DEPRECATED: Optional text response for robots.txt */
143 robots?: string;
144 /** DEPRECATED: Optional API request handler for all non-GET methods */
145 api?: RequestHandler;
146 /** DEPRECATED: data fetcher to set initial props based on request */
147 getInitialProps?: DataFetcher<any>;
34 const useMiddleware = Array.isArray(opts); // for backwards compat
35 const options: ReactStreamOptions = !Array.isArray(opts) ? opts : {};
36 const { api, getInitialProps } = options;
37
38 if (typeof document !== "undefined" && module) {
48 // DEPRECATED (for backwards compat)
49 options.robots && robots(options.robots),
50 options.api && deprecatedCustomAPI(options.api),
51 options.getInitialProps && deprecatedGetInitiaProps(options.getInitialProps),
52 // New custom middleware
127// DEPRECATED
128// DEPRECATE (for backwards compat)
129const deprecatedCustomAPI = (api?: RequestHandler): Middleware => async (req, res, next) => {
130 if (!api) return next();
131 if (req.method === "GET") return next();
132 return api(req);
133};
134const deprecatedGetInitiaProps = (getProps: DataFetcher<any>): Middleware => async (req, res, next) => {
142 /** DEPRECATED: Optional text response for robots.txt */
143 robots?: string;
144 /** DEPRECATED: Optional API request handler for all non-GET methods */
145 api?: RequestHandler;
146 /** DEPRECATED: data fetcher to set initial props based on request */
147 getInitialProps?: DataFetcher<any>;
1// RSS middleware for jxnblk.com
2
3// import data from "https://blog.jxnblk.com/api/all.json" with { type: "json" };
4import type { Middleware } from "https://esm.town/v/jxnblk/ReactStream";
5
11import { rss } from "./rss";
12
13import data from "https://blog.jxnblk.com/api/all.json?d4" with { type: "json" };
14
15interface Params {
34 const useMiddleware = Array.isArray(opts); // for backwards compat
35 const options: ReactStreamOptions = !Array.isArray(opts) ? opts : {};
36 const { api, getInitialProps } = options;
37
38 if (typeof document !== "undefined" && module) {
48 // DEPRECATED (for backwards compat)
49 options.robots && robots(options.robots),
50 options.api && deprecatedCustomAPI(options.api),
51 options.getInitialProps && deprecatedGetInitiaProps(options.getInitialProps),
52 // New custom middleware
127// DEPRECATED
128// DEPRECATE (for backwards compat)
129const deprecatedCustomAPI = (api?: RequestHandler): Middleware => async (req, res, next) => {
130 if (!api) return next();
131 if (req.method === "GET") return next();
132 return api(req);
133};
134const deprecatedGetInitiaProps = (getProps: DataFetcher<any>): Middleware => async (req, res, next) => {
142 /** DEPRECATED: Optional text response for robots.txt */
143 robots?: string;
144 /** DEPRECATED: Optional API request handler for all non-GET methods */
145 api?: RequestHandler;
146 /** DEPRECATED: data fetcher to set initial props based on request */
147 getInitialProps?: DataFetcher<any>;
1// RSS middleware for jxnblk.com
2
3// import data from "https://blog.jxnblk.com/api/all.json" with { type: "json" };
4import type { Middleware } from "https://esm.town/v/jxnblk/ReactStream";
5
11import { rss } from "./rss";
12
13import data from "https://blog.jxnblk.com/api/all.json?d4" with { type: "json" };
14
15interface Params {
