1/**
2 * Reddit /r/hardwareswap Scraper for Val Town
3 * Scrapes posts from Reddit and stores them in Supabase using Reddit's OAuth API
4 * Set this as a cron job to run periodically
5 */
72 const auth = btoa(`${this.clientId}:${this.clientSecret}`);
73
74 const response = await fetch("https://www.reddit.com/api/v1/access_token", {
75 method: "POST",
76 headers: {
101
102 /**
103 * Fetch new posts from /r/hardwareswap using Reddit's OAuth API
104 */
105 async fetchNewPosts(): Promise<RedditPost[]> {
128
129 if (!retryResponse.ok) {
130 throw new Error(`Reddit API error: ${retryResponse.status} ${retryResponse.statusText}`);
131 }
132
135 }
136
137 throw new Error(`Reddit API error: ${response.status} ${response.statusText}`);
138 }
139
147
148 /**
149 * Parse Reddit API response into our RedditPost format
150 */
151 private parseRedditResponse(data: any): RedditPost[] {
152 if (!data.data || !data.data.children) {
153 console.error("Unexpected Reddit API response format");
154 return [];
155 }
222
223 /**
224 * Main scraping function
225 */
226 async scrape(): Promise<void> {
227 const currentTime = new Date().toISOString();
228 console.log(`๐ Scraping /r/hardwareswap... at ${currentTime}`);
229
230 const posts = await this.fetchNewPosts();
35 const auth = btoa(`${clientId}:${clientSecret}`);
36
37 const tokenResponse = await fetch("https://www.reddit.com/api/v1/access_token", {
38 method: "POST",
39 headers: {
68 console.log("โ
OAuth token received");
69
70 // Step 2: Test API call
71 console.log("๐ก Testing API call...");
72 const apiResponse = await fetch("https://oauth.reddit.com/r/hardwareswap/new?limit=50", {
73 headers: {
74 "Authorization": `Bearer ${tokenData.access_token}`,
77 });
78
79 if (!apiResponse.ok) {
80 const errorText = await apiResponse.text();
81 return new Response(
82 JSON.stringify(
83 {
84 error: "API call failed",
85 status: apiResponse.status,
86 statusText: apiResponse.statusText,
87 response: errorText,
88 },
97 }
98
99 const apiData = await apiResponse.json();
100 console.log("โ
API call successful");
101
102 // Step 3: Parse and return results
103 const posts = apiData.data.children.map((child: any) => ({
104 id: child.data.id,
105 title: child.data.title.substring(0, 100) + (child.data.title.length > 100 ? "..." : ""),
114 {
115 success: true,
116 message: "Reddit OAuth and API test successful! ๐",
117 token_info: {
118 token_type: tokenData.token_type,
121 },
122 sample_posts: posts,
123 total_posts_available: apiData.data.children.length,
124 },
125 null,
5## Features
6
7- ๐ Automated scraping of /r/hardwareswap posts using Reddit's official OAuth API
8- ๐๏ธ Stores posts in Supabase PostgreSQL database
9- ๐ซ Duplicate detection to avoid storing the same post twice
14## Setup Instructions
15
16### 1. Reddit API Setup
17
181. Go to [Reddit App Preferences](https://www.reddit.com/prefs/apps)
322. Go to the SQL Editor in your Supabase dashboard
333. Copy and paste the contents of `database-schema.sql` and run it
344. Go to Settings > API to get your project URL and anon key
35
36### 3. Environment Variables
42- `SUPABASE_ANON_KEY`: Your Supabase anon/public key
43
44**Reddit API:**
45- `REDDIT_CLIENT_ID`: Your Reddit app's client ID
46- `REDDIT_CLIENT_SECRET`: Your Reddit app's client secret
813. Check for duplicates in the database
824. Save new posts to Supabase
835. Log statistics about the scraping session
84
85## Technical Details
911. Authenticates using your app's client ID and secret
922. Receives an access token from Reddit
933. Uses the token to make authenticated API requests
944. Automatically refreshes the token if it expires
95
107- Number of new posts scraped
108- Number of duplicates skipped
109- Any errors during scraping
110- Performance metrics
111
125- `Missing Reddit credentials`: Set REDDIT_CLIENT_ID and REDDIT_CLIENT_SECRET
126- `OAuth error`: Check your Reddit app credentials and app type
127- `Reddit API error`: May indicate rate limiting or API issues
128- `Error saving post`: Check Supabase connection and table schema
129
1/**
2 * HTTP API for Reddit Scraper
3 * Provides endpoints to manually trigger scraping and view statistics
4 */
5
62 const auth = btoa(`${this.clientId}:${this.clientSecret}`);
63
64 const response = await fetch('https://www.reddit.com/api/v1/access_token', {
65 method: 'POST',
66 headers: {
114
115 if (!retryResponse.ok) {
116 throw new Error(`Reddit API error: ${retryResponse.status} ${retryResponse.statusText}`);
117 }
118
121 }
122
123 throw new Error(`Reddit API error: ${response.status} ${response.statusText}`);
124 }
125
134 private parseRedditResponse(data: any): RedditPost[] {
135 if (!data.data || !data.data.children) {
136 console.error('Unexpected Reddit API response format');
137 return [];
138 }
293 <html>
294 <head>
295 <title>Reddit Scraper API</title>
296 <style>
297 body { font-family: Arial, sans-serif; max-width: 800px; margin: 0 auto; padding: 20px; }
301 </head>
302 <body>
303 <h1>๐ค Reddit Scraper API</h1>
304 <p>API endpoints for the /r/hardwareswap scraper</p>
305
306 <div class="endpoint">
307 <div class="method">GET /stats</div>
308 <div>Get scraping statistics and database info</div>
309 </div>
310
311 <div class="endpoint">
312 <div class="method">POST /scrape</div>
313 <div>Manually trigger a scraping session</div>
314 </div>
315
389 }
390 } catch (error) {
391 console.error('API Error:', error);
392 return new Response(JSON.stringify({
393 error: 'Internal server error',
6import { ValTransport } from "./mcp.tsx";
7
8const GEMINI_API_KEY = process.env.GEMINI_API_KEY;
9const QDRANT_API_KEY = process.env.QDRANT_API_KEY;
10
11const qdrant = new QdrantClient({
12 url: "https://05156014-0c24-4284-b10a-dea7700ec410.us-east4-0.gcp.cloud.qdrant.io:6333",
13 apiKey: QDRANT_API_KEY,
14});
15
16const ai = new GoogleGenAI({ apiKey: GEMINI_API_KEY });
17
18export async function handler(request: Request) {
97
98export async function sendDailyBriefing(chatId?: string, today?: DateTime) {
99 // Get API keys from environment
100 const apiKey = Deno.env.get("ANTHROPIC_API_KEY");
101 const telegramToken = Deno.env.get("TELEGRAM_TOKEN");
102
106 }
107
108 if (!apiKey) {
109 console.error("Anthropic API key is not configured.");
110 return;
111 }
122
123 // Initialize Anthropic client
124 const anthropic = new Anthropic({ apiKey });
125
126 // Initialize Telegram bot
162
163 // disabled title for now, it seemes unnecessary...
164 // await bot.api.sendMessage(chatId, `*${title}*`, { parse_mode: "Markdown" });
165
166 // Then send the main content
169
170 if (content.length <= MAX_LENGTH) {
171 await bot.api.sendMessage(chatId, content, { parse_mode: "Markdown" });
172 // Store the briefing in chat history
173 await storeChatMessage(
198 // Send each chunk as a separate message and store in chat history
199 for (const chunk of chunks) {
200 await bot.api.sendMessage(chatId, chunk, { parse_mode: "Markdown" });
201 // Store each chunk in chat history
202 await storeChatMessage(
53You'll need to set up some environment variables to make it run.
54
55- `ANTHROPIC_API_KEY` for LLM calls
56- You'll need to follow [these instructions](https://docs.val.town/integrations/telegram/) to make a telegram bot, and set `TELEGRAM_TOKEN`. You'll also need to get a `TELEGRAM_CHAT_ID` in order to have the bot remember chat contents.
57- For the Google Calendar integration you'll need `GOOGLE_CALENDAR_ACCOUNT_ID` and `GOOGLE_CALENDAR_CALENDAR_ID`. See [these instuctions](https://www.val.town/v/stevekrouse/pipedream) for details.
8## Hono
9
10This app uses [Hono](https://hono.dev/) as the API framework. You can think of Hono as a replacement for [ExpressJS](https://expressjs.com/) that works in serverless environments like Val Town or Cloudflare Workers. If you come from Python or Ruby, Hono is also a lot like [Flask](https://github.com/pallets/flask) or [Sinatra](https://github.com/sinatra/sinatra), respectively.
11
12## Serving assets to the frontend
20### `index.html`
21
22The most complicated part of this backend API is serving index.html. In this app (like most apps) we serve it at the root, ie `GET /`.
23
24We *bootstrap* `index.html` with some initial data from the server, so that it gets dynamically injected JSON data without having to make another round-trip request to the server to get that data on the frontend. This is a common pattern for client-side rendered apps.
25
26## CRUD API Routes
27
28This app has two CRUD API routes: for reading and inserting into the messages table. They both speak JSON, which is standard. They import their functions from `/backend/database/queries.ts`. These routes are called from the React app to refresh and update data.
29
30## Errors
31
32Hono and other API frameworks have a habit of swallowing up Errors. We turn off this default behavior by re-throwing errors, because we think most of the time you'll want to see the full stack trace instead of merely "Internal Server Error". You can customize how you want errors to appear.
8import { type Memory } from "../../shared/types.ts";
9
10const API_BASE = "/api/memories";
11const MEMORIES_PER_PAGE = 20;
12
71 setError(null);
72 try {
73 const response = await fetch(API_BASE);
74 if (!response.ok) {
75 throw new Error(`HTTP error! status: ${response.status}`);
100
101 try {
102 const response = await fetch(API_BASE, {
103 method: "POST",
104 headers: { "Content-Type": "application/json" },
123
124 try {
125 const response = await fetch(`${API_BASE}/${id}`, {
126 method: "DELETE",
127 });
155
156 try {
157 const response = await fetch(`${API_BASE}/${editingMemory.id}`, {
158 method: "PUT",
159 headers: { "Content-Type": "application/json" },
18});
19
20// --- API Routes for Memories ---
21
22// GET /api/memories - Retrieve all memories
23app.get("/api/memories", async (c) => {
24 const memories = await getAllMemories();
25 return c.json(memories);
26});
27
28// POST /api/memories - Create a new memory
29app.post("/api/memories", async (c) => {
30 const body = await c.req.json<Omit<Memory, "id">>();
31 if (!body.text) {
36});
37
38// PUT /api/memories/:id - Update an existing memory
39app.put("/api/memories/:id", async (c) => {
40 const id = c.req.param("id");
41 const body = await c.req.json<Partial<Omit<Memory, "id">>>();
58});
59
60// DELETE /api/memories/:id - Delete a memory
61app.delete("/api/memories/:id", async (c) => {
62 const id = c.req.param("id");
63 try {
75// --- Blob Image Serving Routes ---
76
77// GET /api/images/:filename - Serve images from blob storage
78app.get("/api/images/:filename", async (c) => {
79 const filename = c.req.param("filename");
80
